Logistic Regression and MLP

Setup

import os
os.environ["KERAS_BACKEND"] = "torch"
import keras
print("Keras", keras.__version__, 'on', keras.backend.backend())
assert keras.__version__.startswith('3'), "Somehow got Keras 2; try stopping and restarting the session"
import numpy as np
np.set_printoptions(precision=3)
import matplotlib
import matplotlib.pyplot as plt
import pandas as pd

def show_weight_images(linear_layer):
    weights, bias = linear_layer.get_weights()
    weight_images = weights.T.reshape(-1, 28, 28)
    if weight_images.shape[0] == 10:
        nrow, ncol = 2, 5
    else:
        nrow, ncol = 1, 1
    with matplotlib.rc_context(rc={'image.cmap': 'RdBu'}):
        vmax = np.abs(weight_images).max()
        fig, axs = plt.subplots(nrow, ncol, squeeze=False)
        for i, ax in enumerate(axs.flatten()):
            ax.imshow(weight_images[i], vmax=vmax, vmin=-vmax)
            ax.set(title=f'{i}')
            ax.axis('off')

Keras 3.8.0 on torch

Data Loading

from keras.datasets import mnist

DATASET = 'mnist'

if DATASET == 'toy':
    x = np.array([0, 1, 2, 3])[:, np.newaxis]
    y_true = np.array([-1, .5, 2.0, 3.5])[:, np.newaxis]
elif DATASET == 'toy2':
    x = np.array([0, 1, 2, 3])[:, np.newaxis]
    y_true = np.array([-1, .5, 2.0, 25])[:, np.newaxis]
elif DATASET == "temps":
    data = pd.read_csv("https://data.giss.nasa.gov/gistemp/graphs_v4/graph_data/Global_Mean_Estimates_based_on_Land_and_Ocean_Data/graph.csv", skiprows=1)
    # Shape x to be items-by-features
    x = data.iloc[:, 0].values.astype(np.float32)[:, np.newaxis]
    # scale x to a reasonable range
    x -= 1880.0
    x /= 100.
    y_true = data.iloc[:, 1].values.astype(np.float32)[:, np.newaxis]
elif DATASET == 'mnist':
    (train_images, train_labels), (test_images, test_labels) = mnist.load_data()

    train_images = train_images.reshape((60000, 28 * 28))
    train_images = train_images.astype("float32") / 255
    test_images = test_images.reshape((10000, 28 * 28))
    test_images = test_images.astype("float32") / 255
    
    # Rename to match the convention of our notebook
    x, y_true = train_images, train_labels
    del train_images, train_labels
else:
    raise Exception("Unknown dataset")

Show the data (or an example item from it)

if len(x) < 50:
    plt.scatter(x, y_true)
elif x.shape[1] == 1:
    plt.plot(x, y_true)
else:
    with matplotlib.rc_context(rc={'image.cmap': 'gray_r'}):
        plt.imshow(x[0].reshape(28, 28))
print("x shape", x.shape, "y_true shape", y_true.shape)

x shape (60000, 784) y_true shape (60000,)

Train and Evaluate Model

model = keras.Sequential([
    keras.layers.Input(shape=(784,)),
    keras.layers.Dense(1)
])
model.compile(
    optimizer=keras.optimizers.SGD(learning_rate=0.001),
    loss='mse',
)
history = model.fit(x, y_true, epochs=10)

Epoch 1/10
1875/1875 ━━━━━━━━━━━━━━━━━━━━ 5s 3ms/step - loss: 5.8180
Epoch 2/10
1875/1875 ━━━━━━━━━━━━━━━━━━━━ 5s 3ms/step - loss: 3.9435
Epoch 3/10
1875/1875 ━━━━━━━━━━━━━━━━━━━━ 5s 3ms/step - loss: 3.7055
Epoch 4/10
1875/1875 ━━━━━━━━━━━━━━━━━━━━ 5s 3ms/step - loss: 3.5927
Epoch 5/10
1875/1875 ━━━━━━━━━━━━━━━━━━━━ 5s 2ms/step - loss: 3.4791
Epoch 6/10
1875/1875 ━━━━━━━━━━━━━━━━━━━━ 5s 3ms/step - loss: 3.3971
Epoch 7/10
1875/1875 ━━━━━━━━━━━━━━━━━━━━ 5s 3ms/step - loss: 3.3583
Epoch 8/10
1875/1875 ━━━━━━━━━━━━━━━━━━━━ 5s 3ms/step - loss: 3.3745
Epoch 9/10
1875/1875 ━━━━━━━━━━━━━━━━━━━━ 5s 2ms/step - loss: 3.3421
Epoch 10/10
1875/1875 ━━━━━━━━━━━━━━━━━━━━ 5s 3ms/step - loss: 3.3614

# Extract results from Keras
y_pred = model.predict(x)
losses = history.history['loss']
linear_layer = model.layers[-1]
weights, bias = linear_layer.get_weights()

# Plot the results
fig, axs = plt.subplots(ncols=2, figsize=(10, 4))
axs[0].plot(losses)
axs[0].set(title="Loss", xlabel="iteration", ylabel="Loss")
if x.shape[1] == 1:
    axs[1].scatter(x, y_true)
    axs[1].plot(x, y_pred, 'r');
    axs[1].set(title="Fitted model", xlabel='x', ylabel='y')
else:
    axs[1].hist(y_pred.flatten(), bins=30)
    axs[1].set(title="Values in predictions")
print(f"Final Loss: {losses[-1]:.3f}, final weights: {weights.flatten() if len(weights.flatten()) < 25 else weights.shape}, final bias: {bias}")

1875/1875 ━━━━━━━━━━━━━━━━━━━━ 1s 532us/step
Final Loss: 3.331, final weights: (784, 1), final bias: [2.798]

# show_weight_images(linear_layer)

Analysis