Do a tutorial - MNIST, tf 2, Keras API

Ulf Hamster 4 min.
python Keras API tensorflow2 mnist cnn tutorial

%%capture
!pip install tensorflow-gpu==2.0.0-beta1
import tensorflow as tf
print(tf.__version__)
print("GPU" if tf.test.is_gpu_available() else "CPU")
2.0.0-beta1
GPU

Load Dataset

data = tf.keras.datasets.mnist
(X_train, Y_train), (X_test, Y_test) = data.load_data()
# pre-processing
X_train = X_train / 255.
X_test = X_test / 255.
# Add a channels dimension
X_train = X_train[..., tf.newaxis]
X_test = X_test[..., tf.newaxis]
train_ds = tf.data.Dataset.from_tensor_slices(
    (X_train, Y_train)).shuffle(10000).batch(32)

test_ds = tf.data.Dataset.from_tensor_slices((X_test, Y_test)).batch(32)

EDA

X_train[0].shape
(28, 28, 1)
import matplotlib.pyplot as plt
%matplotlib inline

idx = 10
img = X_train[idx][:, : ,0]
plt.imshow(img, cmap='gray')

print(Y_train[idx])
3

png

Baseline Model

copied from tutorial

#?tf.keras.layers.Conv2D
print("specify neural net")
from tensorflow.keras import Model
from tensorflow.keras.layers import Conv2D, Dense, Flatten

class MyModel(Model):
    def __init__(self):
        super(MyModel, self).__init__()
        self.conv1 = Conv2D(filters=32, kernel_size=3, activation='relu')
        self.flatten = Flatten()
        self.d1 = Dense(units=128, activation='relu')
        self.d2 = Dense(units=10, activation='softmax')
        
    def call(self, x):
        x = self.conv1(x)
        x = self.flatten(x)
        x = self.d1(x)
        x = self.d2(x)
        return x


print("instantiate model")
model = MyModel()


print("specify loss function")
from tensorflow.keras.losses import SparseCategoricalCrossentropy
loss_fn = SparseCategoricalCrossentropy()


print("specify optimizer")
from tensorflow.keras.optimizers import Adam
optimizer = Adam()
specify neural net
instantiate model
specify loss function
specify optimizer
print("specify metrics")
train_loss = tf.keras.metrics.Mean(name='train_loss')
train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')
test_loss = tf.keras.metrics.Mean(name='test_loss')
test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')


print("training steps")
@tf.function
def train_step(images, labels):
    with tf.GradientTape() as tape:
        predictions = model(images)
        loss = loss_fn(labels, predictions)
    gradients = tape.gradient(loss, model.trainable_variables)
    optimizer.apply_gradients(zip(gradients, model.trainable_variables))

    train_loss(loss)
    train_accuracy(labels, predictions)
#train_step = tf.function(train_step)


print("test steps")
@tf.function
def test_step(images, labels):
    predictions = model(images)
    t_loss = loss_fn(labels, predictions)
    test_loss(t_loss)
    test_accuracy(labels, predictions)
#test_step = tf.function(test_step)


print("training loop")
EPOCHS = 5
for epoch in range(EPOCHS):
    for images, labels in train_ds:
        train_step(images, labels)

    for test_images, test_labels in test_ds:
        test_step(test_images, test_labels)

    template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'
    print(template.format(epoch+1,
                          train_loss.result(),
                          train_accuracy.result()*100,
                          test_loss.result(),
                          test_accuracy.result()*100))
specify metrics
training steps
test steps
training loop
Epoch 1, Loss: 0.12805062532424927, Accuracy: 96.17666625976562, Test Loss: 0.06109802424907684, Test Accuracy: 98.0199966430664
Epoch 2, Loss: 0.08283258974552155, Accuracy: 97.5199966430664, Test Loss: 0.061016496270895004, Test Accuracy: 98.0050048828125
Epoch 3, Loss: 0.06148386001586914, Accuracy: 98.14500427246094, Test Loss: 0.061450324952602386, Test Accuracy: 98.09667205810547
Epoch 4, Loss: 0.049275267869234085, Accuracy: 98.50083923339844, Test Loss: 0.06578657776117325, Test Accuracy: 98.07250213623047
Epoch 5, Loss: 0.041146039962768555, Accuracy: 98.74333190917969, Test Loss: 0.06371036171913147, Test Accuracy: 98.18199920654297

Grad Student Descent, Trial 1

print("specify neural net")
from tensorflow.keras import Model
from tensorflow.keras.layers import (
    Dense, Flatten, Dropout, Activation, BatchNormalization, MaxPooling2D)

class MyModel(Model):
    def __init__(self):
        super(MyModel, self).__init__()
        # 1st Conv Layer
        self.conv1 = Conv2D(filters=32, kernel_size=(3,3))  # use_bias=False
        self.act1 = Activation('relu')
        self.pool1 = MaxPooling2D()
        self.drop1 = Dropout(rate=0.1, seed=23)
        # 2nd Conv Layer
        self.conv2 = Conv2D(filters=32, kernel_size=(3,3))  # use_bias=False
        self.batch2 = BatchNormalization()
        self.act2 = Activation('relu')
        self.flat2 = Flatten()
        self.drop2 = Dropout(rate=0.1, seed=23)
        # 1st NN layer
        self.dense3 = Dense(units=128)
        self.batch3 = BatchNormalization()
        self.act3 = Activation('relu')
        self.drop3 = Dropout(rate=0.1, seed=23)
        # Output Layer
        self.dense4 = Dense(units=10)
        self.act4 = Activation('softmax')
        
    def call(self, x):
        # 1st Conv Layer
        x = self.conv1(x)
        x = self.act1(x)
        x = self.pool1(x)
        x = self.drop1(x)
        # 2nd Conv Layer
        x = self.conv2(x)
        x = self.act2(x)
        x = self.flat2(x)
        x = self.batch2(x)
        x = self.drop2(x)
        # 1st NN layer
        x = self.dense3(x)
        x = self.batch3(x)
        x = self.act3(x)
        x = self.drop3(x)
        # Output Layer
        x = self.dense4(x)
        x = self.act4(x)
        return x


print("instantiate model")
model = MyModel()


print("specify loss function")
from tensorflow.keras.losses import SparseCategoricalCrossentropy
loss_fn = SparseCategoricalCrossentropy()


print("specify optimizer")
from tensorflow.keras.optimizers import Adam
optimizer = Adam()
specify neural net
instantiate model
specify loss function
specify optimizer
print("specify metrics")
train_loss = tf.keras.metrics.Mean(name='train_loss')
train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')
test_loss = tf.keras.metrics.Mean(name='test_loss')
test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')


print("training steps")
@tf.function
def train_step(images, labels):
    with tf.GradientTape() as tape:
        predictions = model(images)
        loss = loss_fn(labels, predictions)
    gradients = tape.gradient(loss, model.trainable_variables)
    optimizer.apply_gradients(zip(gradients, model.trainable_variables))

    train_loss(loss)
    train_accuracy(labels, predictions)
#train_step = tf.function(train_step)


print("test steps")
@tf.function
def test_step(images, labels):
    predictions = model(images)
    t_loss = loss_fn(labels, predictions)
    test_loss(t_loss)
    test_accuracy(labels, predictions)
#test_step = tf.function(test_step)


print("training loop")
EPOCHS = 5
for epoch in range(EPOCHS):
    for images, labels in train_ds:
        train_step(images, labels)

    for test_images, test_labels in test_ds:
        test_step(test_images, test_labels)

    template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'
    print(template.format(epoch+1,
                          train_loss.result(),
                          train_accuracy.result()*100,
                          test_loss.result(),
                          test_accuracy.result()*100))
specify metrics
training steps
test steps
training loop
Epoch 1, Loss: 0.11910822242498398, Accuracy: 96.49166107177734, Test Loss: 0.05564611777663231, Test Accuracy: 98.13999938964844
Epoch 2, Loss: 0.07943907380104065, Accuracy: 97.625, Test Loss: 0.0474611334502697, Test Accuracy: 98.43500518798828
Epoch 3, Loss: 0.060903701931238174, Accuracy: 98.17277526855469, Test Loss: 0.04613471403717995, Test Accuracy: 98.5633316040039
Epoch 4, Loss: 0.050394900143146515, Accuracy: 98.48291778564453, Test Loss: 0.04583245515823364, Test Accuracy: 98.61500549316406
Epoch 5, Loss: 0.043013688176870346, Accuracy: 98.6969985961914, Test Loss: 0.045130882412195206, Test Accuracy: 98.69599914550781

References