Perceptron Implementation From Scratch: Handwritten Text Parser

Gino Prasad

10/02/2022

import numpy as np
from pylab import imshow
import random
import tensorflow as tf
import matplotlib.pyplot as plt

train_data, val_data = tf.keras.datasets.mnist.load_data()

d = len(train_data[0][0].flatten())

q = 10

weights = np.array([[1 for _ in range(q)] for _ in range(d + 1)]).astype(np.int32)

transformation = lambda x: 1 if x != 0 else -1
transformation = np.vectorize(transformation)

x_train = np.array([transformation(arr.flatten()) for arr in train_data[0]], dtype=np.float32).astype(np.float64)

x_val = np.array([transformation(arr.flatten()) for arr in val_data[0]], dtype=np.float32).astype(np.float64)

x_train.dtype, x_val.dtype

(dtype('float64'), dtype('float64'))

(x_train).shape, x_val.shape

((60000, 784), (10000, 784))

Adding Bias Column

x_train = np.hstack((x_train, np.zeros((len(x_train), 1))))
x_val = np.hstack((x_val, np.zeros((len(x_val), 1))))

x_train.shape, weights.shape

((60000, 785), (785, 10))

y_train_raw = train_data[1].astype(np.int32)

y_val_raw = val_data[1].astype(np.int32)

Converting to One Hot

y_train = tf.cast(tf.one_hot(y_train_raw, depth=10), tf.float64)

Metal device set to: Apple M1

2022-12-20 13:35:05.563688: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:306] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.
2022-12-20 13:35:05.564549: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:272] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)

y_val = tf.cast(tf.one_hot(y_val_raw, depth=10), tf.float64)

x_train.shape, weights.shape

((60000, 785), (785, 10))

def predict(X, W):
    logits = tf.linalg.matmul(X, W)
    return tf.nn.softmax(
        logits, axis=1, name=None
    )

def batch_normalize(y_hat):
    sum_y_hat = tf.math.reduce_sum(
        y_hat, axis=0, keepdims=False, name=None
    )
    
    return tf.math.divide(
        sum_y_hat, len(y_hat), name=None
    )

def loss(X, W, Y):
    log_predictions = tf.math.log(predict(X, W))
    loss_sums = log_predictions * Y
    sum_loss_sums = tf.math.reduce_sum(
        loss_sums, axis=1, keepdims=False, name=None
    )
    return float(-batch_normalize(sum_loss_sums))

def delta(X, W, Y):
    predictions = predict(X, W)
    deltas = (predictions * Y) - Y
    return tf.linalg.matmul(tf.transpose(X), deltas) / len(X) * LEARNING_RATE

def train(X, W, Y, epochs=200):
    W = W.copy()
    for epoch in range(epochs):
        if not epoch % 10:
            print(loss(X, W, Y), loss(x_val, W, y_val))
        delta_epoch = delta(X,  W, Y)
        W -= delta_epoch
    return W

def accuracy(X, W, Y):
    y_hat = predict(X, W)
    y_hat_label = tf.math.argmax(
        y_hat,
        axis=1,
    )
    y_label = tf.math.argmax(
        Y,
        axis=1
    )
    return sum(np.array(y_hat_label == y_label).astype(int)) / len(y_hat_label)

def classify(img, W):
    preprocessed_img = np.expand_dims(np.hstack((transformation(img.flatten()), [0])), axis=0).astype(np.float64)
    return int(tf.math.argmax(predict(preprocessed_img, final_weights)[0]))

def plot_classify(img, W):
    imshow(img)
    plt.show()
    print(f"Predicted: {classify(img, W)}")

LEARNING_RATE=0.2

final_weights = train(x_train, weights, y_train, epochs=50)

2.30258509299404 2.30258509299404
1.4135291084254098 1.3964045568574046
0.7383918268053692 0.7069490867372294
0.6110303876622842 0.5875629094255373
0.6214144033063683 0.5988141840601373

accuracy(x_val, final_weights, y_val)

0.8567

plot_classify(val_data[0][21], final_weights)

Predicted: 6