3-Reimplementing-layers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8615c866",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.datasets import mnist\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd43edf8",
   "metadata": {},
   "source": [
    "**Preparing the image data**\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4ccff8ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n",
    "\n",
    "train_images = train_images.reshape((60000, 28 * 28))\n",
    "train_images = train_images.astype(\"float32\") / 255\n",
    "test_images = test_images.reshape((10000, 28 * 28))\n",
    "test_images = test_images.astype(\"float32\") / 255"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29bb1b99",
   "metadata": {},
   "source": [
    "**__call__ in Python**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "9972dedb",
   "metadata": {},
   "outputs": [],
   "source": [
    "class A:\n",
    "    def __init__(self, a, b):\n",
    "        self.a=a\n",
    "        self.b=b\n",
    "    def __call__(self, x):\n",
    "        return self.a*x+self.b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "a90a9d40",
   "metadata": {},
   "outputs": [],
   "source": [
    "a = A(10, 5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "83808cd3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d76785f4",
   "metadata": {},
   "source": [
    "**A simple Dense class**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "76a548cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "class NaiveDense:\n",
    "    def __init__(self, input_size, output_size, activation):\n",
    "        self.activation = activation\n",
    "\n",
    "        w_shape = (input_size, output_size)\n",
    "        w_initial_value = tf.random.uniform(w_shape, minval=0, maxval=1e-1)\n",
    "        self.W = tf.Variable(w_initial_value)\n",
    "\n",
    "        b_shape = (output_size,)\n",
    "        b_initial_value = tf.zeros(b_shape)\n",
    "        self.b = tf.Variable(b_initial_value)\n",
    "\n",
    "    def __call__(self, inputs):\n",
    "        return self.activation(tf.matmul(inputs, self.W) + self.b)\n",
    "\n",
    "    @property\n",
    "    def weights(self):\n",
    "        return [self.W, self.b]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c472d186",
   "metadata": {},
   "source": [
    "**A simple Sequential class**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fee03e4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "class NaiveSequential:\n",
    "    def __init__(self, layers):\n",
    "        self.layers = layers\n",
    "\n",
    "    def __call__(self, inputs):\n",
    "        x = inputs\n",
    "        for layer in self.layers:\n",
    "           x = layer(x)\n",
    "        return x\n",
    "\n",
    "    @property\n",
    "    def weights(self):\n",
    "        weights = []\n",
    "        for layer in self.layers:\n",
    "            weights += layer.weights\n",
    "        return weights"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "618f3449",
   "metadata": {},
   "source": [
    "**The network architecture**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a1ef338c",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = NaiveSequential([\n",
    "    NaiveDense(input_size=28 * 28, output_size=512, activation=tf.nn.relu),\n",
    "    NaiveDense(input_size=512, output_size=10, activation=tf.nn.softmax)\n",
    "])\n",
    "assert len(model.weights) == 4"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4011ed73",
   "metadata": {},
   "source": [
    "**Model weights!**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "55892af8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable:0 (784, 512)\n",
      "Variable:0 (512,)\n",
      "Variable:0 (512, 10)\n",
      "Variable:0 (10,)\n"
     ]
    }
   ],
   "source": [
    "model(train_images[0:32])\n",
    "weights_and_bias = model.weights\n",
    "for item in weights_and_bias:\n",
    "    print(item.name, item.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78c3a07d",
   "metadata": {},
   "source": [
    "**Running one training step**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "89109d5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_training_step(model, images_batch, labels_batch):\n",
    "    with tf.GradientTape() as tape:\n",
    "        predictions = model(images_batch)\n",
    "        per_sample_losses = tf.keras.losses.sparse_categorical_crossentropy(\n",
    "            labels_batch, predictions)\n",
    "        average_loss = tf.reduce_mean(per_sample_losses)\n",
    "    gradients = tape.gradient(average_loss, model.weights)\n",
    "    update_weights(gradients, model.weights)\n",
    "    return average_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "2649d6e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 1e-3\n",
    "\n",
    "def update_weights(gradients, weights):\n",
    "    for g, w in zip(gradients, weights):\n",
    "        w.assign_sub(g * learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "89a411ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras import optimizers\n",
    "\n",
    "optimizer = optimizers.SGD(learning_rate=1e-3)\n",
    "\n",
    "def update_weights(gradients, weights):\n",
    "    optimizer.apply_gradients(zip(gradients, weights))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aad7c32c",
   "metadata": {},
   "source": [
    "**A batch generator**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "97d5a788",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "class BatchGenerator:\n",
    "    def __init__(self, images, labels, batch_size=128):\n",
    "        assert len(images) == len(labels)\n",
    "        self.index = 0\n",
    "        self.images = images\n",
    "        self.labels = labels\n",
    "        self.batch_size = batch_size\n",
    "        self.num_batches = math.ceil(len(images) / batch_size)\n",
    "\n",
    "    def next(self):\n",
    "        images = self.images[self.index : self.index + self.batch_size]\n",
    "        labels = self.labels[self.index : self.index + self.batch_size]\n",
    "        self.index += self.batch_size\n",
    "        return images, labels"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7010bd4",
   "metadata": {},
   "source": [
    "**The full training loop**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c781ae20",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit(model, images, labels, epochs, batch_size=128):\n",
    "    for epoch_counter in range(epochs):\n",
    "        print(f\"Epoch {epoch_counter}\")\n",
    "        batch_generator = BatchGenerator(images, labels)\n",
    "        for batch_counter in range(batch_generator.num_batches):\n",
    "            images_batch, labels_batch = batch_generator.next()\n",
    "            loss = one_training_step(model, images_batch, labels_batch)\n",
    "            if batch_counter % 100 == 0:\n",
    "                print(f\"loss at batch {batch_counter}: {loss:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2c9c002",
   "metadata": {},
   "source": [
    "**fit the model with custom train loop!**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ebb9a62f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0\n",
      "loss at batch 0: 3.97\n",
      "loss at batch 100: 2.19\n",
      "loss at batch 200: 2.16\n",
      "loss at batch 300: 2.05\n",
      "loss at batch 400: 2.21\n",
      "Epoch 1\n",
      "loss at batch 0: 1.87\n",
      "loss at batch 100: 1.83\n",
      "loss at batch 200: 1.78\n",
      "loss at batch 300: 1.67\n",
      "loss at batch 400: 1.81\n",
      "Epoch 2\n",
      "loss at batch 0: 1.55\n",
      "loss at batch 100: 1.53\n",
      "loss at batch 200: 1.46\n",
      "loss at batch 300: 1.38\n",
      "loss at batch 400: 1.49\n",
      "Epoch 3\n",
      "loss at batch 0: 1.29\n",
      "loss at batch 100: 1.30\n",
      "loss at batch 200: 1.20\n",
      "loss at batch 300: 1.17\n",
      "loss at batch 400: 1.26\n",
      "Epoch 4\n",
      "loss at batch 0: 1.10\n",
      "loss at batch 100: 1.13\n",
      "loss at batch 200: 1.01\n",
      "loss at batch 300: 1.02\n",
      "loss at batch 400: 1.10\n",
      "Epoch 5\n",
      "loss at batch 0: 0.96\n",
      "loss at batch 100: 0.99\n",
      "loss at batch 200: 0.87\n",
      "loss at batch 300: 0.90\n",
      "loss at batch 400: 0.98\n",
      "Epoch 6\n",
      "loss at batch 0: 0.86\n",
      "loss at batch 100: 0.89\n",
      "loss at batch 200: 0.77\n",
      "loss at batch 300: 0.82\n",
      "loss at batch 400: 0.90\n",
      "Epoch 7\n",
      "loss at batch 0: 0.78\n",
      "loss at batch 100: 0.81\n",
      "loss at batch 200: 0.70\n",
      "loss at batch 300: 0.75\n",
      "loss at batch 400: 0.83\n",
      "Epoch 8\n",
      "loss at batch 0: 0.72\n",
      "loss at batch 100: 0.75\n",
      "loss at batch 200: 0.64\n",
      "loss at batch 300: 0.70\n",
      "loss at batch 400: 0.78\n",
      "Epoch 9\n",
      "loss at batch 0: 0.67\n",
      "loss at batch 100: 0.70\n",
      "loss at batch 200: 0.59\n",
      "loss at batch 300: 0.65\n",
      "loss at batch 400: 0.74\n"
     ]
    }
   ],
   "source": [
    "fit(model, train_images, train_labels, epochs=10, batch_size=128)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bbabd596",
   "metadata": {},
   "source": [
    "**Evaluating the model on new data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "4205dd2a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.82\n"
     ]
    }
   ],
   "source": [
    "predictions = model(test_images)\n",
    "predictions = predictions.numpy()\n",
    "predicted_labels = np.argmax(predictions, axis=1)\n",
    "matches = predicted_labels == test_labels\n",
    "print(f\"accuracy: {matches.mean():.2f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf2.5",
   "language": "python",
   "name": "tensorflow_25"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}