2-Gradient-tape.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8615c866",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.datasets import mnist\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd43edf8",
   "metadata": {},
   "source": [
    "**Preparing the image data**\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4ccff8ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n",
    "\n",
    "train_images = train_images.reshape((60000, 28 * 28))\n",
    "train_images = train_images.astype(\"float32\") / 255\n",
    "test_images = test_images.reshape((10000, 28 * 28))\n",
    "test_images = test_images.astype(\"float32\") / 255"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "618f3449",
   "metadata": {},
   "source": [
    "**The network architecture**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a1ef338c",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.Sequential([\n",
    "    layers.Dense(512, activation=\"relu\"),\n",
    "    layers.Dense(10, activation=\"softmax\")\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4011ed73",
   "metadata": {},
   "source": [
    "**Model weights!**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "55892af8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dense/kernel:0 (784, 512)\n",
      "dense/bias:0 (512,)\n",
      "dense_1/kernel:0 (512, 10)\n",
      "dense_1/bias:0 (10,)\n"
     ]
    }
   ],
   "source": [
    "model(train_images[0:32])\n",
    "weights_and_bias = model.trainable_weights\n",
    "for item in weights_and_bias:\n",
    "    print(item.name, item.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78c3a07d",
   "metadata": {},
   "source": [
    "**Running one training step**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "89109d5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_training_step(model, images_batch, labels_batch):\n",
    "    with tf.GradientTape() as tape:\n",
    "        predictions = model(images_batch)\n",
    "        per_sample_losses = tf.keras.losses.sparse_categorical_crossentropy(\n",
    "            labels_batch, predictions)\n",
    "        average_loss = tf.reduce_mean(per_sample_losses)\n",
    "    gradients = tape.gradient(average_loss, model.trainable_weights)\n",
    "    update_weights(gradients, model.trainable_weights)\n",
    "    return average_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2649d6e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 1e-3\n",
    "\n",
    "def update_weights(gradients, weights):\n",
    "    for g, w in zip(gradients, weights):\n",
    "        w.assign_sub(g * learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "89a411ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras import optimizers\n",
    "\n",
    "optimizer = optimizers.SGD(learning_rate=1e-3)\n",
    "\n",
    "def update_weights(gradients, weights):\n",
    "    optimizer.apply_gradients(zip(gradients, weights))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aad7c32c",
   "metadata": {},
   "source": [
    "**A batch generator**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "97d5a788",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "class BatchGenerator:\n",
    "    def __init__(self, images, labels, batch_size=128):\n",
    "        assert len(images) == len(labels)\n",
    "        self.index = 0\n",
    "        self.images = images\n",
    "        self.labels = labels\n",
    "        self.batch_size = batch_size\n",
    "        self.num_batches = math.ceil(len(images) / batch_size)\n",
    "\n",
    "    def next(self):\n",
    "        images = self.images[self.index : self.index + self.batch_size]\n",
    "        labels = self.labels[self.index : self.index + self.batch_size]\n",
    "        self.index += self.batch_size\n",
    "        return images, labels"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7010bd4",
   "metadata": {},
   "source": [
    "**The full training loop**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c781ae20",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit(model, images, labels, epochs, batch_size=128):\n",
    "    for epoch_counter in range(epochs):\n",
    "        print(f\"Epoch {epoch_counter}\")\n",
    "        batch_generator = BatchGenerator(images, labels)\n",
    "        for batch_counter in range(batch_generator.num_batches):\n",
    "            images_batch, labels_batch = batch_generator.next()\n",
    "            loss = one_training_step(model, images_batch, labels_batch)\n",
    "            if batch_counter % 100 == 0:\n",
    "                print(f\"loss at batch {batch_counter}: {loss:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2c9c002",
   "metadata": {},
   "source": [
    "**fit the model with custom train loop!**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ebb9a62f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0\n",
      "loss at batch 0: 2.40\n",
      "loss at batch 100: 2.18\n",
      "loss at batch 200: 2.06\n",
      "loss at batch 300: 1.98\n",
      "loss at batch 400: 1.89\n",
      "Epoch 1\n",
      "loss at batch 0: 1.82\n",
      "loss at batch 100: 1.74\n",
      "loss at batch 200: 1.58\n",
      "loss at batch 300: 1.56\n",
      "loss at batch 400: 1.52\n",
      "Epoch 2\n",
      "loss at batch 0: 1.42\n",
      "loss at batch 100: 1.41\n",
      "loss at batch 200: 1.24\n",
      "loss at batch 300: 1.26\n",
      "loss at batch 400: 1.26\n",
      "Epoch 3\n",
      "loss at batch 0: 1.15\n",
      "loss at batch 100: 1.17\n",
      "loss at batch 200: 1.00\n",
      "loss at batch 300: 1.05\n",
      "loss at batch 400: 1.08\n",
      "Epoch 4\n",
      "loss at batch 0: 0.97\n",
      "loss at batch 100: 1.00\n",
      "loss at batch 200: 0.84\n",
      "loss at batch 300: 0.91\n",
      "loss at batch 400: 0.95\n",
      "Epoch 5\n",
      "loss at batch 0: 0.84\n",
      "loss at batch 100: 0.87\n",
      "loss at batch 200: 0.73\n",
      "loss at batch 300: 0.81\n",
      "loss at batch 400: 0.86\n",
      "Epoch 6\n",
      "loss at batch 0: 0.75\n",
      "loss at batch 100: 0.78\n",
      "loss at batch 200: 0.65\n",
      "loss at batch 300: 0.74\n",
      "loss at batch 400: 0.80\n",
      "Epoch 7\n",
      "loss at batch 0: 0.69\n",
      "loss at batch 100: 0.71\n",
      "loss at batch 200: 0.59\n",
      "loss at batch 300: 0.68\n",
      "loss at batch 400: 0.74\n",
      "Epoch 8\n",
      "loss at batch 0: 0.63\n",
      "loss at batch 100: 0.66\n",
      "loss at batch 200: 0.54\n",
      "loss at batch 300: 0.63\n",
      "loss at batch 400: 0.70\n",
      "Epoch 9\n",
      "loss at batch 0: 0.59\n",
      "loss at batch 100: 0.61\n",
      "loss at batch 200: 0.50\n",
      "loss at batch 300: 0.60\n",
      "loss at batch 400: 0.67\n"
     ]
    }
   ],
   "source": [
    "fit(model, train_images, train_labels, epochs=10, batch_size=128)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bbabd596",
   "metadata": {},
   "source": [
    "**Evaluating the model on new data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4205dd2a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.87\n"
     ]
    }
   ],
   "source": [
    "predictions = model(test_images)\n",
    "predictions = predictions.numpy()\n",
    "predicted_labels = np.argmax(predictions, axis=1)\n",
    "matches = predicted_labels == test_labels\n",
    "print(f\"accuracy: {matches.mean():.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dee8316a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf2.5",
   "language": "python",
   "name": "tensorflow_25"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}