46_GrdientTape.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "NDCuMPYybp0J"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Gradient of a Variable"
      ],
      "metadata": {
        "id": "HBamy8t4b9mV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "x = tf.Variable(3.0)\n",
        "with tf.GradientTape() as g:\n",
        "  y = x * x\n",
        "dy_dx = g.gradient(y, x)\n",
        "print(dy_dx)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O_6C2wXKbwnz",
        "outputId": "f3f1a7e6-9139-467f-f836-a80fa647a5da"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tf.Tensor(6.0, shape=(), dtype=float32)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Gradient of a Constant"
      ],
      "metadata": {
        "id": "Un4LdcZPcKs6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "x = tf.constant(3.0)\n",
        "with tf.GradientTape() as g:\n",
        "  y = x * x\n",
        "dy_dx = g.gradient(y, x)\n",
        "print(dy_dx)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pr5oKsFrcMrW",
        "outputId": "cd1d28de-b204-490d-f4f3-d57db0666d35"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "None\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "we have to use watch!"
      ],
      "metadata": {
        "id": "al2D8caKcUUh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "x = tf.constant(3.0)\n",
        "with tf.GradientTape() as g:\n",
        "  g.watch(x) # because x is not variable\n",
        "  y = x * x\n",
        "dy_dx = g.gradient(y, x)\n",
        "print(dy_dx)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7o1F5LVZcR-T",
        "outputId": "80a982f5-b1de-4885-f764-1eeb6defa090"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tf.Tensor(6.0, shape=(), dtype=float32)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## A non-persistent VS Persistent Tape"
      ],
      "metadata": {
        "id": "4aCc-neMcl6T"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "x = tf.constant(3.0)\n",
        "with tf.GradientTape() as g:\n",
        "  g.watch(x)\n",
        "  y = x * x\n",
        "  z = y * y\n",
        "dz_dx = g.gradient(z, x)\n",
        "print(dz_dx)\n",
        "\n",
        "\n",
        "dy_dx = g.gradient(y, x)\n",
        "print(dy_dx)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 391
        },
        "id": "FjNGJBgBcbkY",
        "outputId": "9adf4e22-b170-4694-d338-a50609e69604"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tf.Tensor(108.0, shape=(), dtype=float32)\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-5-4b3a12839d22>\u001b[0m in \u001b[0;36m<cell line: 10>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mdy_dx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgradient\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdy_dx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/backprop.py\u001b[0m in \u001b[0;36mgradient\u001b[0;34m(self, target, sources, output_gradients, unconnected_gradients)\u001b[0m\n\u001b[1;32m   1001\u001b[0m     \"\"\"\n\u001b[1;32m   1002\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tape\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1003\u001b[0;31m       raise RuntimeError(\"A non-persistent GradientTape can only be used to \"\n\u001b[0m\u001b[1;32m   1004\u001b[0m                          \"compute one set of gradients (or jacobians)\")\n\u001b[1;32m   1005\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_recording\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mRuntimeError\u001b[0m: A non-persistent GradientTape can only be used to compute one set of gradients (or jacobians)"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x = tf.constant(3.0)\n",
        "with tf.GradientTape(persistent=True) as g:\n",
        "  g.watch(x)\n",
        "  y = x * x\n",
        "  z = y * y\n",
        "dz_dx = g.gradient(z, x)  # (4*x^3 at x = 3)\n",
        "print(dz_dx)\n",
        "\n",
        "\n",
        "dy_dx = g.gradient(y, x)\n",
        "print(dy_dx)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wte9AadQcyCR",
        "outputId": "03bb033f-ef87-4bc1-9fe1-2d0c0d52046b"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tf.Tensor(108.0, shape=(), dtype=float32)\n",
            "tf.Tensor(6.0, shape=(), dtype=float32)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## GradientTape on a Model"
      ],
      "metadata": {
        "id": "aHaDn7ppnEOA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "mnist = tf.keras.datasets.mnist\n",
        "\n",
        "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
        "x_train, x_test = x_train / 255.0, x_test / 255.0\n",
        "\n",
        "x_train = x_train.reshape(-1,784)\n",
        "x_test = x_test.reshape(-1,784)"
      ],
      "metadata": {
        "id": "M8mp2A4Nn3R-"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = tf.keras.Sequential([\n",
        "  tf.keras.layers.Dense(10, activation=tf.nn.relu, input_shape=(784,)),\n",
        "  tf.keras.layers.Dense(10, activation=tf.nn.relu),\n",
        "  tf.keras.layers.Dense(10)\n",
        "])\n",
        "\n",
        "\n",
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)"
      ],
      "metadata": {
        "id": "MdJKTindnGGN"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def loss(model, x, y, training):\n",
        "  # training=training is needed only if there are layers with different\n",
        "  # behavior during training versus inference (e.g. Dropout).\n",
        "  y_ = model(x, training=training)\n",
        "\n",
        "  return loss_object(y_true=y, y_pred=y_)\n",
        "\n",
        "l = loss(model, x_train, y_train, training=False)\n",
        "print(\"Loss test: {}\".format(l))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JpQFHtcmnaPv",
        "outputId": "03e5039c-92f9-4519-aba2-2baa97731b7d"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss test: 2.3115293979644775\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Use the tf.GradientTape context to calculate the gradients used to optimize your model:"
      ],
      "metadata": {
        "id": "YlDLmbEfoTOY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def grad(model, inputs, targets):\n",
        "  with tf.GradientTape() as tape:\n",
        "    loss_value = loss(model, inputs, targets, training=True)\n",
        "  return loss_value, tape.gradient(loss_value, model.trainable_variables)"
      ],
      "metadata": {
        "id": "ukZ69grdoUpV"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)"
      ],
      "metadata": {
        "id": "sf9i6f4TogQe"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loss_value, grads = grad(model, x_train[:256,:], y_train[:256])"
      ],
      "metadata": {
        "id": "eG1Rk3feo8VS"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Step: {}, Initial Loss: {}\".format(optimizer.iterations.numpy(),\n",
        "                                          loss_value.numpy()))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I_PgHYkipH9r",
        "outputId": "57319e99-2b26-40df-f907-a19dc102d944"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Step: 0, Initial Loss: 2.3078858852386475\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "optimizer.apply_gradients(zip(grads, model.trainable_variables))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7m11ZldSpKsW",
        "outputId": "466461f8-3b1d-422e-d9bd-8d6a24054f20"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Variable 'UnreadVariable' shape=() dtype=int64, numpy=1>"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Step: {},         Loss: {}\".format(optimizer.iterations.numpy(),\n",
        "                                          loss(model, x_train[:256,:], y_train[:256], training=False).numpy()))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fTVoAESRpTvc",
        "outputId": "d3efcd11-888a-43c9-d284-06341a22b0f2"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Step: 1,         Loss: 2.305161952972412\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### use loop to train a model"
      ],
      "metadata": {
        "id": "LTm3zfifqnRE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))\n",
        "test_dataset = tf.data.Dataset.from_tensor_slices((x_test, y_test))\n",
        "\n",
        "# Define the batch size\n",
        "batch_size = 256\n",
        "\n",
        "# Batch the datasets\n",
        "train_dataset = train_dataset.batch(batch_size)\n",
        "test_dataset = test_dataset.batch(batch_size)\n"
      ],
      "metadata": {
        "id": "CS4JI-k5pyHK"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "optimizer = tf.keras.optimizers.Adam()"
      ],
      "metadata": {
        "id": "WdNi5ebHq9cI"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "epochs = 5\n",
        "for epoch in range(epochs):\n",
        "    print(\"\\nStart of epoch %d\" % (epoch,))\n",
        "    # Iterate over the batches of the dataset.\n",
        "    for step, (x_batch_train, y_batch_train) in enumerate(train_dataset):\n",
        "        # Open a GradientTape to record the operations run\n",
        "        # during the forward pass, which enables auto-differentiation.\n",
        "        with tf.GradientTape() as tape:\n",
        "            # Run the forward pass of the layer.\n",
        "            # The operations that the layer applies\n",
        "            # to its inputs are going to be recorded\n",
        "            # on the GradientTape.\n",
        "            logits = model(x_batch_train, training=True)  # Logits for this minibatch\n",
        "            # Compute the loss value for this minibatch.\n",
        "            loss_value = loss_fn(y_batch_train, logits)\n",
        "        # Use the gradient tape to automatically retrieve\n",
        "        # the gradients of the trainable variables with respect to the loss.\n",
        "        grads = tape.gradient(loss_value, model.trainable_weights)\n",
        "        # Run one step of gradient descent by updating\n",
        "        # the value of the variables to minimize the loss.\n",
        "        optimizer.apply_gradients(zip(grads, model.trainable_weights))\n",
        "        # Log every 200 batches.\n",
        "        if step % 200 == 0:\n",
        "            print(\n",
        "                \"Training loss (for one batch) at step %d: %.4f\"\n",
        "                % (step, float(loss_value))\n",
        "            )\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z3o4m5NEqz9-",
        "outputId": "0d17fa11-30c4-408d-a5cc-7e824a5f0685"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Start of epoch 0\n",
            "Training loss (for one batch) at step 0: 2.3052\n",
            "Training loss (for one batch) at step 200: 0.6672\n",
            "\n",
            "Start of epoch 1\n",
            "Training loss (for one batch) at step 0: 0.5424\n",
            "Training loss (for one batch) at step 200: 0.4090\n",
            "\n",
            "Start of epoch 2\n",
            "Training loss (for one batch) at step 0: 0.3562\n",
            "Training loss (for one batch) at step 200: 0.3385\n",
            "\n",
            "Start of epoch 3\n",
            "Training loss (for one batch) at step 0: 0.2972\n",
            "Training loss (for one batch) at step 200: 0.2924\n",
            "\n",
            "Start of epoch 4\n",
            "Training loss (for one batch) at step 0: 0.2667\n",
            "Training loss (for one batch) at step 200: 0.2595\n"
          ]
        }
      ]
    }
  ]
}