SLP_1_0_0.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/byanez15/gerstman-gang/blob/main/SLP_1_0_0.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "blef31L63bvZ",
        "outputId": "5fc87585-5f73-4a31-be3e-c84b6c95e4b5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: labtools3 in /usr/local/lib/python3.9/dist-packages (1.1.3.11)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.9/dist-packages (9.4.0)\n"
          ]
        }
      ],
      "source": [
        "#@title Install Packages\n",
        "!pip install labtools3 \n",
        "!pip install --upgrade Pillow"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XVvlWxUkS8zd",
        "outputId": "0ce2d576-8429-4dd5-cd99-3bd98cbc876c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423,
          "referenced_widgets": [
            "765159e27987410d8a1fa40e596f9879",
            "316abce840654cd095ecd559e55f5b01",
            "9a5edd0bf33d440bbf68cc4c83afdc29",
            "9e5f93cbba6b4fcab6dfb9d84b3f4d09",
            "f0464d4e9d084b108190ed51029a88c1",
            "4e9cbf9035b3446aaa3947683ed52fe6",
            "c5f5a4bccd924db9936bcbc1ae62ef1d",
            "b0cf56e758784f609999e65aa21023d8",
            "6c8c8e3748774935892a6b456b28890d",
            "c2fc96e6d4d94812a73dced45ecbe208",
            "b9261f7c581b4a53ac12a5f347601386",
            "7cf97933218a4f4292eae704cb6566e5",
            "7822b403bf9e477c9f198d74420f47ef",
            "7c60e0db1d834ee7b517fe173fc3dd7a",
            "91b3e96c7b2e4a7aa119277d9faf83c7",
            "e36d8cbb54ba4226bcc74778778131f8",
            "b9e693c2ec2a490e8d35e6729e7d65f4",
            "4708ef12720447848e96ad692eea9a27",
            "204ae6fd20cd4bb5a0b814d34ddbce9c",
            "c549d913001a4594852bb7f1d92e6988",
            "b8614efca9ba41b6abbd939c2326fd3d",
            "d206bb4635b744299007c304d7abbe31",
            "ee0c44627e1645688d9171009d396bf3",
            "f3931f1334724d0f8ba643874c4d90f0",
            "c3c27b2ce4ac472e82e1fb48605c21ac",
            "e2ba1837f14b412e919a61b8040ba5a9",
            "882dc860b8604bc2812c8b662a86b61e",
            "a3f41e9efa144f7082033a16b7e4e316",
            "94a2bac4195749f98df4825d39eab3f1",
            "c0ffed22b27f48fb958c67d693060dcc",
            "f3d19517f8374f4883965e5627c49581",
            "6e374633889c4cbf83eaae5fcdfe481e",
            "021d6b8d1fb74567b109d875207e19fd",
            "2eae3cc183584c3c8426af786d9683b7",
            "3750e853fef54f329e4b0a3babf53ac3",
            "89fdb54347c640f592e26e97f1df9094",
            "b279949d0b014c7c94dfc73b23304feb",
            "964b7584e52c4cebb0040e1ff1de4c49",
            "9da47fba93db42958a020a5131545d63",
            "296e2be3ab2e43c987dd66254f060811",
            "74ce2c34be43425f80afc31cca87e5db",
            "8074d408a10b493fa0b771d0e3e5fd43",
            "3c5a03c8a99543f1bdaacab2e6b4b99b",
            "a48e32089ada4131a46d5a76405cde9e"
          ]
        },
        "id": "nAGxV9EU6VaY",
        "outputId": "8f066f25-32a8-424b-dd3d-d9b8e805a9ea"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to ./MNIST/raw/train-images-idx3-ubyte.gz\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/9912422 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "765159e27987410d8a1fa40e596f9879"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./MNIST/raw/train-images-idx3-ubyte.gz to ./MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to ./MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/28881 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7cf97933218a4f4292eae704cb6566e5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./MNIST/raw/train-labels-idx1-ubyte.gz to ./MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to ./MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1648877 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ee0c44627e1645688d9171009d396bf3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./MNIST/raw/t10k-images-idx3-ubyte.gz to ./MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to ./MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/4542 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2eae3cc183584c3c8426af786d9683b7"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./MNIST/raw/t10k-labels-idx1-ubyte.gz to ./MNIST/raw\n",
            "\n"
          ]
        }
      ],
      "source": [
        "#@title Import Packages and MNIST Data\n",
        "from PIL import Image, ImageDraw\n",
        "import random \n",
        "import pandas as pd \n",
        "import numpy as np\n",
        "import torch as t\n",
        "import torchvision\n",
        "import torchvision.datasets as datasets\n",
        "from torchvision import transforms\n",
        "import torch.functional\n",
        "import torch.nn as nn\n",
        "import LT.box as B\n",
        "import torch.nn.utils.prune as prune\n",
        "import copy\n",
        "from datetime import datetime\n",
        "\n",
        "#MNIST DATA DOWNLOAD!!!!\n",
        "\n",
        "transform = transforms.ToTensor()\n",
        "\n",
        "# Apply the transform to the train and test datasets\n",
        "dataset = datasets.MNIST(root='.', train=True, transform=transform, download=True)\n",
        "\n",
        "\n",
        "#Segregates training and test data\n",
        "train_size = int(0.8 * len(dataset))\n",
        "test_size = len(dataset) - train_size\n",
        "train_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kw3KMzh5WjVK"
      },
      "outputs": [],
      "source": [
        "#@title Import Packages and MNIST Dataset (DETERMINISTIC, RANDOM NUMBERS SET!)\n",
        "from PIL import Image, ImageDraw\n",
        "import random \n",
        "import pandas as pd \n",
        "import numpy as np\n",
        "import torch as t\n",
        "import torchvision\n",
        "import torchvision.datasets as datasets\n",
        "from torchvision import transforms\n",
        "import torch.functional\n",
        "import torch.nn as nn\n",
        "import LT.box as B\n",
        "import torch.nn.utils.prune as prune\n",
        "import copy\n",
        "from datetime import datetime\n",
        "\n",
        "t.manual_seed(0)\n",
        "np.random.seed(0)\n",
        "torch.backends.cudnn.benchmark = False #PLEASE SEE PYTORCH DOCUMENTATION ON REPRODUCIBILITY https://pytorch.org/docs/stable/notes/randomness.html\n",
        "torch.use_deterministic_algorithms(mode = True, warn_only = True ) #THROWS ERROR FOR ANY NONDETERMINISM\n",
        "\n",
        "#MNIST DATA DOWNLOAD!!!!\n",
        "\n",
        "transform = transforms.ToTensor()\n",
        "\n",
        "# Apply the transform to the train and test datasets\n",
        "dataset = datasets.MNIST(root='.', train=True, transform=transform, download=True)\n",
        "\n",
        "\n",
        "#Segregates training and test data\n",
        "train_size = int(0.8 * len(dataset))\n",
        "test_size = len(dataset) - train_size\n",
        "train_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "ish0iAed6eS4"
      },
      "outputs": [],
      "source": [
        "#@title Neural Network Class\n",
        "t.set_printoptions(threshold=100000000000)#  -- For printing full tensor information\n",
        "rng = lambda : torch.Generator().manual_seed(123456789)\n",
        "\n",
        "#CONSTANTS\n",
        "\n",
        "EPSILON_CE=1e-3     #ce criterion\n",
        "EPSILON_ACC = 1e-3    #accuracy is given in percentage so we use .1 to represent .001\n",
        "\n",
        "\n",
        "\"\"\"\n",
        "Nural_Network Class\n",
        "@author: Kevin Jenkins, Kevin Acosta, Brian Yanez\n",
        "\n",
        "Can define:\n",
        "learning_rate\n",
        "batch_size\n",
        "\"\"\"\n",
        "\n",
        "class Neural_Network(nn.Module):\n",
        "  def __init__(self, batch_size, learning_rate):\n",
        "    super(Neural_Network, self).__init__()\n",
        "    self.index = 0\n",
        "\n",
        "    #Create the data loaders and empty data sets\n",
        "    self.data = {'IPA' : [], 'CE_m' : [], 'Acc_m': [], 'Epoch' : [], 'Running_Acc' : []}\n",
        "    \n",
        "    self.batch_size = batch_size\n",
        "    self.train_loader = t.utils.data.DataLoader(train_dataset, batch_size = self.batch_size, shuffle = True)\n",
        "    self.test_loader = t.utils.data.DataLoader(test_dataset, batch_size=self.batch_size, shuffle=False)\n",
        "\n",
        "    input_size=28*28\n",
        "    output_size = 10\n",
        "    self.linlayer = nn.Linear(input_size, output_size,bias = False)\n",
        "    self.model = nn.Sequential(self.linlayer)\n",
        "\n",
        "    #Manually fix weight matrices to ones\n",
        "    with torch.no_grad():\n",
        "      for name, param in self.model.named_parameters():\n",
        "          if param.requires_grad:\n",
        "              param.copy_(t.ones_like(param))\n",
        "\n",
        "\n",
        "    self.criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "    self.my_lr = learning_rate\n",
        "    self.optimizer = t.optim.Adadelta(self.model.parameters(), lr = self.my_lr)\n",
        "\n",
        "  def convert_data(self):\n",
        "    self.data = pd.DataFrame(data = self.data)\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "K8VoInMP80u0"
      },
      "outputs": [],
      "source": [
        "#@title Miscellaneous Functions\n",
        "def average_acc(l: list, window_size = 5, stopping_val = EPSILON_ACC):\n",
        "  moving_aves = []\n",
        "  if (len(l) < window_size):\n",
        "    pass\n",
        "\n",
        "  stop = False\n",
        "  vals = [0]\n",
        "  for i, x in enumerate(l, 1):\n",
        "    vals.append(vals[i-1] + x)\n",
        "    if i >= window_size:\n",
        "      moving_ave = (vals[i] - vals[i - window_size])/window_size\n",
        "      moving_aves.append(moving_ave)\n",
        "      if moving_ave <= stopping_val:\n",
        "        stop = True\n",
        "\n",
        "  return stop, moving_aves\n",
        "  \n",
        "\n",
        "def evaluate_accuracy(m, test_loader, device):\n",
        "    m.model.eval()\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    with torch.no_grad():\n",
        "        for inputs, targets in test_loader:\n",
        "            inputs, targets = inputs.to(device), targets.to(device)\n",
        "            inputs = inputs.view(-1, 28*28) #Flatten data\n",
        "            outputs = m.model(inputs)\n",
        "            _, preds = torch.max(outputs, 1)\n",
        "            total += targets.size(0)\n",
        "            correct += (preds == targets).sum().item()\n",
        "    return (correct / total) * 100\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "XTFk651KCJcq"
      },
      "outputs": [],
      "source": [
        "from torchvision.datasets.mnist import string\n",
        "#@title Training Method\n",
        "\n",
        "def train( a : Neural_Network, suppressLog = True, CUDA = False, percentage = 0, criterion = 'CE', stop_epoch = 1000):\n",
        "\n",
        "  elements = int((percentage/100) * (28**2)) #number of gradient matrix entries to be set to zero \n",
        "\n",
        "  m_L, epoques, losses, dCE, accuracies, ce, delta_acc, ipa = [[] for i in range(8)]\n",
        "  \n",
        "\n",
        "  if (criterion == 'Accuracy'):\n",
        "    flag = True\n",
        "    \n",
        "    #TRAINING LOOP\n",
        "    epoch, deltaCE = 0,0\n",
        "    while epoch <= stop_epoch and flag:\n",
        "\n",
        "      #Empty gradients before next step\n",
        "      a.optimizer.zero_grad()\n",
        "\n",
        "      loss=0\n",
        "      m = 1\n",
        "\n",
        "      for datap, target in a.train_loader:\n",
        "        if (CUDA):\n",
        "          model = a.model.to(0)  # move the model to GPU\n",
        "          datap = datap.to(0)  # move the data to GPU\n",
        "          target = target.to(0)  # move the targets to GPU\n",
        "\n",
        "        datap = datap.view(-1, 28*28) #Flatten data\n",
        "\n",
        "\n",
        "        # Clear the gradients\n",
        "        a.optimizer.zero_grad()\n",
        "        # Forward pass\n",
        "        output = model(datap)\n",
        "        # Compute the loss\n",
        "        loss = a.criterion(output, target)\n",
        "\n",
        "        \n",
        "\n",
        "        DEVICE = 0 ####DO THIS BETTER LATER\n",
        "        if (m==1 and epoch == 0):\n",
        "          print(f'Initial Loss: {loss}')\n",
        "          #ce.append(loss.cpu().detach().numpy())\n",
        "          with torch.no_grad():\n",
        "            acc = evaluate_accuracy(a, a.test_loader, DEVICE)\n",
        "            accuracies.append(acc)\n",
        "            a.data['Acc_m'].append(acc)\n",
        "            a.data['CE_m'].append(loss.cpu().detach().numpy().tolist())\n",
        "            ce.append(loss.cpu().detach().numpy())\n",
        "            a.data['IPA'].append(0.0)\n",
        "          print(f'Accuracy {acc}')\n",
        "\n",
        "\n",
        "        # Backward pass\n",
        "        loss.backward()\n",
        "\n",
        "        #Before we update the weights, 0 the gradients of neurons we want fixed at 0 FROM LEFT TO RIGHT\n",
        "        a.linlayer.weight.grad.data[:, 0:elements].fill_(0)\n",
        "\n",
        "        # Update the weights\n",
        "        a.optimizer.step()\n",
        "\n",
        "        #For graphing\n",
        "\n",
        "        m = m+1\n",
        "        \n",
        "      if (epoch != 0):\n",
        "        # ACCURACY \n",
        "        print(f'Loss: {loss} Epoch: {epoch}')\n",
        "        with torch.no_grad():\n",
        "          acc = evaluate_accuracy(a, a.test_loader, DEVICE)\n",
        "          accuracies.append(acc)\n",
        "          a.data['Acc_m'].append(acc)  #appending data to our model\n",
        "        print(f'Accuracy {acc}')\n",
        "\n",
        "        epoques.append(epoch)\n",
        "\n",
        "        #CE_m\n",
        "        with torch.no_grad():\n",
        "\n",
        "          ce.append(loss.cpu().detach().numpy())\n",
        "          a.data['CE_m'].append(loss.cpu().detach().numpy().tolist())  #appending data to our model\n",
        "\n",
        "        #IPA\n",
        "        with torch.no_grad():\n",
        "          curr_ipa = (ce[epoch] - ce[epoch - 1])/epoch\n",
        "          a.data['IPA'].append(abs(curr_ipa)) #appending data to our model\n",
        "\n",
        "      if (epoch > 1):\n",
        "        #Stopping condition (delta CE part)\n",
        "        key_value = (accuracies[epoch] - accuracies[epoch - 1])/accuracies[epoch - 1]\n",
        "        delta_acc.append(abs(key_value))\n",
        "      \n",
        "      a.data['Epoch'].append(epoch)\n",
        "      epoch += 1\n",
        "\n",
        "      if (average_acc(delta_acc)):\n",
        "        print(delta_acc)\n",
        "        flag = False\n",
        "        print('\\nThe running average of the accuracies has dropped below epsilon.\\nTraining Complete!\\n')\n",
        "\n",
        "\n",
        "    #end of Acc criterion\n",
        "\n",
        "  if (criterion == 'CE'):\n",
        "    flag = True\n",
        "    #TRAINING LOOP\n",
        "    epoch, deltaCE = 0,0\n",
        "    while epoch <= stop_epoch and flag:\n",
        "\n",
        "      #Empty gradients before next step\n",
        "      a.optimizer.zero_grad()\n",
        "\n",
        "      loss=0\n",
        "      m = 1\n",
        "\n",
        "      for datap, target in a.train_loader:\n",
        "        if (CUDA):\n",
        "          model = a.model.to(0)  # move the model to GPU\n",
        "          datap = datap.to(0)  # move the data to GPU\n",
        "          target = target.to(0)  # move the targets to GPU\n",
        "\n",
        "        datap = datap.view(-1, 28*28) #Flatten data\n",
        "\n",
        "\n",
        "        # Clear the gradients\n",
        "        a.optimizer.zero_grad()\n",
        "        # Forward pass\n",
        "        output = model(datap)\n",
        "        # Compute the loss\n",
        "        loss = a.criterion(output, target)\n",
        "\n",
        "        \n",
        "\n",
        "        DEVICE = 0 ####DO THIS BETTER LATER\n",
        "        if (m==1 and epoch == 0):\n",
        "          print(f'Initial Loss: {loss}')\n",
        "          #ce.append(loss.cpu().detach().numpy())\n",
        "          with torch.no_grad():\n",
        "            acc = evaluate_accuracy(a, a.test_loader, DEVICE)\n",
        "            accuracies.append(acc)\n",
        "            a.data['Acc_m'].append(acc)\n",
        "            a.data['CE_m'].append(loss.cpu().detach().numpy().tolist())\n",
        "            ce.append(loss.cpu().detach().numpy())\n",
        "            a.data['IPA'].append(0.0)\n",
        "          print(f'Accuracy {acc}')\n",
        "\n",
        "\n",
        "        # Backward pass\n",
        "        loss.backward()\n",
        "\n",
        "        #Before we update the weights, 0 the gradients of neurons we want fixed at 0 FROM LEFT TO RIGHT\n",
        "        a.linlayer.weight.grad.data[:, 0:elements].fill_(0)\n",
        "\n",
        "        # Update the weights\n",
        "        a.optimizer.step()\n",
        "\n",
        "        #For graphing\n",
        "\n",
        "        m = m+1\n",
        "        \n",
        "      if (epoch != 0):\n",
        "        # ACCURACY \n",
        "        print(f'Loss: {loss} Epoch: {epoch}')\n",
        "        with torch.no_grad():\n",
        "          acc = evaluate_accuracy(a, a.test_loader, DEVICE)\n",
        "          accuracies.append(acc)\n",
        "          a.data['Acc_m'].append(acc)  #appending data to our model\n",
        "        print(f'Accuracy {acc}')\n",
        "\n",
        "        epoques.append(epoch)\n",
        "\n",
        "        #CE_m\n",
        "        with torch.no_grad():\n",
        "\n",
        "          ce.append(loss.cpu().detach().numpy())\n",
        "          a.data['CE_m'].append(loss.cpu().detach().numpy().tolist())  #appending data to our model\n",
        "\n",
        "        #IPA\n",
        "        with torch.no_grad():\n",
        "          curr_ipa = (ce[epoch] - ce[epoch - 1])/epoch\n",
        "          a.data['IPA'].append(abs(curr_ipa)) #appending data to our model\n",
        "\n",
        "      if (epoch > 1):\n",
        "        #Stopping condition (delta CE part)\n",
        "        key_value = (accuracies[epoch] - accuracies[epoch - 1])/accuracies[epoch - 1]\n",
        "        delta_acc.append(abs(key_value))\n",
        "      \n",
        "      a.data['Epoch'].append(epoch)\n",
        "      epoch += 1\n",
        "      if len(ce) >= 2:\n",
        "        Delta_CE = ce[-1] - ce[-2]\n",
        "        print(EPSILON_CE)\n",
        "        print(Delta_CE)\n",
        "        if (Delta_CE <= EPSILON_CE):\n",
        "          print(Delta_CE)\n",
        "          flag = False\n",
        "          print('\\nThe running average of the cross entropies has dropped below epsilon.\\nTraining Complete!\\n')\n",
        "\n",
        "    #end CE criterion\n",
        "\n",
        "  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "Zyek6kF9CqP2"
      },
      "outputs": [],
      "source": [
        "#@title Training Method 2 (a bit more concise)\n",
        "from torchvision.datasets.mnist import string\n",
        "\n",
        "\n",
        "def train( a : Neural_Network, suppressLog = True, CUDA = False, percentage = 0, criterion = 'CE', stop_epoch = 1000):\n",
        "\n",
        "  elements = int((percentage/100) * (28**2)) #number of gradient matrix entries to be set to zero \n",
        "\n",
        "  m_L, epoques, losses, dCE, accuracies, ce, delta_acc, ipa = [[] for i in range(8)]\n",
        "  \n",
        "  flag = True\n",
        "    #TRAINING LOOP\n",
        "  epoch, deltaCE = 0,0\n",
        "  while epoch <= stop_epoch and flag:\n",
        "\n",
        "    #Empty gradients before next step\n",
        "    a.optimizer.zero_grad()\n",
        "\n",
        "    loss=0\n",
        "    m = 1\n",
        "\n",
        "    for datap, target in a.train_loader:\n",
        "      if (CUDA):\n",
        "        model = a.model.to(0)  # move the model to GPU\n",
        "        datap = datap.to(0)  # move the data to GPU\n",
        "        target = target.to(0)  # move the targets to GPU\n",
        "\n",
        "      datap = datap.view(-1, 28*28) #Flatten data\n",
        "\n",
        "\n",
        "      # Clear the gradients\n",
        "      a.optimizer.zero_grad()\n",
        "      # Forward pass\n",
        "      output = model(datap)\n",
        "      # Compute the loss\n",
        "      loss = a.criterion(output, target)\n",
        "\n",
        "      \n",
        "\n",
        "      DEVICE = 0 ####DO THIS BETTER LATER\n",
        "      if (m==1 and epoch == 0):\n",
        "        print(f'Initial Loss: {loss}')\n",
        "        #ce.append(loss.cpu().detach().numpy())\n",
        "        with torch.no_grad():\n",
        "          acc = evaluate_accuracy(a, a.test_loader, DEVICE)\n",
        "          accuracies.append(acc)\n",
        "          a.data['Acc_m'].append(acc)\n",
        "          a.data['CE_m'].append(loss.cpu().detach().numpy().tolist())\n",
        "          ce.append(loss.cpu().detach().numpy())\n",
        "          a.data['IPA'].append(0.0)\n",
        "        print(f'Accuracy {acc}')\n",
        "\n",
        "\n",
        "      # Backward pass\n",
        "      loss.backward()\n",
        "\n",
        "      #Before we update the weights, 0 the gradients of neurons we want fixed at 0 FROM LEFT TO RIGHT\n",
        "      a.linlayer.weight.grad.data[:, 0:elements].fill_(0)\n",
        "\n",
        "      # Update the weights\n",
        "      a.optimizer.step()\n",
        "\n",
        "      #For graphing\n",
        "\n",
        "      m = m+1\n",
        "      \n",
        "    if (epoch != 0):\n",
        "      # ACCURACY \n",
        "      print(f'Loss: {loss} Epoch: {epoch}')\n",
        "      with torch.no_grad():\n",
        "        acc = evaluate_accuracy(a, a.test_loader, DEVICE)\n",
        "        accuracies.append(acc)\n",
        "        a.data['Acc_m'].append(acc)  #appending data to our model\n",
        "      print(f'Accuracy {acc}')\n",
        "\n",
        "      epoques.append(epoch)\n",
        "\n",
        "      #CE_m\n",
        "      with torch.no_grad():\n",
        "\n",
        "        ce.append(loss.cpu().detach().numpy())\n",
        "        a.data['CE_m'].append(loss.cpu().detach().numpy().tolist())  #appending data to our model\n",
        "\n",
        "      #IPA\n",
        "      with torch.no_grad():\n",
        "        curr_ipa = (ce[epoch] - ce[epoch - 1])/epoch\n",
        "        a.data['IPA'].append(abs(curr_ipa)) #appending data to our model\n",
        "\n",
        "    if (epoch > 1):\n",
        "      #Stopping condition (delta CE part)\n",
        "      key_value = (accuracies[epoch] - accuracies[epoch - 1])/accuracies[epoch - 1]\n",
        "      delta_acc.append(abs(key_value))\n",
        "    \n",
        "    a.data['Epoch'].append(epoch)\n",
        "    epoch += 1\n",
        "    \n",
        "\n",
        "\n",
        "    if criterion == 'Accuracy':\n",
        "      running_ave_list = [0 for i in range(6)] + average_acc(delta_acc)[1] #THE 5 IN THIS CASE IS WINDOW SIZE PLUS ONE. PLEASE MIND CHANGING THIS IF U CHANGE WINDOW SIZE\n",
        "      if (average_acc(delta_acc)[0]):\n",
        "        #print(delta_acc) These commented out lines are used for debugging arraysize errors.\n",
        "        flag = False\n",
        "        print(f\"\\nThe running average of the accuracies has dropped below epsilon.\\n We've run for {a.data['Epoch'][-1]} epochs. \\n Training Complete!\\n\")\n",
        "        a.data['Running_Acc'] = running_ave_list\n",
        "        #print(len(running_ave_list)) These commented out lines are used for debugging arraysize errors.\n",
        "        #print(len(a.data['Acc_m'])) These commented out lines are used for debugging arraysize errors.\n",
        "\n",
        "    if criterion == 'CE':\n",
        "      if len(ce) >= 2:\n",
        "        Delta_CE = ce[-1] - ce[-2]\n",
        "        print(EPSILON_CE)\n",
        "        print(Delta_CE)\n",
        "        if (Delta_CE <= EPSILON_CE):\n",
        "          print(Delta_CE)\n",
        "          flag = False\n",
        "          print('\\nThe running average of the cross entropies has dropped below epsilon.\\nTraining Complete!\\n')\n",
        "          a.data['Running_Acc'] = average_acc(delta_acc)[1]\n",
        "\n",
        "  #end CE criterion"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U01wGGD0NeVc",
        "outputId": "03903b23-0674-438e-83ef-730cd4cd66c1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Initial Loss: 2.3025989532470703\n",
            "Accuracy 10.216666666666667\n",
            "Loss: 1.891339898109436 Epoch: 1\n",
            "Accuracy 73.84166666666667\n",
            "Loss: 1.672989845275879 Epoch: 2\n",
            "Accuracy 77.39166666666667\n",
            "Loss: 1.4895737171173096 Epoch: 3\n",
            "Accuracy 78.95\n",
            "Loss: 1.3504905700683594 Epoch: 4\n",
            "Accuracy 79.80833333333334\n",
            "Loss: 1.230902910232544 Epoch: 5\n",
            "Accuracy 80.70833333333334\n",
            "Loss: 1.1337484121322632 Epoch: 6\n",
            "Accuracy 81.29166666666666\n",
            "Loss: 1.0584311485290527 Epoch: 7\n",
            "Accuracy 81.85\n",
            "Loss: 0.991409957408905 Epoch: 8\n",
            "Accuracy 82.29166666666666\n",
            "Loss: 0.9369120001792908 Epoch: 9\n",
            "Accuracy 82.55\n",
            "Loss: 0.8873306512832642 Epoch: 10\n",
            "Accuracy 83.49166666666666\n",
            "Loss: 0.8499456644058228 Epoch: 11\n",
            "Accuracy 83.775\n",
            "Loss: 0.8160408735275269 Epoch: 12\n",
            "Accuracy 84.15\n",
            "Loss: 0.7833898067474365 Epoch: 13\n",
            "Accuracy 84.51666666666667\n",
            "Loss: 0.7589935064315796 Epoch: 14\n",
            "Accuracy 84.73333333333333\n",
            "Loss: 0.7321499586105347 Epoch: 15\n",
            "Accuracy 84.93333333333334\n",
            "Loss: 0.7032679915428162 Epoch: 16\n",
            "Accuracy 85.29166666666667\n",
            "Loss: 0.6905365586280823 Epoch: 17\n",
            "Accuracy 85.5\n",
            "Loss: 0.6653633117675781 Epoch: 18\n",
            "Accuracy 85.50833333333333\n",
            "Loss: 0.6463012099266052 Epoch: 19\n",
            "Accuracy 85.75\n",
            "Loss: 0.6394060850143433 Epoch: 20\n",
            "Accuracy 86.175\n",
            "Loss: 0.6200100779533386 Epoch: 21\n",
            "Accuracy 86.25833333333334\n",
            "Loss: 0.603178083896637 Epoch: 22\n",
            "Accuracy 86.44166666666668\n",
            "Loss: 0.5939618349075317 Epoch: 23\n",
            "Accuracy 86.625\n",
            "Loss: 0.5806171298027039 Epoch: 24\n",
            "Accuracy 86.65\n",
            "Loss: 0.5721257328987122 Epoch: 25\n",
            "Accuracy 86.85833333333333\n",
            "Loss: 0.5591240525245667 Epoch: 26\n",
            "Accuracy 87.03333333333333\n",
            "Loss: 0.5443881154060364 Epoch: 27\n",
            "Accuracy 87.14166666666667\n",
            "Loss: 0.5348609089851379 Epoch: 28\n",
            "Accuracy 87.21666666666667\n",
            "Loss: 0.5341454744338989 Epoch: 29\n",
            "Accuracy 87.16666666666667\n",
            "Loss: 0.52350252866745 Epoch: 30\n",
            "Accuracy 87.33333333333333\n",
            "Loss: 0.5117895007133484 Epoch: 31\n",
            "Accuracy 87.50833333333333\n",
            "Loss: 0.502549409866333 Epoch: 32\n",
            "Accuracy 87.55833333333334\n",
            "Loss: 0.5011043548583984 Epoch: 33\n",
            "Accuracy 87.76666666666667\n",
            "Loss: 0.49553823471069336 Epoch: 34\n",
            "Accuracy 87.76666666666667\n",
            "Loss: 0.47736799716949463 Epoch: 35\n",
            "Accuracy 87.84166666666667\n",
            "Loss: 0.4775591790676117 Epoch: 36\n",
            "Accuracy 87.94999999999999\n",
            "Loss: 0.4658317565917969 Epoch: 37\n",
            "Accuracy 88.06666666666668\n",
            "Loss: 0.46117356419563293 Epoch: 38\n",
            "Accuracy 88.06666666666668\n",
            "\n",
            "The running average of the accuracies has dropped below epsilon.\n",
            " We've run for 38 epochs. \n",
            " Training Complete!\n",
            "\n",
            "            IPA      CE_m      Acc_m  Running_Acc\n",
            "Epoch                                            \n",
            "0      0.000000  2.302599  10.216667     0.000000\n",
            "1      0.411259  1.891340  73.841667     0.000000\n",
            "2      0.109175  1.672990  77.391667     0.000000\n",
            "3      0.061139  1.489574  78.950000     0.000000\n",
            "4      0.034771  1.350491  79.808333     0.000000\n",
            "5      0.023918  1.230903  80.708333     0.000000\n",
            "6      0.016192  1.133748  81.291667     0.019518\n",
            "7      0.010760  1.058431  81.850000     0.011276\n",
            "8      0.008378  0.991410  82.291667     0.008328\n",
            "9      0.006055  0.936912  82.550000     0.006782\n",
            "10     0.004958  0.887331  83.491667     0.006808\n",
            "11     0.003399  0.849946  83.775000     0.006041\n",
            "12     0.002825  0.816041  84.150000     0.005562\n",
            "13     0.002512  0.783390  84.516667     0.005355\n",
            "14     0.001743  0.758994  84.733333     0.005240\n",
            "15     0.001790  0.732150  84.933333     0.003430\n",
            "16     0.001805  0.703268  85.291667     0.003595\n",
            "17     0.000749  0.690537  85.500000     0.003189\n",
            "18     0.001399  0.665363  85.508333     0.002337\n",
            "19     0.001003  0.646301  85.750000     0.002389\n",
            "20     0.000345  0.639406  86.175000     0.002908\n",
            "21     0.000924  0.620010  86.258333     0.002258\n",
            "22     0.000765  0.603178  86.441667     0.002194\n",
            "23     0.000401  0.593962  86.625000     0.002599\n",
            "24     0.000556  0.580617  86.650000     0.002092\n",
            "25     0.000340  0.572126  86.858333     0.001581\n",
            "26     0.000500  0.559124  87.033333     0.001791\n",
            "27     0.000546  0.544388  87.141667     0.001615\n",
            "28     0.000340  0.534861  87.216667     0.001363\n",
            "29     0.000025  0.534145  87.166667     0.001420\n",
            "30     0.000355  0.523503  87.333333     0.001321\n",
            "31     0.000378  0.511790  87.508333     0.001319\n",
            "32     0.000289  0.502549  87.558333     0.001184\n",
            "33     0.000044  0.501104  87.766667     0.001488\n",
            "34     0.000164  0.495538  87.766667     0.001373\n",
            "35     0.000519  0.477368  87.841667     0.001162\n",
            "36     0.000005  0.477559  87.950000     0.001008\n",
            "37     0.000317  0.465832  88.066667     0.001159\n",
            "38     0.000123  0.461174  88.066667     0.000683\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/linear.py:114: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  return F.linear(input, self.weight, self.bias)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Initial Loss: 2.3025989532470703\n",
            "Accuracy 10.216666666666667\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/autograd/__init__.py:197: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss: 1.889973521232605 Epoch: 1\n",
            "Accuracy 74.84166666666667\n",
            "Loss: 1.667312741279602 Epoch: 2\n",
            "Accuracy 77.16666666666666\n",
            "Loss: 1.490411400794983 Epoch: 3\n",
            "Accuracy 78.45\n",
            "Loss: 1.3486385345458984 Epoch: 4\n",
            "Accuracy 79.27499999999999\n",
            "Loss: 1.2321621179580688 Epoch: 5\n",
            "Accuracy 80.675\n",
            "Loss: 1.1351617574691772 Epoch: 6\n",
            "Accuracy 81.33333333333333\n",
            "Loss: 1.0572856664657593 Epoch: 7\n",
            "Accuracy 81.39166666666667\n",
            "Loss: 0.9898413419723511 Epoch: 8\n",
            "Accuracy 82.20833333333334\n",
            "Loss: 0.9453521966934204 Epoch: 9\n",
            "Accuracy 82.91666666666667\n",
            "Loss: 0.8950093388557434 Epoch: 10\n",
            "Accuracy 83.43333333333334\n",
            "Loss: 0.8506465554237366 Epoch: 11\n",
            "Accuracy 83.78333333333333\n",
            "Loss: 0.8148708343505859 Epoch: 12\n",
            "Accuracy 84.11666666666666\n",
            "Loss: 0.7790570259094238 Epoch: 13\n",
            "Accuracy 84.31666666666666\n",
            "Loss: 0.7532163262367249 Epoch: 14\n",
            "Accuracy 84.75833333333334\n",
            "Loss: 0.7340375185012817 Epoch: 15\n",
            "Accuracy 85.09166666666667\n",
            "Loss: 0.7074829339981079 Epoch: 16\n",
            "Accuracy 85.32499999999999\n",
            "Loss: 0.6897716522216797 Epoch: 17\n",
            "Accuracy 85.39166666666667\n",
            "Loss: 0.6633176803588867 Epoch: 18\n",
            "Accuracy 85.70833333333333\n",
            "Loss: 0.6492182016372681 Epoch: 19\n",
            "Accuracy 85.78333333333333\n",
            "Loss: 0.6252207159996033 Epoch: 20\n",
            "Accuracy 86.06666666666666\n",
            "Loss: 0.6286511421203613 Epoch: 21\n",
            "Accuracy 86.29166666666667\n",
            "Loss: 0.6063229441642761 Epoch: 22\n",
            "Accuracy 86.33333333333333\n",
            "Loss: 0.590890109539032 Epoch: 23\n",
            "Accuracy 86.50833333333333\n",
            "Loss: 0.5853081941604614 Epoch: 24\n",
            "Accuracy 86.74166666666666\n",
            "Loss: 0.5678351521492004 Epoch: 25\n",
            "Accuracy 86.88333333333334\n",
            "Loss: 0.5587192177772522 Epoch: 26\n",
            "Accuracy 86.89166666666667\n",
            "Loss: 0.5425002574920654 Epoch: 27\n",
            "Accuracy 87.1\n",
            "Loss: 0.537100076675415 Epoch: 28\n",
            "Accuracy 87.19166666666666\n",
            "Loss: 0.5235033631324768 Epoch: 29\n",
            "Accuracy 87.26666666666667\n",
            "Loss: 0.5176742672920227 Epoch: 30\n",
            "Accuracy 87.38333333333334\n",
            "Loss: 0.5148864984512329 Epoch: 31\n",
            "Accuracy 87.63333333333333\n",
            "Loss: 0.4989761710166931 Epoch: 32\n",
            "Accuracy 87.66666666666667\n",
            "Loss: 0.49269747734069824 Epoch: 33\n",
            "Accuracy 87.68333333333334\n",
            "Loss: 0.4898184537887573 Epoch: 34\n",
            "Accuracy 87.75\n",
            "Loss: 0.4871963560581207 Epoch: 35\n",
            "Accuracy 87.86666666666667\n",
            "Loss: 0.4760024845600128 Epoch: 36\n",
            "Accuracy 88.00833333333333\n",
            "\n",
            "The running average of the accuracies has dropped below epsilon.\n",
            " We've run for 36 epochs. \n",
            " Training Complete!\n",
            "\n",
            "            IPA      CE_m      Acc_m  Running_Acc\n",
            "Epoch                                            \n",
            "0      0.000000  2.302599  10.216667     0.000000\n",
            "1      0.412625  1.889974  74.841667     0.000000\n",
            "2      0.111330  1.667313  77.166667     0.000000\n",
            "3      0.058967  1.490411  78.450000     0.000000\n",
            "4      0.035443  1.348639  79.275000     0.000000\n",
            "5      0.023295  1.232162  80.675000     0.000000\n",
            "6      0.016167  1.135162  81.333333     0.016807\n",
            "7      0.011125  1.057286  81.391667     0.010737\n",
            "8      0.008431  0.989841  82.208333     0.009418\n",
            "9      0.004943  0.945352  82.916667     0.009038\n",
            "10     0.005034  0.895009  83.433333     0.006752\n",
            "11     0.004033  0.850647  83.783333     0.005959\n",
            "12     0.002981  0.814871  84.116667     0.006611\n",
            "13     0.002755  0.779057  84.316667     0.005080\n",
            "14     0.001846  0.753216  84.758333     0.004404\n",
            "15     0.001279  0.734038  85.091667     0.003944\n",
            "16     0.001660  0.707483  85.325000     0.003654\n",
            "17     0.001042  0.689772  85.391667     0.003014\n",
            "18     0.001470  0.663318  85.708333     0.003281\n",
            "19     0.000742  0.649218  85.783333     0.002408\n",
            "20     0.001200  0.625221  86.066667     0.002282\n",
            "21     0.000163  0.628651  86.291667     0.002256\n",
            "22     0.001015  0.606323  86.333333     0.002197\n",
            "23     0.000671  0.590890  86.508333     0.001860\n",
            "24     0.000233  0.585308  86.741667     0.002225\n",
            "25     0.000699  0.567835  86.883333     0.001891\n",
            "26     0.000351  0.558719  86.891667     0.001387\n",
            "27     0.000601  0.542500  87.100000     0.001770\n",
            "28     0.000193  0.537100  87.191667     0.001575\n",
            "29     0.000469  0.523503  87.266667     0.001208\n",
            "30     0.000194  0.517674  87.383333     0.001149\n",
            "31     0.000090  0.514886  87.633333     0.001702\n",
            "32     0.000497  0.498976  87.666667     0.001298\n",
            "33     0.000190  0.492697  87.683333     0.001126\n",
            "34     0.000085  0.489818  87.750000     0.001106\n",
            "35     0.000075  0.487196  87.866667     0.001104\n",
            "36     0.000311  0.476002  88.008333     0.000855\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/linear.py:114: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  return F.linear(input, self.weight, self.bias)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Initial Loss: 2.3025989532470703\n",
            "Accuracy 10.216666666666667\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/autograd/__init__.py:197: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss: 1.8896920680999756 Epoch: 1\n",
            "Accuracy 74.10833333333333\n",
            "Loss: 1.6726027727127075 Epoch: 2\n",
            "Accuracy 77.08333333333334\n",
            "Loss: 1.4916493892669678 Epoch: 3\n",
            "Accuracy 78.93333333333334\n",
            "Loss: 1.3445769548416138 Epoch: 4\n",
            "Accuracy 79.80833333333334\n",
            "Loss: 1.2301193475723267 Epoch: 5\n",
            "Accuracy 80.50833333333334\n",
            "Loss: 1.1357827186584473 Epoch: 6\n",
            "Accuracy 81.29166666666666\n",
            "Loss: 1.0591607093811035 Epoch: 7\n",
            "Accuracy 81.925\n",
            "Loss: 0.9938955307006836 Epoch: 8\n",
            "Accuracy 82.28333333333333\n",
            "Loss: 0.9438817501068115 Epoch: 9\n",
            "Accuracy 82.95\n",
            "Loss: 0.8915561437606812 Epoch: 10\n",
            "Accuracy 83.41666666666666\n",
            "Loss: 0.8537536263465881 Epoch: 11\n",
            "Accuracy 83.68333333333334\n",
            "Loss: 0.8183847069740295 Epoch: 12\n",
            "Accuracy 84.325\n",
            "Loss: 0.7853532433509827 Epoch: 13\n",
            "Accuracy 84.38333333333333\n",
            "Loss: 0.753559410572052 Epoch: 14\n",
            "Accuracy 84.8\n",
            "Loss: 0.7262809872627258 Epoch: 15\n",
            "Accuracy 84.96666666666667\n",
            "Loss: 0.7119388580322266 Epoch: 16\n",
            "Accuracy 85.275\n",
            "Loss: 0.6912626624107361 Epoch: 17\n",
            "Accuracy 85.45833333333334\n",
            "Loss: 0.6635698676109314 Epoch: 18\n",
            "Accuracy 85.65833333333333\n",
            "Loss: 0.6533074378967285 Epoch: 19\n",
            "Accuracy 85.925\n",
            "Loss: 0.6333101391792297 Epoch: 20\n",
            "Accuracy 86.14166666666667\n",
            "Loss: 0.62119460105896 Epoch: 21\n",
            "Accuracy 86.21666666666667\n",
            "Loss: 0.5993605256080627 Epoch: 22\n",
            "Accuracy 86.31666666666666\n",
            "Loss: 0.5894849300384521 Epoch: 23\n",
            "Accuracy 86.55000000000001\n",
            "Loss: 0.582454264163971 Epoch: 24\n",
            "Accuracy 86.73333333333333\n",
            "Loss: 0.5636127591133118 Epoch: 25\n",
            "Accuracy 86.8\n",
            "Loss: 0.5566394329071045 Epoch: 26\n",
            "Accuracy 87.0\n",
            "Loss: 0.5497735738754272 Epoch: 27\n",
            "Accuracy 87.075\n",
            "Loss: 0.5367971658706665 Epoch: 28\n",
            "Accuracy 87.11666666666666\n",
            "Loss: 0.5277440547943115 Epoch: 29\n",
            "Accuracy 87.225\n",
            "Loss: 0.5273212790489197 Epoch: 30\n",
            "Accuracy 87.36666666666667\n",
            "Loss: 0.5086995363235474 Epoch: 31\n",
            "Accuracy 87.53333333333333\n",
            "Loss: 0.5010318756103516 Epoch: 32\n",
            "Accuracy 87.625\n",
            "Loss: 0.4935411810874939 Epoch: 33\n",
            "Accuracy 87.70833333333333\n",
            "Loss: 0.48757949471473694 Epoch: 34\n",
            "Accuracy 87.775\n",
            "Loss: 0.4879997968673706 Epoch: 35\n",
            "Accuracy 87.81666666666666\n",
            "Loss: 0.4732232391834259 Epoch: 36\n",
            "Accuracy 87.99166666666667\n",
            "Loss: 0.4723709225654602 Epoch: 37\n",
            "Accuracy 88.00833333333333\n",
            "\n",
            "The running average of the accuracies has dropped below epsilon.\n",
            " We've run for 37 epochs. \n",
            " Training Complete!\n",
            "\n",
            "            IPA      CE_m      Acc_m  Running_Acc\n",
            "Epoch                                            \n",
            "0      0.000000  2.302599  10.216667     0.000000\n",
            "1      0.412907  1.889692  74.108333     0.000000\n",
            "2      0.108545  1.672603  77.083333     0.000000\n",
            "3      0.060318  1.491649  78.933333     0.000000\n",
            "4      0.036768  1.344577  79.808333     0.000000\n",
            "5      0.022892  1.230119  80.508333     0.000000\n",
            "6      0.015723  1.135783  81.291667     0.018746\n",
            "7      0.010946  1.059161  81.925000     0.012275\n",
            "8      0.008158  0.993896  82.283333     0.008350\n",
            "9      0.005557  0.943882  82.950000     0.007754\n",
            "10     0.005233  0.891556  83.416667     0.007125\n",
            "11     0.003437  0.853754  83.683333     0.005818\n",
            "12     0.002947  0.818385  84.325000     0.005793\n",
            "13     0.002541  0.785353  84.383333     0.005057\n",
            "14     0.002271  0.753559  84.800000     0.004424\n",
            "15     0.001819  0.726281  84.966667     0.003692\n",
            "16     0.000896  0.711939  85.275000     0.003778\n",
            "17     0.001216  0.691263  85.458333     0.002675\n",
            "18     0.001538  0.663570  85.658333     0.003004\n",
            "19     0.000540  0.653307  85.925000     0.002640\n",
            "20     0.001000  0.633310  86.141667     0.002751\n",
            "21     0.000577  0.621195  86.216667     0.002199\n",
            "22     0.000992  0.599361  86.316667     0.002001\n",
            "23     0.000429  0.589485  86.550000     0.002074\n",
            "24     0.000293  0.582454  86.733333     0.001875\n",
            "25     0.000754  0.563613  86.800000     0.001524\n",
            "26     0.000268  0.556639  87.000000     0.001811\n",
            "27     0.000254  0.549774  87.075000     0.001751\n",
            "28     0.000463  0.536797  87.116667     0.001306\n",
            "29     0.000312  0.527744  87.225000     0.001131\n",
            "30     0.000014  0.527321  87.366667     0.001302\n",
            "31     0.000601  0.508700  87.533333     0.001223\n",
            "32     0.000240  0.501032  87.625000     0.001260\n",
            "33     0.000227  0.493541  87.708333     0.001355\n",
            "34     0.000175  0.487579  87.775000     0.001258\n",
            "35     0.000012  0.488000  87.816667     0.001028\n",
            "36     0.000410  0.473223  87.991667     0.001045\n",
            "37     0.000023  0.472371  88.008333     0.000874\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/linear.py:114: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  return F.linear(input, self.weight, self.bias)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Initial Loss: 2.3025989532470703\n",
            "Accuracy 10.216666666666667\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/autograd/__init__.py:197: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss: 1.8901410102844238 Epoch: 1\n",
            "Accuracy 75.275\n",
            "Loss: 1.6698247194290161 Epoch: 2\n",
            "Accuracy 77.45833333333333\n",
            "Loss: 1.4906821250915527 Epoch: 3\n",
            "Accuracy 78.29166666666667\n",
            "Loss: 1.3449586629867554 Epoch: 4\n",
            "Accuracy 79.95\n",
            "Loss: 1.2313878536224365 Epoch: 5\n",
            "Accuracy 80.81666666666666\n",
            "Loss: 1.1313303709030151 Epoch: 6\n",
            "Accuracy 81.39166666666667\n",
            "Loss: 1.0581458806991577 Epoch: 7\n",
            "Accuracy 81.84166666666667\n",
            "Loss: 0.9956945776939392 Epoch: 8\n",
            "Accuracy 82.39166666666667\n",
            "Loss: 0.93387371301651 Epoch: 9\n",
            "Accuracy 82.83333333333334\n",
            "Loss: 0.890978217124939 Epoch: 10\n",
            "Accuracy 83.475\n",
            "Loss: 0.8475794196128845 Epoch: 11\n",
            "Accuracy 83.71666666666667\n",
            "Loss: 0.8141325116157532 Epoch: 12\n",
            "Accuracy 84.18333333333334\n",
            "Loss: 0.7796094417572021 Epoch: 13\n",
            "Accuracy 84.39166666666667\n",
            "Loss: 0.7600417733192444 Epoch: 14\n",
            "Accuracy 84.58333333333333\n",
            "Loss: 0.7308843731880188 Epoch: 15\n",
            "Accuracy 84.98333333333333\n",
            "Loss: 0.702124834060669 Epoch: 16\n",
            "Accuracy 85.23333333333333\n",
            "Loss: 0.6876309514045715 Epoch: 17\n",
            "Accuracy 85.5\n",
            "Loss: 0.6652773022651672 Epoch: 18\n",
            "Accuracy 85.68333333333334\n",
            "Loss: 0.6509560942649841 Epoch: 19\n",
            "Accuracy 85.875\n",
            "Loss: 0.6341744065284729 Epoch: 20\n",
            "Accuracy 86.08333333333333\n",
            "Loss: 0.6242992281913757 Epoch: 21\n",
            "Accuracy 86.18333333333334\n",
            "Loss: 0.610666036605835 Epoch: 22\n",
            "Accuracy 86.325\n",
            "Loss: 0.5944643020629883 Epoch: 23\n",
            "Accuracy 86.48333333333333\n",
            "Loss: 0.5836537480354309 Epoch: 24\n",
            "Accuracy 86.68333333333334\n",
            "Loss: 0.562806248664856 Epoch: 25\n",
            "Accuracy 86.75\n",
            "Loss: 0.5562439560890198 Epoch: 26\n",
            "Accuracy 86.94166666666666\n",
            "Loss: 0.5497539639472961 Epoch: 27\n",
            "Accuracy 87.10833333333333\n",
            "Loss: 0.5370574593544006 Epoch: 28\n",
            "Accuracy 87.28333333333333\n",
            "Loss: 0.5317605137825012 Epoch: 29\n",
            "Accuracy 87.33333333333333\n",
            "Loss: 0.5177881121635437 Epoch: 30\n",
            "Accuracy 87.46666666666667\n",
            "Loss: 0.5087615251541138 Epoch: 31\n",
            "Accuracy 87.53333333333333\n",
            "Loss: 0.5037685036659241 Epoch: 32\n",
            "Accuracy 87.66666666666667\n",
            "Loss: 0.49549952149391174 Epoch: 33\n",
            "Accuracy 87.725\n",
            "Loss: 0.4880851209163666 Epoch: 34\n",
            "Accuracy 87.79166666666667\n",
            "Loss: 0.4776630401611328 Epoch: 35\n",
            "Accuracy 87.81666666666666\n",
            "\n",
            "The running average of the accuracies has dropped below epsilon.\n",
            " We've run for 35 epochs. \n",
            " Training Complete!\n",
            "\n",
            "            IPA      CE_m      Acc_m  Running_Acc\n",
            "Epoch                                            \n",
            "0      0.000000  2.302599  10.216667     0.000000\n",
            "1      0.412458  1.890141  75.275000     0.000000\n",
            "2      0.110158  1.669825  77.458333     0.000000\n",
            "3      0.059714  1.490682  78.291667     0.000000\n",
            "4      0.036431  1.344959  79.950000     0.000000\n",
            "5      0.022714  1.231388  80.816667     0.000000\n",
            "6      0.016676  1.131330  81.391667     0.015780\n",
            "7      0.010455  1.058146  81.841667     0.011085\n",
            "8      0.007806  0.995695  82.391667     0.010277\n",
            "9      0.006869  0.933874  82.833333     0.007113\n",
            "10     0.004290  0.890978  83.475000     0.006494\n",
            "11     0.003945  0.847579  83.716667     0.005650\n",
            "12     0.002787  0.814133  84.183333     0.005659\n",
            "13     0.002656  0.779609  84.391667     0.004810\n",
            "14     0.001398  0.760042  84.583333     0.004192\n",
            "15     0.001944  0.730884  84.983333     0.003589\n",
            "16     0.001797  0.702125  85.233333     0.003598\n",
            "17     0.000853  0.687631  85.500000     0.003109\n",
            "18     0.001242  0.665277  85.683333     0.003043\n",
            "19     0.000754  0.650956  85.875000     0.003036\n",
            "20     0.000839  0.634174  86.083333     0.002576\n",
            "21     0.000470  0.624299  86.183333     0.002220\n",
            "22     0.000620  0.610666  86.325000     0.001923\n",
            "23     0.000704  0.594464  86.483333     0.001861\n",
            "24     0.000450  0.583654  86.683333     0.001876\n",
            "25     0.000834  0.562806  86.750000     0.001544\n",
            "26     0.000252  0.556244  86.941667     0.001754\n",
            "27     0.000240  0.549754  87.108333     0.001808\n",
            "28     0.000453  0.537057  87.283333     0.001843\n",
            "29     0.000183  0.531761  87.333333     0.001495\n",
            "30     0.000466  0.517788  87.466667     0.001647\n",
            "31     0.000291  0.508762  87.533333     0.001358\n",
            "32     0.000156  0.503769  87.666667     0.001279\n",
            "33     0.000251  0.495500  87.725000     0.001010\n",
            "34     0.000218  0.488085  87.791667     0.001047\n",
            "35     0.000298  0.477663  87.816667     0.000799\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/linear.py:114: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  return F.linear(input, self.weight, self.bias)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Initial Loss: 2.3025989532470703\n",
            "Accuracy 10.216666666666667\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/autograd/__init__.py:197: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss: 1.8905292749404907 Epoch: 1\n",
            "Accuracy 74.35833333333333\n",
            "Loss: 1.6727955341339111 Epoch: 2\n",
            "Accuracy 77.825\n",
            "Loss: 1.4852774143218994 Epoch: 3\n",
            "Accuracy 78.175\n",
            "Loss: 1.3420523405075073 Epoch: 4\n",
            "Accuracy 79.68333333333332\n",
            "Loss: 1.2245535850524902 Epoch: 5\n",
            "Accuracy 80.70833333333334\n",
            "Loss: 1.1359732151031494 Epoch: 6\n",
            "Accuracy 81.31666666666668\n",
            "Loss: 1.0554090738296509 Epoch: 7\n",
            "Accuracy 81.78333333333333\n",
            "Loss: 0.9926603436470032 Epoch: 8\n",
            "Accuracy 82.30833333333332\n",
            "Loss: 0.9416406154632568 Epoch: 9\n",
            "Accuracy 83.00833333333333\n",
            "Loss: 0.8905845284461975 Epoch: 10\n",
            "Accuracy 83.51666666666667\n",
            "Loss: 0.8438325524330139 Epoch: 11\n",
            "Accuracy 83.84166666666667\n",
            "Loss: 0.811427652835846 Epoch: 12\n",
            "Accuracy 84.275\n",
            "Loss: 0.7790567278862 Epoch: 13\n",
            "Accuracy 84.51666666666667\n",
            "Loss: 0.7505707144737244 Epoch: 14\n",
            "Accuracy 84.70833333333333\n",
            "Loss: 0.7272417545318604 Epoch: 15\n",
            "Accuracy 84.94166666666668\n",
            "Loss: 0.7089877724647522 Epoch: 16\n",
            "Accuracy 85.275\n",
            "Loss: 0.6927222609519958 Epoch: 17\n",
            "Accuracy 85.44166666666668\n",
            "Loss: 0.6682900190353394 Epoch: 18\n",
            "Accuracy 85.6\n",
            "Loss: 0.649955153465271 Epoch: 19\n",
            "Accuracy 85.81666666666666\n",
            "Loss: 0.6299166083335876 Epoch: 20\n",
            "Accuracy 86.11666666666666\n",
            "Loss: 0.6206786632537842 Epoch: 21\n",
            "Accuracy 86.175\n",
            "Loss: 0.601603627204895 Epoch: 22\n",
            "Accuracy 86.31666666666666\n",
            "Loss: 0.5856514573097229 Epoch: 23\n",
            "Accuracy 86.63333333333333\n",
            "Loss: 0.5842530727386475 Epoch: 24\n",
            "Accuracy 86.625\n",
            "Loss: 0.5708379149436951 Epoch: 25\n",
            "Accuracy 86.86666666666667\n",
            "Loss: 0.5564621090888977 Epoch: 26\n",
            "Accuracy 87.08333333333333\n",
            "Loss: 0.545384407043457 Epoch: 27\n",
            "Accuracy 87.14166666666667\n",
            "Loss: 0.5420868396759033 Epoch: 28\n",
            "Accuracy 87.18333333333334\n",
            "Loss: 0.5282434225082397 Epoch: 29\n",
            "Accuracy 87.225\n",
            "Loss: 0.5204820036888123 Epoch: 30\n",
            "Accuracy 87.36666666666667\n",
            "Loss: 0.5185960531234741 Epoch: 31\n",
            "Accuracy 87.50833333333333\n",
            "\n",
            "The running average of the accuracies has dropped below epsilon.\n",
            " We've run for 31 epochs. \n",
            " Training Complete!\n",
            "\n",
            "            IPA      CE_m      Acc_m  Running_Acc\n",
            "Epoch                                            \n",
            "0      0.000000  2.302599  10.216667     0.000000\n",
            "1      0.412070  1.890529  74.358333     0.000000\n",
            "2      0.108867  1.672796  77.825000     0.000000\n",
            "3      0.062506  1.485277  78.175000     0.000000\n",
            "4      0.035806  1.342052  79.683333     0.000000\n",
            "5      0.023500  1.224554  80.708333     0.000000\n",
            "6      0.014763  1.135973  81.316667     0.018163\n",
            "7      0.011509  1.055409  81.783333     0.009986\n",
            "8      0.007844  0.992660  82.308333     0.010371\n",
            "9      0.005669  0.941641  83.008333     0.008213\n",
            "10     0.005106  0.890585  83.516667     0.006865\n",
            "11     0.004250  0.843833  83.841667     0.006136\n",
            "12     0.002700  0.811428  84.275000     0.006022\n",
            "13     0.002490  0.779057  84.516667     0.005311\n",
            "14     0.002035  0.750571  84.708333     0.004064\n",
            "15     0.001555  0.727242  84.941667     0.003390\n",
            "16     0.001141  0.708988  85.275000     0.003397\n",
            "17     0.000957  0.692722  85.441667     0.002754\n",
            "18     0.001357  0.668290  85.600000     0.002551\n",
            "19     0.000965  0.649955  85.816667     0.002604\n",
            "20     0.001002  0.629917  86.116667     0.002752\n",
            "21     0.000440  0.620679  86.175000     0.002102\n",
            "22     0.000867  0.601604  86.316667     0.002040\n",
            "23     0.000694  0.585651  86.633333     0.002403\n",
            "24     0.000058  0.584253  86.625000     0.001916\n",
            "25     0.000537  0.570838  86.866667     0.001775\n",
            "26     0.000553  0.556462  87.083333     0.002139\n",
            "27     0.000410  0.545384  87.141667     0.001944\n",
            "28     0.000118  0.542087  87.183333     0.001306\n",
            "29     0.000477  0.528243  87.225000     0.001382\n",
            "30     0.000259  0.520482  87.366667     0.001149\n",
            "31     0.000061  0.518596  87.508333     0.000974\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/linear.py:114: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  return F.linear(input, self.weight, self.bias)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Initial Loss: 2.3025989532470703\n",
            "Accuracy 10.216666666666667\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/autograd/__init__.py:197: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss: 1.8910458087921143 Epoch: 1\n",
            "Accuracy 74.25\n",
            "Loss: 1.6704988479614258 Epoch: 2\n",
            "Accuracy 77.875\n",
            "Loss: 1.4898216724395752 Epoch: 3\n",
            "Accuracy 79.14166666666667\n",
            "Loss: 1.3476485013961792 Epoch: 4\n",
            "Accuracy 79.925\n",
            "Loss: 1.2286522388458252 Epoch: 5\n",
            "Accuracy 80.63333333333334\n",
            "Loss: 1.1395423412322998 Epoch: 6\n",
            "Accuracy 81.21666666666667\n",
            "Loss: 1.0563079118728638 Epoch: 7\n",
            "Accuracy 81.825\n",
            "Loss: 0.9972947239875793 Epoch: 8\n",
            "Accuracy 82.34166666666667\n",
            "Loss: 0.9444856643676758 Epoch: 9\n",
            "Accuracy 82.83333333333334\n",
            "Loss: 0.8932732343673706 Epoch: 10\n",
            "Accuracy 83.66666666666667\n",
            "Loss: 0.844897985458374 Epoch: 11\n",
            "Accuracy 84.04166666666667\n",
            "Loss: 0.8131933808326721 Epoch: 12\n",
            "Accuracy 84.15\n",
            "Loss: 0.7780600190162659 Epoch: 13\n",
            "Accuracy 84.52499999999999\n",
            "Loss: 0.7556237578392029 Epoch: 14\n",
            "Accuracy 84.79166666666667\n",
            "Loss: 0.7314147353172302 Epoch: 15\n",
            "Accuracy 85.02499999999999\n",
            "Loss: 0.711959183216095 Epoch: 16\n",
            "Accuracy 85.28333333333333\n",
            "Loss: 0.6923105716705322 Epoch: 17\n",
            "Accuracy 85.41666666666666\n",
            "Loss: 0.6702122092247009 Epoch: 18\n",
            "Accuracy 85.56666666666666\n",
            "Loss: 0.6493304967880249 Epoch: 19\n",
            "Accuracy 85.85833333333333\n",
            "Loss: 0.6358963251113892 Epoch: 20\n",
            "Accuracy 86.125\n",
            "Loss: 0.621818482875824 Epoch: 21\n",
            "Accuracy 86.16666666666667\n",
            "Loss: 0.5977468490600586 Epoch: 22\n",
            "Accuracy 86.275\n",
            "Loss: 0.5968338847160339 Epoch: 23\n",
            "Accuracy 86.53333333333333\n",
            "Loss: 0.579856276512146 Epoch: 24\n",
            "Accuracy 86.725\n",
            "Loss: 0.5758302807807922 Epoch: 25\n",
            "Accuracy 86.74166666666666\n",
            "Loss: 0.5662686228752136 Epoch: 26\n",
            "Accuracy 86.925\n",
            "Loss: 0.543464720249176 Epoch: 27\n",
            "Accuracy 87.10833333333333\n",
            "Loss: 0.5398737192153931 Epoch: 28\n",
            "Accuracy 87.15833333333333\n",
            "Loss: 0.5306658744812012 Epoch: 29\n",
            "Accuracy 87.325\n",
            "Loss: 0.5251679420471191 Epoch: 30\n",
            "Accuracy 87.3\n",
            "Loss: 0.5143964290618896 Epoch: 31\n",
            "Accuracy 87.53333333333333\n",
            "Loss: 0.50103360414505 Epoch: 32\n",
            "Accuracy 87.59166666666667\n",
            "Loss: 0.4977748990058899 Epoch: 33\n",
            "Accuracy 87.68333333333334\n",
            "Loss: 0.49684804677963257 Epoch: 34\n",
            "Accuracy 87.775\n",
            "Loss: 0.48020628094673157 Epoch: 35\n",
            "Accuracy 87.85833333333333\n",
            "Loss: 0.4750239849090576 Epoch: 36\n",
            "Accuracy 87.95833333333334\n",
            "\n",
            "The running average of the accuracies has dropped below epsilon.\n",
            " We've run for 36 epochs. \n",
            " Training Complete!\n",
            "\n",
            "            IPA      CE_m      Acc_m  Running_Acc\n",
            "Epoch                                            \n",
            "0      0.000000  2.302599  10.216667     0.000000\n",
            "1      0.411553  1.891046  74.250000     0.000000\n",
            "2      0.110273  1.670499  77.875000     0.000000\n",
            "3      0.060226  1.489822  79.141667     0.000000\n",
            "4      0.035543  1.347649  79.925000     0.000000\n",
            "5      0.023799  1.228652  80.633333     0.000000\n",
            "6      0.014852  1.139542  81.216667     0.018216\n",
            "7      0.011891  1.056308  81.825000     0.009950\n",
            "8      0.007377  0.997295  82.341667     0.007960\n",
            "9      0.005868  0.944486  82.833333     0.007174\n",
            "10     0.005121  0.893273  83.666667     0.007414\n",
            "11     0.004398  0.844898  84.041667     0.006864\n",
            "12     0.002642  0.813193  84.150000     0.005623\n",
            "13     0.002703  0.778060  84.525000     0.005252\n",
            "14     0.001603  0.755624  84.791667     0.004689\n",
            "15     0.001614  0.731415  85.025000     0.003227\n",
            "16     0.001216  0.711959  85.283333     0.002938\n",
            "17     0.001156  0.692311  85.416667     0.002993\n",
            "18     0.001228  0.670212  85.566667     0.002453\n",
            "19     0.001099  0.649330  85.858333     0.002504\n",
            "20     0.000672  0.635896  86.125000     0.002574\n",
            "21     0.000670  0.621818  86.166667     0.002064\n",
            "22     0.001094  0.597747  86.275000     0.002002\n",
            "23     0.000040  0.596834  86.533333     0.002250\n",
            "24     0.000707  0.579856  86.725000     0.002011\n",
            "25     0.000161  0.575830  86.741667     0.001428\n",
            "26     0.000368  0.566269  86.925000     0.001754\n",
            "27     0.000845  0.543465  87.108333     0.001925\n",
            "28     0.000128  0.539874  87.158333     0.001441\n",
            "29     0.000318  0.530666  87.325000     0.001380\n",
            "30     0.000183  0.525168  87.300000     0.001399\n",
            "31     0.000347  0.514396  87.533333     0.001511\n",
            "32     0.000418  0.501034  87.591667     0.001222\n",
            "33     0.000099  0.497775  87.683333     0.001317\n",
            "34     0.000027  0.496848  87.775000     0.001143\n",
            "35     0.000475  0.480206  87.858333     0.001276\n",
            "36     0.000144  0.475024  87.958333     0.000969\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/linear.py:114: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  return F.linear(input, self.weight, self.bias)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Initial Loss: 2.3025989532470703\n",
            "Accuracy 10.216666666666667\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/autograd/__init__.py:197: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss: 1.8915804624557495 Epoch: 1\n",
            "Accuracy 74.6\n",
            "Loss: 1.6698434352874756 Epoch: 2\n",
            "Accuracy 77.09166666666667\n",
            "Loss: 1.4929873943328857 Epoch: 3\n",
            "Accuracy 78.57499999999999\n",
            "Loss: 1.347090721130371 Epoch: 4\n",
            "Accuracy 79.83333333333333\n",
            "Loss: 1.224437952041626 Epoch: 5\n",
            "Accuracy 80.48333333333333\n",
            "Loss: 1.1372321844100952 Epoch: 6\n",
            "Accuracy 81.26666666666667\n",
            "Loss: 1.0556292533874512 Epoch: 7\n",
            "Accuracy 81.825\n",
            "Loss: 0.9926770925521851 Epoch: 8\n",
            "Accuracy 82.30833333333332\n",
            "Loss: 0.9359555244445801 Epoch: 9\n",
            "Accuracy 82.825\n",
            "Loss: 0.8942230939865112 Epoch: 10\n",
            "Accuracy 83.11666666666667\n",
            "Loss: 0.8517083525657654 Epoch: 11\n",
            "Accuracy 83.78333333333333\n",
            "Loss: 0.8170813322067261 Epoch: 12\n",
            "Accuracy 83.78333333333333\n",
            "Loss: 0.7854544520378113 Epoch: 13\n",
            "Accuracy 84.39166666666667\n",
            "Loss: 0.7562490701675415 Epoch: 14\n",
            "Accuracy 84.61666666666666\n",
            "Loss: 0.7260082960128784 Epoch: 15\n",
            "Accuracy 84.98333333333333\n",
            "Loss: 0.7085961103439331 Epoch: 16\n",
            "Accuracy 85.3\n",
            "Loss: 0.6854485869407654 Epoch: 17\n",
            "Accuracy 85.35000000000001\n",
            "Loss: 0.6696322560310364 Epoch: 18\n",
            "Accuracy 85.63333333333333\n",
            "Loss: 0.6492342948913574 Epoch: 19\n",
            "Accuracy 85.88333333333334\n",
            "Loss: 0.6349031925201416 Epoch: 20\n",
            "Accuracy 86.00833333333333\n",
            "Loss: 0.6169605851173401 Epoch: 21\n",
            "Accuracy 86.20833333333333\n",
            "Loss: 0.6035875082015991 Epoch: 22\n",
            "Accuracy 86.31666666666666\n",
            "Loss: 0.5852065682411194 Epoch: 23\n",
            "Accuracy 86.63333333333333\n",
            "Loss: 0.5782994031906128 Epoch: 24\n",
            "Accuracy 86.68333333333334\n",
            "Loss: 0.5695876479148865 Epoch: 25\n",
            "Accuracy 86.925\n",
            "Loss: 0.5559898018836975 Epoch: 26\n",
            "Accuracy 87.075\n",
            "Loss: 0.5505074858665466 Epoch: 27\n",
            "Accuracy 87.04166666666666\n",
            "Loss: 0.5412852764129639 Epoch: 28\n",
            "Accuracy 87.225\n",
            "Loss: 0.5216044783592224 Epoch: 29\n",
            "Accuracy 87.3\n",
            "Loss: 0.517917811870575 Epoch: 30\n",
            "Accuracy 87.35833333333333\n",
            "Loss: 0.509852409362793 Epoch: 31\n",
            "Accuracy 87.48333333333333\n",
            "Loss: 0.4996033012866974 Epoch: 32\n",
            "Accuracy 87.59166666666667\n",
            "Loss: 0.4992232620716095 Epoch: 33\n",
            "Accuracy 87.675\n",
            "Loss: 0.4874379336833954 Epoch: 34\n",
            "Accuracy 87.83333333333333\n",
            "Loss: 0.4771186113357544 Epoch: 35\n",
            "Accuracy 87.825\n",
            "Loss: 0.4869690239429474 Epoch: 36\n",
            "Accuracy 87.90833333333333\n",
            "Loss: 0.4655849039554596 Epoch: 37\n",
            "Accuracy 88.06666666666668\n",
            "Loss: 0.46869128942489624 Epoch: 38\n",
            "Accuracy 88.16666666666667\n",
            "Loss: 0.46671420335769653 Epoch: 39\n",
            "Accuracy 88.175\n",
            "\n",
            "The running average of the accuracies has dropped below epsilon.\n",
            " We've run for 39 epochs. \n",
            " Training Complete!\n",
            "\n",
            "            IPA      CE_m      Acc_m  Running_Acc\n",
            "Epoch                                            \n",
            "0      0.000000  2.302599  10.216667     0.000000\n",
            "1      0.411018  1.891580  74.600000     0.000000\n",
            "2      0.110869  1.669843  77.091667     0.000000\n",
            "3      0.058952  1.492987  78.575000     0.000000\n",
            "4      0.036474  1.347091  79.833333     0.000000\n",
            "5      0.024531  1.224438  80.483333     0.000000\n",
            "6      0.014534  1.137232  81.266667     0.017306\n",
            "7      0.011658  1.055629  81.825000     0.012000\n",
            "8      0.007869  0.992677  82.308333     0.009333\n",
            "9      0.006302  0.935956  82.825000     0.007386\n",
            "10     0.004173  0.894223  83.116667     0.006462\n",
            "11     0.003865  0.851708  83.783333     0.006119\n",
            "12     0.002886  0.817081  83.783333     0.004745\n",
            "13     0.002433  0.785454  84.391667     0.005016\n",
            "14     0.002086  0.756249  84.616667     0.004294\n",
            "15     0.002016  0.726008  84.983333     0.004456\n",
            "16     0.001088  0.708596  85.300000     0.003597\n",
            "17     0.001362  0.685449  85.350000     0.003715\n",
            "18     0.000879  0.669632  85.633333     0.002926\n",
            "19     0.001074  0.649234  85.883333     0.002977\n",
            "20     0.000717  0.634903  86.008333     0.002401\n",
            "21     0.000854  0.616961  86.208333     0.002121\n",
            "22     0.000608  0.603588  86.316667     0.002255\n",
            "23     0.000799  0.585207  86.633333     0.002325\n",
            "24     0.000288  0.578299  86.683333     0.001857\n",
            "25     0.000348  0.569588  86.925000     0.002123\n",
            "26     0.000523  0.555990  87.075000     0.002003\n",
            "27     0.000203  0.550507  87.041667     0.001828\n",
            "28     0.000329  0.541285  87.225000     0.001516\n",
            "29     0.000679  0.521604  87.300000     0.001572\n",
            "30     0.000123  0.517918  87.358333     0.001149\n",
            "31     0.000260  0.509852  87.483333     0.001090\n",
            "32     0.000320  0.499603  87.591667     0.001261\n",
            "33     0.000012  0.499223  87.675000     0.001030\n",
            "34     0.000347  0.487438  87.833333     0.001219\n",
            "35     0.000295  0.477119  87.825000     0.001104\n",
            "36     0.000274  0.486969  87.908333     0.001008\n",
            "37     0.000578  0.465585  88.066667     0.001120\n",
            "38     0.000082  0.468691  88.166667     0.001157\n",
            "39     0.000051  0.466714  88.175000     0.000815\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/linear.py:114: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  return F.linear(input, self.weight, self.bias)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Initial Loss: 2.3025989532470703\n",
            "Accuracy 10.216666666666667\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/autograd/__init__.py:197: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss: 1.8917862176895142 Epoch: 1\n",
            "Accuracy 75.275\n",
            "Loss: 1.6686304807662964 Epoch: 2\n",
            "Accuracy 77.80833333333334\n",
            "Loss: 1.4914743900299072 Epoch: 3\n",
            "Accuracy 78.77499999999999\n",
            "Loss: 1.3468260765075684 Epoch: 4\n",
            "Accuracy 80.15833333333333\n",
            "Loss: 1.224792242050171 Epoch: 5\n",
            "Accuracy 80.56666666666666\n",
            "Loss: 1.1386057138442993 Epoch: 6\n",
            "Accuracy 81.18333333333332\n",
            "Loss: 1.0571303367614746 Epoch: 7\n",
            "Accuracy 81.63333333333334\n",
            "Loss: 0.9943615198135376 Epoch: 8\n",
            "Accuracy 82.36666666666666\n",
            "Loss: 0.9312151670455933 Epoch: 9\n",
            "Accuracy 82.85833333333333\n",
            "Loss: 0.8949388265609741 Epoch: 10\n",
            "Accuracy 83.3\n",
            "Loss: 0.848685085773468 Epoch: 11\n",
            "Accuracy 83.78333333333333\n",
            "Loss: 0.8051274418830872 Epoch: 12\n",
            "Accuracy 84.14166666666667\n",
            "Loss: 0.7855566143989563 Epoch: 13\n",
            "Accuracy 84.36666666666667\n",
            "Loss: 0.7521807551383972 Epoch: 14\n",
            "Accuracy 84.73333333333333\n",
            "Loss: 0.7266879081726074 Epoch: 15\n",
            "Accuracy 84.94166666666668\n",
            "Loss: 0.7126633524894714 Epoch: 16\n",
            "Accuracy 85.35000000000001\n",
            "Loss: 0.6889196634292603 Epoch: 17\n",
            "Accuracy 85.45\n",
            "Loss: 0.6699514389038086 Epoch: 18\n",
            "Accuracy 85.56666666666666\n",
            "Loss: 0.6530645489692688 Epoch: 19\n",
            "Accuracy 85.86666666666667\n",
            "Loss: 0.6335507035255432 Epoch: 20\n",
            "Accuracy 85.93333333333332\n",
            "Loss: 0.6202826499938965 Epoch: 21\n",
            "Accuracy 86.275\n",
            "Loss: 0.6049890518188477 Epoch: 22\n",
            "Accuracy 86.45833333333334\n",
            "Loss: 0.5833069086074829 Epoch: 23\n",
            "Accuracy 86.5\n",
            "Loss: 0.5782331228256226 Epoch: 24\n",
            "Accuracy 86.6\n",
            "Loss: 0.5678837299346924 Epoch: 25\n",
            "Accuracy 86.85000000000001\n",
            "Loss: 0.5599324107170105 Epoch: 26\n",
            "Accuracy 87.09166666666667\n",
            "Loss: 0.546840488910675 Epoch: 27\n",
            "Accuracy 87.08333333333333\n",
            "Loss: 0.5371208786964417 Epoch: 28\n",
            "Accuracy 87.28333333333333\n",
            "Loss: 0.5280238389968872 Epoch: 29\n",
            "Accuracy 87.31666666666666\n",
            "Loss: 0.5153853893280029 Epoch: 30\n",
            "Accuracy 87.49166666666667\n",
            "Loss: 0.5143100619316101 Epoch: 31\n",
            "Accuracy 87.56666666666668\n",
            "Loss: 0.5026889443397522 Epoch: 32\n",
            "Accuracy 87.625\n",
            "Loss: 0.4957817494869232 Epoch: 33\n",
            "Accuracy 87.53333333333333\n",
            "\n",
            "The running average of the accuracies has dropped below epsilon.\n",
            " We've run for 33 epochs. \n",
            " Training Complete!\n",
            "\n",
            "            IPA      CE_m      Acc_m  Running_Acc\n",
            "Epoch                                            \n",
            "0      0.000000  2.302599  10.216667     0.000000\n",
            "1      0.410813  1.891786  75.275000     0.000000\n",
            "2      0.111578  1.668630  77.808333     0.000000\n",
            "3      0.059052  1.491474  78.775000     0.000000\n",
            "4      0.036162  1.346826  80.158333     0.000000\n",
            "5      0.024407  1.224792  80.566667     0.000000\n",
            "6      0.014364  1.138606  81.183333     0.015277\n",
            "7      0.011639  1.057130  81.633333     0.009655\n",
            "8      0.007846  0.994362  82.366667     0.008967\n",
            "9      0.007016  0.931215  82.858333     0.006649\n",
            "10     0.003628  0.894939  83.300000     0.006696\n",
            "11     0.004205  0.848685  83.783333     0.006326\n",
            "12     0.003630  0.805127  84.141667     0.006072\n",
            "13     0.001505  0.785557  84.366667     0.004811\n",
            "14     0.002384  0.752181  84.733333     0.004486\n",
            "15     0.001700  0.726688  84.941667     0.003912\n",
            "16     0.000877  0.712663  85.350000     0.003713\n",
            "17     0.001397  0.688920  85.450000     0.003092\n",
            "18     0.001054  0.669951  85.566667     0.002830\n",
            "19     0.000889  0.653065  85.866667     0.002662\n",
            "20     0.000976  0.633551  85.933333     0.002325\n",
            "21     0.000632  0.620283  86.275000     0.002159\n",
            "22     0.000695  0.604989  86.458333     0.002350\n",
            "23     0.000943  0.583307  86.500000     0.002173\n",
            "24     0.000211  0.578233  86.600000     0.001703\n",
            "25     0.000414  0.567884  86.850000     0.002125\n",
            "26     0.000306  0.559932  87.091667     0.001886\n",
            "27     0.000485  0.546840  87.083333     0.001481\n",
            "28     0.000347  0.537121  87.283333     0.001844\n",
            "29     0.000314  0.528024  87.316667     0.001689\n",
            "30     0.000421  0.515385  87.491667     0.001512\n",
            "31     0.000035  0.514310  87.566667     0.001127\n",
            "32     0.000363  0.502689  87.625000     0.001241\n",
            "33     0.000209  0.495782  87.533333     0.000991\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/linear.py:114: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  return F.linear(input, self.weight, self.bias)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Initial Loss: 2.3025989532470703\n",
            "Accuracy 10.216666666666667\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/autograd/__init__.py:197: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss: 1.8935481309890747 Epoch: 1\n",
            "Accuracy 76.08333333333334\n",
            "Loss: 1.670814871788025 Epoch: 2\n",
            "Accuracy 77.90833333333333\n",
            "Loss: 1.4935656785964966 Epoch: 3\n",
            "Accuracy 79.15833333333333\n",
            "Loss: 1.344481348991394 Epoch: 4\n",
            "Accuracy 80.08333333333333\n",
            "Loss: 1.2409831285476685 Epoch: 5\n",
            "Accuracy 80.875\n",
            "Loss: 1.1387698650360107 Epoch: 6\n",
            "Accuracy 81.01666666666667\n",
            "Loss: 1.061334490776062 Epoch: 7\n",
            "Accuracy 81.65833333333333\n",
            "Loss: 0.9981861114501953 Epoch: 8\n",
            "Accuracy 82.29166666666666\n",
            "Loss: 0.9439547657966614 Epoch: 9\n",
            "Accuracy 82.73333333333333\n",
            "Loss: 0.8883967399597168 Epoch: 10\n",
            "Accuracy 83.35833333333333\n",
            "Loss: 0.8495694398880005 Epoch: 11\n",
            "Accuracy 83.54166666666667\n",
            "Loss: 0.8110390901565552 Epoch: 12\n",
            "Accuracy 84.06666666666666\n",
            "Loss: 0.7832357287406921 Epoch: 13\n",
            "Accuracy 84.5\n",
            "Loss: 0.7584236860275269 Epoch: 14\n",
            "Accuracy 84.59166666666667\n",
            "Loss: 0.7307506203651428 Epoch: 15\n",
            "Accuracy 85.04166666666667\n",
            "Loss: 0.708432137966156 Epoch: 16\n",
            "Accuracy 85.10833333333333\n",
            "Loss: 0.6898943781852722 Epoch: 17\n",
            "Accuracy 85.475\n",
            "Loss: 0.6633111238479614 Epoch: 18\n",
            "Accuracy 85.625\n",
            "Loss: 0.6572896838188171 Epoch: 19\n",
            "Accuracy 85.70833333333333\n",
            "Loss: 0.6342423558235168 Epoch: 20\n",
            "Accuracy 85.81666666666666\n",
            "Loss: 0.6208329796791077 Epoch: 21\n",
            "Accuracy 86.1\n",
            "Loss: 0.6085728406906128 Epoch: 22\n",
            "Accuracy 86.34166666666667\n",
            "Loss: 0.5891072750091553 Epoch: 23\n",
            "Accuracy 86.55000000000001\n",
            "Loss: 0.5804514288902283 Epoch: 24\n",
            "Accuracy 86.725\n",
            "Loss: 0.5670924186706543 Epoch: 25\n",
            "Accuracy 86.95833333333334\n",
            "Loss: 0.5588619709014893 Epoch: 26\n",
            "Accuracy 86.98333333333333\n",
            "Loss: 0.548026978969574 Epoch: 27\n",
            "Accuracy 87.05833333333334\n",
            "Loss: 0.5394799113273621 Epoch: 28\n",
            "Accuracy 87.25\n",
            "Loss: 0.5281217098236084 Epoch: 29\n",
            "Accuracy 87.25\n",
            "Loss: 0.5227575302124023 Epoch: 30\n",
            "Accuracy 87.425\n",
            "Loss: 0.5140307545661926 Epoch: 31\n",
            "Accuracy 87.5\n",
            "Loss: 0.5020768046379089 Epoch: 32\n",
            "Accuracy 87.64999999999999\n",
            "Loss: 0.5037429332733154 Epoch: 33\n",
            "Accuracy 87.74166666666666\n",
            "Loss: 0.48956504464149475 Epoch: 34\n",
            "Accuracy 87.84166666666667\n",
            "Loss: 0.4807603657245636 Epoch: 35\n",
            "Accuracy 87.85833333333333\n",
            "\n",
            "The running average of the accuracies has dropped below epsilon.\n",
            " We've run for 35 epochs. \n",
            " Training Complete!\n",
            "\n",
            "            IPA      CE_m      Acc_m  Running_Acc\n",
            "Epoch                                            \n",
            "0      0.000000  2.302599  10.216667     0.000000\n",
            "1      0.409051  1.893548  76.083333     0.000000\n",
            "2      0.111367  1.670815  77.908333     0.000000\n",
            "3      0.059083  1.493566  79.158333     0.000000\n",
            "4      0.037271  1.344481  80.083333     0.000000\n",
            "5      0.020700  1.240983  80.875000     0.000000\n",
            "6      0.017036  1.138770  81.016667     0.012671\n",
            "7      0.011062  1.061334  81.658333     0.009457\n",
            "8      0.007894  0.998186  82.291667     0.007800\n",
            "9      0.006026  0.943955  82.733333     0.006536\n",
            "10     0.005556  0.888397  83.358333     0.006070\n",
            "11     0.003530  0.849569  83.541667     0.006159\n",
            "12     0.003211  0.811039  84.066667     0.005832\n",
            "13     0.002139  0.783236  84.500000     0.005312\n",
            "14     0.001772  0.758424  84.591667     0.004455\n",
            "15     0.001845  0.730751  85.041667     0.004009\n",
            "16     0.001395  0.708432  85.108333     0.003725\n",
            "17     0.001090  0.689894  85.475000     0.003330\n",
            "18     0.001477  0.663311  85.625000     0.002650\n",
            "19     0.000317  0.657290  85.708333     0.002628\n",
            "20     0.001152  0.634242  85.816667     0.001817\n",
            "21     0.000639  0.620833  86.100000     0.002320\n",
            "22     0.000557  0.608573  86.341667     0.002020\n",
            "23     0.000846  0.589107  86.550000     0.002152\n",
            "24     0.000361  0.580451  86.725000     0.002361\n",
            "25     0.000534  0.567092  86.958333     0.002647\n",
            "26     0.000317  0.558862  86.983333     0.002044\n",
            "27     0.000401  0.548027  87.058333     0.001655\n",
            "28     0.000305  0.539480  87.250000     0.001613\n",
            "29     0.000392  0.528122  87.250000     0.001208\n",
            "30     0.000179  0.522758  87.425000     0.001071\n",
            "31     0.000282  0.514031  87.500000     0.001185\n",
            "32     0.000374  0.502077  87.650000     0.001356\n",
            "33     0.000050  0.503743  87.741667     0.001125\n",
            "34     0.000417  0.489565  87.841667     0.001353\n",
            "35     0.000252  0.480760  87.858333     0.000989\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/linear.py:114: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  return F.linear(input, self.weight, self.bias)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Initial Loss: 2.3025989532470703\n",
            "Accuracy 10.216666666666667\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/autograd/__init__.py:197: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss: 1.8933298587799072 Epoch: 1\n",
            "Accuracy 75.86666666666667\n",
            "Loss: 1.6704013347625732 Epoch: 2\n",
            "Accuracy 77.79166666666667\n",
            "Loss: 1.4907525777816772 Epoch: 3\n",
            "Accuracy 78.61666666666667\n",
            "Loss: 1.3451288938522339 Epoch: 4\n",
            "Accuracy 79.525\n",
            "Loss: 1.233992576599121 Epoch: 5\n",
            "Accuracy 80.85833333333333\n",
            "Loss: 1.1393392086029053 Epoch: 6\n",
            "Accuracy 81.21666666666667\n",
            "Loss: 1.0580780506134033 Epoch: 7\n",
            "Accuracy 82.06666666666666\n",
            "Loss: 0.9934723377227783 Epoch: 8\n",
            "Accuracy 82.175\n",
            "Loss: 0.9420905113220215 Epoch: 9\n",
            "Accuracy 83.19166666666666\n",
            "Loss: 0.8881763219833374 Epoch: 10\n",
            "Accuracy 83.325\n",
            "Loss: 0.8512718677520752 Epoch: 11\n",
            "Accuracy 83.825\n",
            "Loss: 0.8194407224655151 Epoch: 12\n",
            "Accuracy 84.05\n",
            "Loss: 0.7827358245849609 Epoch: 13\n",
            "Accuracy 84.6\n",
            "Loss: 0.7514404058456421 Epoch: 14\n",
            "Accuracy 84.78333333333333\n",
            "Loss: 0.7339921593666077 Epoch: 15\n",
            "Accuracy 85.06666666666666\n",
            "Loss: 0.7055457830429077 Epoch: 16\n",
            "Accuracy 85.29166666666667\n",
            "Loss: 0.6850253343582153 Epoch: 17\n",
            "Accuracy 85.38333333333334\n",
            "Loss: 0.6605629920959473 Epoch: 18\n",
            "Accuracy 85.66666666666667\n",
            "Loss: 0.6430298686027527 Epoch: 19\n",
            "Accuracy 85.80833333333334\n",
            "Loss: 0.6283044219017029 Epoch: 20\n",
            "Accuracy 85.90833333333333\n",
            "Loss: 0.6206321120262146 Epoch: 21\n",
            "Accuracy 86.11666666666666\n",
            "Loss: 0.6056341528892517 Epoch: 22\n",
            "Accuracy 86.35833333333333\n",
            "Loss: 0.5926758646965027 Epoch: 23\n",
            "Accuracy 86.66666666666667\n",
            "Loss: 0.5768190622329712 Epoch: 24\n",
            "Accuracy 86.825\n",
            "Loss: 0.5592200756072998 Epoch: 25\n",
            "Accuracy 86.85833333333333\n",
            "Loss: 0.5515626668930054 Epoch: 26\n",
            "Accuracy 86.95833333333334\n",
            "Loss: 0.5447475910186768 Epoch: 27\n",
            "Accuracy 87.0\n",
            "Loss: 0.538040280342102 Epoch: 28\n",
            "Accuracy 87.19166666666666\n",
            "Loss: 0.528672993183136 Epoch: 29\n",
            "Accuracy 87.16666666666667\n",
            "\n",
            "The running average of the accuracies has dropped below epsilon.\n",
            " We've run for 29 epochs. \n",
            " Training Complete!\n",
            "\n",
            "            IPA      CE_m      Acc_m  Running_Acc\n",
            "Epoch                                            \n",
            "0      0.000000  2.302599  10.216667     0.000000\n",
            "1      0.409269  1.893330  75.866667     0.000000\n",
            "2      0.111464  1.670401  77.791667     0.000000\n",
            "3      0.059883  1.490753  78.616667     0.000000\n",
            "4      0.036406  1.345129  79.525000     0.000000\n",
            "5      0.022227  1.233993  80.858333     0.000000\n",
            "6      0.015776  1.139339  81.216667     0.013746\n",
            "7      0.011609  1.058078  82.066667     0.010765\n",
            "8      0.008076  0.993472  82.175000     0.008908\n",
            "9      0.005709  0.942091  83.191667     0.009071\n",
            "10     0.005391  0.888176  83.325000     0.006038\n",
            "11     0.003355  0.851272  83.825000     0.006352\n",
            "12     0.002653  0.819441  84.050000     0.004796\n",
            "13     0.002823  0.782736  84.600000     0.005841\n",
            "14     0.002235  0.751440  84.783333     0.003800\n",
            "15     0.001163  0.733992  85.066667     0.004147\n",
            "16     0.001778  0.705546  85.291667     0.003476\n",
            "17     0.001207  0.685025  85.383333     0.003154\n",
            "18     0.001359  0.660563  85.666667     0.002509\n",
            "19     0.000923  0.643030  85.808333     0.002407\n",
            "20     0.000736  0.628304  85.908333     0.001971\n",
            "21     0.000365  0.620632  86.116667     0.001927\n",
            "22     0.000682  0.605634  86.358333     0.002274\n",
            "23     0.000563  0.592676  86.666667     0.002324\n",
            "24     0.000661  0.576819  86.825000     0.002359\n",
            "25     0.000704  0.559220  86.858333     0.002203\n",
            "26     0.000295  0.551563  86.958333     0.001948\n",
            "27     0.000252  0.544748  87.000000     0.001482\n",
            "28     0.000240  0.538040  87.191667     0.001209\n",
            "29     0.000323  0.528673  87.166667     0.000901\n"
          ]
        }
      ],
      "source": [
        "#TESTING\n",
        "for i in range (10): #RUNS TEN ITERATIONS TO TEST CONSISTENCY AND REPRODUCIBILITY OF RESULTS. TAKES A WHILE.\n",
        "  m = Neural_Network(batch_size = 30000, learning_rate = 1).to(0)\n",
        "  train(m, CUDA = True, criterion = 'Accuracy', stop_epoch = 100)\n",
        "  datadf = pd.DataFrame(data = m.data)\n",
        "  datadf.set_index('Epoch', inplace = True)\n",
        "  print(datadf)\n",
        "  datadf.to_csv(f'batch_{m.batch_size}.lr_{m.my_lr}_{i}.csv')"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "765159e27987410d8a1fa40e596f9879": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_316abce840654cd095ecd559e55f5b01",
              "IPY_MODEL_9a5edd0bf33d440bbf68cc4c83afdc29",
              "IPY_MODEL_9e5f93cbba6b4fcab6dfb9d84b3f4d09"
            ],
            "layout": "IPY_MODEL_f0464d4e9d084b108190ed51029a88c1"
          }
        },
        "316abce840654cd095ecd559e55f5b01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4e9cbf9035b3446aaa3947683ed52fe6",
            "placeholder": "​",
            "style": "IPY_MODEL_c5f5a4bccd924db9936bcbc1ae62ef1d",
            "value": "100%"
          }
        },
        "9a5edd0bf33d440bbf68cc4c83afdc29": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b0cf56e758784f609999e65aa21023d8",
            "max": 9912422,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6c8c8e3748774935892a6b456b28890d",
            "value": 9912422
          }
        },
        "9e5f93cbba6b4fcab6dfb9d84b3f4d09": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c2fc96e6d4d94812a73dced45ecbe208",
            "placeholder": "​",
            "style": "IPY_MODEL_b9261f7c581b4a53ac12a5f347601386",
            "value": " 9912422/9912422 [00:00&lt;00:00, 206158166.32it/s]"
          }
        },
        "f0464d4e9d084b108190ed51029a88c1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4e9cbf9035b3446aaa3947683ed52fe6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c5f5a4bccd924db9936bcbc1ae62ef1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b0cf56e758784f609999e65aa21023d8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6c8c8e3748774935892a6b456b28890d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c2fc96e6d4d94812a73dced45ecbe208": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b9261f7c581b4a53ac12a5f347601386": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7cf97933218a4f4292eae704cb6566e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7822b403bf9e477c9f198d74420f47ef",
              "IPY_MODEL_7c60e0db1d834ee7b517fe173fc3dd7a",
              "IPY_MODEL_91b3e96c7b2e4a7aa119277d9faf83c7"
            ],
            "layout": "IPY_MODEL_e36d8cbb54ba4226bcc74778778131f8"
          }
        },
        "7822b403bf9e477c9f198d74420f47ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b9e693c2ec2a490e8d35e6729e7d65f4",
            "placeholder": "​",
            "style": "IPY_MODEL_4708ef12720447848e96ad692eea9a27",
            "value": "100%"
          }
        },
        "7c60e0db1d834ee7b517fe173fc3dd7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_204ae6fd20cd4bb5a0b814d34ddbce9c",
            "max": 28881,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c549d913001a4594852bb7f1d92e6988",
            "value": 28881
          }
        },
        "91b3e96c7b2e4a7aa119277d9faf83c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b8614efca9ba41b6abbd939c2326fd3d",
            "placeholder": "​",
            "style": "IPY_MODEL_d206bb4635b744299007c304d7abbe31",
            "value": " 28881/28881 [00:00&lt;00:00, 2200466.74it/s]"
          }
        },
        "e36d8cbb54ba4226bcc74778778131f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b9e693c2ec2a490e8d35e6729e7d65f4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4708ef12720447848e96ad692eea9a27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "204ae6fd20cd4bb5a0b814d34ddbce9c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c549d913001a4594852bb7f1d92e6988": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b8614efca9ba41b6abbd939c2326fd3d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d206bb4635b744299007c304d7abbe31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ee0c44627e1645688d9171009d396bf3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f3931f1334724d0f8ba643874c4d90f0",
              "IPY_MODEL_c3c27b2ce4ac472e82e1fb48605c21ac",
              "IPY_MODEL_e2ba1837f14b412e919a61b8040ba5a9"
            ],
            "layout": "IPY_MODEL_882dc860b8604bc2812c8b662a86b61e"
          }
        },
        "f3931f1334724d0f8ba643874c4d90f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a3f41e9efa144f7082033a16b7e4e316",
            "placeholder": "​",
            "style": "IPY_MODEL_94a2bac4195749f98df4825d39eab3f1",
            "value": "100%"
          }
        },
        "c3c27b2ce4ac472e82e1fb48605c21ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c0ffed22b27f48fb958c67d693060dcc",
            "max": 1648877,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f3d19517f8374f4883965e5627c49581",
            "value": 1648877
          }
        },
        "e2ba1837f14b412e919a61b8040ba5a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6e374633889c4cbf83eaae5fcdfe481e",
            "placeholder": "​",
            "style": "IPY_MODEL_021d6b8d1fb74567b109d875207e19fd",
            "value": " 1648877/1648877 [00:00&lt;00:00, 46272214.13it/s]"
          }
        },
        "882dc860b8604bc2812c8b662a86b61e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a3f41e9efa144f7082033a16b7e4e316": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "94a2bac4195749f98df4825d39eab3f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c0ffed22b27f48fb958c67d693060dcc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f3d19517f8374f4883965e5627c49581": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6e374633889c4cbf83eaae5fcdfe481e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "021d6b8d1fb74567b109d875207e19fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2eae3cc183584c3c8426af786d9683b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3750e853fef54f329e4b0a3babf53ac3",
              "IPY_MODEL_89fdb54347c640f592e26e97f1df9094",
              "IPY_MODEL_b279949d0b014c7c94dfc73b23304feb"
            ],
            "layout": "IPY_MODEL_964b7584e52c4cebb0040e1ff1de4c49"
          }
        },
        "3750e853fef54f329e4b0a3babf53ac3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9da47fba93db42958a020a5131545d63",
            "placeholder": "​",
            "style": "IPY_MODEL_296e2be3ab2e43c987dd66254f060811",
            "value": "100%"
          }
        },
        "89fdb54347c640f592e26e97f1df9094": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_74ce2c34be43425f80afc31cca87e5db",
            "max": 4542,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8074d408a10b493fa0b771d0e3e5fd43",
            "value": 4542
          }
        },
        "b279949d0b014c7c94dfc73b23304feb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3c5a03c8a99543f1bdaacab2e6b4b99b",
            "placeholder": "​",
            "style": "IPY_MODEL_a48e32089ada4131a46d5a76405cde9e",
            "value": " 4542/4542 [00:00&lt;00:00, 241687.44it/s]"
          }
        },
        "964b7584e52c4cebb0040e1ff1de4c49": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9da47fba93db42958a020a5131545d63": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "296e2be3ab2e43c987dd66254f060811": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "74ce2c34be43425f80afc31cca87e5db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8074d408a10b493fa0b771d0e3e5fd43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3c5a03c8a99543f1bdaacab2e6b4b99b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a48e32089ada4131a46d5a76405cde9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}