SLP_1_0_1.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/byanez15/gerstman-gang/blob/main/SLP_1_0_1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-Mv_pLdvmWWt"
      },
      "outputs": [],
      "source": [
        "# TEST LINE TO SEE IF THINGS CHANGED\n",
        "\n",
        "\n",
        "import os\n",
        "os.environ[\"CUBLAS_WORKSPACE_CONFIG\"]=\":4096:2\"  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6IFo3SynyK61",
        "outputId": "13958c9a-5bb8-4e62-d94f-10a67cf2051e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "mv: cannot create directory 'gdrive/CSVs': Operation not supported\n"
          ]
        }
      ],
      "source": [
        "mv CSVs/ gdrive/CSVs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XVvlWxUkS8zd",
        "outputId": "637c9b7e-3461-4684-e14e-df2f7efdb645"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "blef31L63bvZ",
        "outputId": "15adc383-b320-49d6-9d60-1c458c1efaac"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: labtools3 in /usr/local/lib/python3.9/dist-packages (1.1.3.11)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.9/dist-packages (9.4.0)\n"
          ]
        }
      ],
      "source": [
        "#@title Install Packages\n",
        "!pip install labtools3 \n",
        "!pip install --upgrade Pillow"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "nAGxV9EU6VaY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "outputId": "d674cbb4-c68e-409d-8087-82619b6bf3df"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"\\n#MNIST DATA DOWNLOAD!!!!\\n\\ntransform = transforms.ToTensor()\\n\\n# Apply the transform to the train and test datasets\\ndataset = datasets.MNIST(root='.', train=True, transform=transform, download=True)\\n\\n\\n#Segregates training and test data\\ntrain_size = int(0.8 * len(dataset))\\ntest_size = len(dataset) - train_size\\ntrain_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size])\\n\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "#@title Import Packages and MNIST Data\n",
        "from PIL import Image, ImageDraw\n",
        "import random \n",
        "import time\n",
        "import pandas as pd \n",
        "import numpy as np\n",
        "import torch as t\n",
        "import torchvision\n",
        "import torchvision.datasets as datasets\n",
        "from torchvision import transforms\n",
        "import torch.functional\n",
        "import torch.nn as nn\n",
        "import LT.box as B\n",
        "import torch.nn.utils.prune as prune\n",
        "import copy\n",
        "from datetime import datetime\n",
        "\"\"\"\n",
        "#MNIST DATA DOWNLOAD!!!!\n",
        "\n",
        "transform = transforms.ToTensor()\n",
        "\n",
        "# Apply the transform to the train and test datasets\n",
        "dataset = datasets.MNIST(root='.', train=True, transform=transform, download=True)\n",
        "\n",
        "\n",
        "#Segregates training and test data\n",
        "train_size = int(0.8 * len(dataset))\n",
        "test_size = len(dataset) - train_size\n",
        "train_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size])\n",
        "\"\"\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kw3KMzh5WjVK"
      },
      "outputs": [],
      "source": [
        "#@title (DETERMINISTIC, SETRANDOM NUMBERS!)\n",
        "\n",
        "t.manual_seed(0)\n",
        "np.random.seed(0)\n",
        "torch.backends.cudnn.benchmark = False #PLEASE SEE PYTORCH DOCUMENTATION ON REPRODUCIBILITY https://pytorch.org/docs/stable/notes/randomness.html\n",
        "torch.use_deterministic_algorithms(mode = True, warn_only = True ) #THROWS ERROR FOR ANY NONDETERMINISM\n",
        "if torch.cuda.is_available():\n",
        "  torch.cuda.manual_seed_all(0)\n",
        "\n",
        "#MNIST DATA DOWNLOAD!!!!\n",
        "\n",
        "transform = transforms.ToTensor()\n",
        "\n",
        "# Apply the transform to the train and test datasets\n",
        "dataset = datasets.MNIST(root='.', train=True, transform=transform, download=True)\n",
        "\n",
        "\n",
        "#Segregates training and test data\n",
        "train_size = int(0.8 * len(dataset))\n",
        "test_size = len(dataset) - train_size\n",
        "train_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ish0iAed6eS4"
      },
      "outputs": [],
      "source": [
        "#@title Neural Network Class\n",
        "t.set_printoptions(threshold=100000000000)#  -- For printing full tensor information\n",
        "rng = lambda : torch.Generator().manual_seed(123456789)\n",
        "\n",
        "#CONSTANTS\n",
        "\n",
        "EPSILON_CE=1e-3     #ce criterion\n",
        "EPSILON_ACC = 1e-3    #accuracy is given in percentage so we use .1 to represent .001\n",
        "\n",
        "\n",
        "\"\"\"\n",
        "Nural_Network Class\n",
        "@author: Kevin Jenkins, Kevin Acosta, Brian Yanez\n",
        "\n",
        "Can define:\n",
        "learning_rate\n",
        "batch_size\n",
        "\"\"\"\n",
        "\n",
        "class Neural_Network(nn.Module):\n",
        "  def __init__(self, batch_size, learning_rate):\n",
        "    super(Neural_Network, self).__init__()\n",
        "    self.index = 0\n",
        "\n",
        "    #Create the data loaders and empty data sets\n",
        "    self.data = {'IPA' : [], 'CE_m' : [], 'Acc_m': [], 'Epoch' : [], 'Running_Acc' : []}\n",
        "    \n",
        "    self.batch_size = batch_size\n",
        "    self.train_loader = t.utils.data.DataLoader(train_dataset, batch_size = self.batch_size, shuffle = True)\n",
        "    self.test_loader = t.utils.data.DataLoader(test_dataset, batch_size=self.batch_size, shuffle=False)\n",
        "\n",
        "    input_size=28*28\n",
        "    output_size = 10\n",
        "    self.linlayer = nn.Linear(input_size, output_size,bias = False)\n",
        "    self.model = nn.Sequential(self.linlayer)\n",
        "\n",
        "    #Manually fix weight matrices to ones\n",
        "    with torch.no_grad():\n",
        "      for name, param in self.model.named_parameters():\n",
        "          if param.requires_grad:\n",
        "              param.copy_(t.ones_like(param))\n",
        "\n",
        "\n",
        "    self.criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "    self.my_lr = learning_rate\n",
        "    self.optimizer = t.optim.Adadelta(self.model.parameters(), lr = self.my_lr)\n",
        "\n",
        "  def convert_data(self):\n",
        "    self.data = pd.DataFrame(data = self.data)\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "K8VoInMP80u0"
      },
      "outputs": [],
      "source": [
        "#@title Miscellaneous Functions\n",
        "def average_acc(l: list, window_size = 5, stopping_val = EPSILON_ACC):\n",
        "  moving_aves = []\n",
        "  if (len(l) < window_size):\n",
        "    pass\n",
        "\n",
        "  stop = False\n",
        "  vals = [0]\n",
        "  for i, x in enumerate(l, 1):\n",
        "    vals.append(vals[i-1] + x)\n",
        "    if i >= window_size:\n",
        "      moving_ave = (vals[i] - vals[i - window_size])/window_size\n",
        "      moving_aves.append(moving_ave)\n",
        "      if moving_ave <= stopping_val:\n",
        "        stop = True\n",
        "\n",
        "  return stop, moving_aves\n",
        "  \n",
        "\n",
        "def evaluate_accuracy(m, test_loader, device):\n",
        "    m.model.eval()\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    with torch.no_grad():\n",
        "        for inputs, targets in test_loader:\n",
        "            inputs, targets = inputs.to(device), targets.to(device)\n",
        "            inputs = inputs.view(-1, 28*28) #Flatten data\n",
        "            outputs = m.model(inputs)\n",
        "            _, preds = torch.max(outputs, 1)\n",
        "            total += targets.size(0)\n",
        "            correct += (preds == targets).sum().item()\n",
        "    return (correct / total) * 100\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Zyek6kF9CqP2"
      },
      "outputs": [],
      "source": [
        "#@title Training Method 2 (a bit more concise)\n",
        "from torchvision.datasets.mnist import string\n",
        "\n",
        "\n",
        "def train( a : Neural_Network, suppressLog = True, CUDA = False, percentage = 0, criterion = 'CE', stop_epoch = 1000):\n",
        "  start = time.time()\n",
        "  os.environ[\"CUBLAS_WORKSPACE_CONFIG\"]=\":4096:2\"  \n",
        "  elements = int((percentage/100) * (28**2)) #number of gradient matrix entries to be set to zero \n",
        "  Acc_m, CE_m, IPA, Epoch = [[] for i in range(4)]\n",
        "  m_L, epoques, losses, dCE, accuracies, ce, delta_acc, ipa = [[] for i in range(8)]\n",
        "  \n",
        "  flag = True\n",
        "    #TRAINING LOOP\n",
        "  epoch, deltaCE = 0,0\n",
        "  while epoch <= stop_epoch and flag:\n",
        "\n",
        "    #Empty gradients before next step\n",
        "    a.optimizer.zero_grad()\n",
        "\n",
        "    loss=0\n",
        "    m = 1\n",
        "\n",
        "    for datap, target in a.train_loader:\n",
        "      if (CUDA):\n",
        "        model = a.model.to(0)  # move the model to GPU\n",
        "        datap = datap.to(0)  # move the data to GPU\n",
        "        target = target.to(0)  # move the targets to GPU\n",
        "\n",
        "      datap = datap.view(-1, 28*28) #Flatten data\n",
        "\n",
        "\n",
        "      # Clear the gradients\n",
        "      a.optimizer.zero_grad()\n",
        "      # Forward pass\n",
        "      output = model(datap)\n",
        "      # Compute the loss\n",
        "      loss = a.criterion(output, target)\n",
        "\n",
        "      \n",
        "\n",
        "      DEVICE = 0 ####DO THIS BETTER LATER\n",
        "      if (m==1 and epoch == 0):\n",
        "        print(f'Initial Loss: {loss}')\n",
        "        #ce.append(loss.cpu().detach().numpy())\n",
        "        with torch.no_grad():\n",
        "          acc = evaluate_accuracy(a, a.test_loader, DEVICE)\n",
        "          accuracies.append(acc)\n",
        "          Acc_m.append(acc)\n",
        "          CE_m.append(loss.cpu().detach().numpy().tolist())\n",
        "          ce.append(loss.cpu().detach().numpy())\n",
        "          IPA.append(0.0)\n",
        "        print(f'Accuracy {acc}')\n",
        "\n",
        "\n",
        "      # Backward pass\n",
        "      loss.backward()\n",
        "\n",
        "      #Before we update the weights, 0 the gradients of neurons we want fixed at 0 FROM LEFT TO RIGHT\n",
        "      a.linlayer.weight.grad.data[:, 0:elements].fill_(0)\n",
        "\n",
        "      # Update the weights\n",
        "      a.optimizer.step()\n",
        "\n",
        "      #For graphing\n",
        "\n",
        "      m = m+1\n",
        "      \n",
        "    if (epoch != 0):\n",
        "      # ACCURACY \n",
        "      print(f'Loss: {loss} Epoch: {epoch}')\n",
        "      with torch.no_grad():\n",
        "        acc = evaluate_accuracy(a, a.test_loader, DEVICE)\n",
        "        accuracies.append(acc)\n",
        "        Acc_m.append(acc)  #appending data to our model\n",
        "      print(f'Accuracy {acc}')\n",
        "\n",
        "      epoques.append(epoch)\n",
        "\n",
        "      #CE_m\n",
        "      with torch.no_grad():\n",
        "\n",
        "        ce.append(loss.cpu().detach().numpy())\n",
        "        CE_m.append(loss.cpu().detach().numpy().tolist())  #appending data to our model\n",
        "\n",
        "      #IPA\n",
        "      with torch.no_grad():\n",
        "        curr_ipa = (ce[epoch] - ce[epoch - 1])/epoch\n",
        "        IPA.append(abs(curr_ipa)) #appending data to our model\n",
        "\n",
        "    if (epoch > 1):\n",
        "      #Stopping condition (delta CE part)\n",
        "      key_value = (accuracies[epoch] - accuracies[epoch - 1])/accuracies[epoch - 1]\n",
        "      delta_acc.append(abs(key_value))\n",
        "    \n",
        "    Epoch.append(epoch)\n",
        "    epoch += 1\n",
        "    \n",
        "\n",
        "    if criterion == 'Accuracy':\n",
        "      running_ave_list = [0 for i in range(6)] + average_acc(delta_acc)[1] #THE 5 IN THIS CASE IS WINDOW SIZE PLUS ONE. PLEASE MIND CHANGING THIS IF U CHANGE WINDOW SIZE\n",
        "      if (average_acc(delta_acc)[0]):\n",
        "        a.data['Acc_m'] = Acc_m \n",
        "        a.data['Epoch'] = Epoch\n",
        "        a.data['IPA'] = IPA\n",
        "        a.data['CE_m'] = CE_m\n",
        "        #print(delta_acc) These commented out lines are used for debugging arraysize errors.\n",
        "        flag = False\n",
        "        end = time.time()\n",
        "        runtime = end - start\n",
        "        a.data['Runtime'] = [runtime for i in range(len(a.data['Epoch']))]\n",
        "        print(f\"\\nThe running average of the accuracies has dropped below epsilon.\\n We've run for {Epoch[-1]} epochs. \\n Training Complete!\\n\")\n",
        "        a.data['Running_Acc'] = running_ave_list\n",
        "        #print(len(running_ave_list)) These commented out lines are used for debugging arraysize errors.\n",
        "        #print(len(Acc_m)) These commented out lines are used for debugging arraysize errors.\n",
        "\n",
        "    if criterion == 'CE':\n",
        "      if len(ce) >= 2 and flag == False:\n",
        "        a.data['Acc_m'] = Acc_m \n",
        "        a.data['Epoch'] = Epoch\n",
        "        a.data['IPA'] = IPA\n",
        "        a.data['CE_m'] = CE_m\n",
        "        Delta_CE = ce[-1] - ce[-2]\n",
        "        print(EPSILON_CE)\n",
        "        print(Delta_CE)\n",
        "        end = time.time()\n",
        "        runtime = end - start\n",
        "        a.data['Runtime'] = [runtime for i in range(len(a.data['Epoch']))]\n",
        "        if (Delta_CE <= EPSILON_CE):\n",
        "          print(Delta_CE)\n",
        "          flag = False\n",
        "          print('\\nThe running average of the cross entropies has dropped below epsilon.\\nTraining Complete!\\n')\n",
        "          a.data['Running_Acc'] = average_acc(delta_acc)[1]\n",
        "\n",
        "  #end CE criterion"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "U01wGGD0NeVc",
        "outputId": "be053f68-482d-4c0e-a5f1-f19772f06725"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Initial Loss: 2.3025989532470703\n",
            "Accuracy 10.216666666666667\n",
            "Loss: 1.8907768726348877 Epoch: 1\n",
            "Accuracy 74.34166666666667\n",
            "Loss: 1.6709028482437134 Epoch: 2\n",
            "Accuracy 77.70833333333333\n",
            "Loss: 1.49075186252594 Epoch: 3\n",
            "Accuracy 78.675\n",
            "Loss: 1.3485442399978638 Epoch: 4\n",
            "Accuracy 79.78333333333333\n",
            "Loss: 1.2275128364562988 Epoch: 5\n",
            "Accuracy 80.55833333333334\n",
            "Loss: 1.1372604370117188 Epoch: 6\n",
            "Accuracy 81.22500000000001\n",
            "Loss: 1.0600100755691528 Epoch: 7\n",
            "Accuracy 81.875\n",
            "Loss: 0.994514524936676 Epoch: 8\n",
            "Accuracy 82.35833333333333\n",
            "Loss: 0.9395619630813599 Epoch: 9\n",
            "Accuracy 82.75833333333334\n",
            "Loss: 0.8924663662910461 Epoch: 10\n",
            "Accuracy 83.42500000000001\n",
            "Loss: 0.8544892072677612 Epoch: 11\n",
            "Accuracy 83.85000000000001\n",
            "Loss: 0.8102606534957886 Epoch: 12\n",
            "Accuracy 84.05833333333334\n",
            "Loss: 0.7836639881134033 Epoch: 13\n",
            "Accuracy 84.25\n",
            "Loss: 0.7557195425033569 Epoch: 14\n",
            "Accuracy 84.575\n",
            "Loss: 0.7209814786911011 Epoch: 15\n",
            "Accuracy 84.98333333333333\n",
            "Loss: 0.7091116309165955 Epoch: 16\n",
            "Accuracy 85.28333333333333\n",
            "Loss: 0.6869233250617981 Epoch: 17\n",
            "Accuracy 85.48333333333333\n",
            "Loss: 0.6754772663116455 Epoch: 18\n",
            "Accuracy 85.775\n",
            "Loss: 0.653043270111084 Epoch: 19\n",
            "Accuracy 85.96666666666667\n",
            "Loss: 0.6359084844589233 Epoch: 20\n",
            "Accuracy 85.95\n",
            "Loss: 0.6188220381736755 Epoch: 21\n",
            "Accuracy 86.09166666666667\n",
            "Loss: 0.6051520705223083 Epoch: 22\n",
            "Accuracy 86.25\n",
            "Loss: 0.5905575752258301 Epoch: 23\n",
            "Accuracy 86.51666666666667\n",
            "Loss: 0.5707417130470276 Epoch: 24\n",
            "Accuracy 86.64166666666667\n",
            "Loss: 0.5747196078300476 Epoch: 25\n",
            "Accuracy 86.81666666666666\n",
            "Loss: 0.5576679110527039 Epoch: 26\n",
            "Accuracy 87.00833333333333\n",
            "Loss: 0.5462320446968079 Epoch: 27\n",
            "Accuracy 87.06666666666666\n",
            "Loss: 0.5416290163993835 Epoch: 28\n",
            "Accuracy 87.30833333333334\n",
            "Loss: 0.5376942157745361 Epoch: 29\n",
            "Accuracy 87.35833333333333\n",
            "Loss: 0.5206193327903748 Epoch: 30\n",
            "Accuracy 87.47500000000001\n",
            "Loss: 0.5186725854873657 Epoch: 31\n",
            "Accuracy 87.58333333333333\n",
            "Loss: 0.5072540044784546 Epoch: 32\n",
            "Accuracy 87.6\n",
            "Loss: 0.5011590123176575 Epoch: 33\n",
            "Accuracy 87.70833333333333\n",
            "\n",
            "The running average of the accuracies has dropped below epsilon.\n",
            " We've run for 33 epochs. \n",
            " Training Complete!\n",
            "\n",
            "            IPA      CE_m      Acc_m  Running_Acc     Runtime\n",
            "Epoch                                                        \n",
            "0      0.000000  2.302599  10.216667     0.000000  207.767025\n",
            "1      0.411822  1.890777  74.341667     0.000000  207.767025\n",
            "2      0.109937  1.670903  77.708333     0.000000  207.767025\n",
            "3      0.060050  1.490752  78.675000     0.000000  207.767025\n",
            "4      0.035552  1.348544  79.783333     0.000000  207.767025\n",
            "5      0.024206  1.227513  80.558333     0.000000  207.767025\n",
            "6      0.015042  1.137260  81.225000     0.017961  207.767025\n",
            "7      0.011036  1.060010  81.875000     0.010504  207.767025\n",
            "8      0.008187  0.994515  82.358333     0.009197  207.767025\n",
            "9      0.006106  0.939562  82.758333     0.007350  207.767025\n",
            "10     0.004710  0.892466  83.425000     0.007019  207.767025\n",
            "11     0.003452  0.854489  83.850000     0.006383  207.767025\n",
            "12     0.003686  0.810261  84.058333     0.005279  207.767025\n",
            "13     0.002046  0.783664  84.250000     0.004554  207.767025\n",
            "14     0.001996  0.755720  84.575000     0.004354  207.767025\n",
            "15     0.002316  0.720981  84.983333     0.003709  207.767025\n",
            "16     0.000742  0.709112  85.283333     0.003396  207.767025\n",
            "17     0.001305  0.686923  85.483333     0.003368  207.767025\n",
            "18     0.000636  0.675477  85.775000     0.003595  207.767025\n",
            "19     0.001181  0.653043  85.966667     0.003270  207.767025\n",
            "20     0.000857  0.635908  85.950000     0.002343  207.767025\n",
            "21     0.000814  0.618822  86.091667     0.001967  207.767025\n",
            "22     0.000621  0.605152  86.250000     0.001866  207.767025\n",
            "23     0.000635  0.590558  86.516667     0.001802  207.767025\n",
            "24     0.000826  0.570742  86.641667     0.001644  207.767025\n",
            "25     0.000159  0.574720  86.816667     0.002009  207.767025\n",
            "26     0.000656  0.557668  87.008333     0.002121  207.767025\n",
            "27     0.000424  0.546232  87.066667     0.001887  207.767025\n",
            "28     0.000164  0.541629  87.308333     0.001824  207.767025\n",
            "29     0.000136  0.537694  87.358333     0.001649  207.767025\n",
            "30     0.000569  0.520619  87.475000     0.001512  207.767025\n",
            "31     0.000063  0.518673  87.583333     0.001319  207.767025\n",
            "32     0.000357  0.507254  87.600000     0.001223  207.767025\n",
            "33     0.000185  0.501159  87.708333     0.000915  207.767025\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/nn/modules/linear.py:114: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  return F.linear(input, self.weight, self.bias)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Initial Loss: 2.3025989532470703\n",
            "Accuracy 10.216666666666667\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/autograd/__init__.py:197: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility (Triggered internally at ../aten/src/ATen/Context.cpp:159.)\n",
            "  Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss: 1.8920530080795288 Epoch: 1\n",
            "Accuracy 75.4\n",
            "Loss: 1.6720962524414062 Epoch: 2\n",
            "Accuracy 78.25833333333333\n",
            "Loss: 1.4919512271881104 Epoch: 3\n",
            "Accuracy 78.85\n",
            "Loss: 1.3471673727035522 Epoch: 4\n",
            "Accuracy 80.10000000000001\n",
            "Loss: 1.2260973453521729 Epoch: 5\n",
            "Accuracy 80.48333333333333\n",
            "Loss: 1.1374082565307617 Epoch: 6\n",
            "Accuracy 81.10000000000001\n",
            "Loss: 1.0590732097625732 Epoch: 7\n",
            "Accuracy 81.78333333333333\n",
            "Loss: 0.9921801090240479 Epoch: 8\n",
            "Accuracy 82.33333333333334\n",
            "Loss: 0.9417298436164856 Epoch: 9\n",
            "Accuracy 82.78333333333333\n",
            "Loss: 0.8879229426383972 Epoch: 10\n",
            "Accuracy 83.35833333333333\n",
            "Loss: 0.8500279188156128 Epoch: 11\n",
            "Accuracy 83.65833333333333\n",
            "Loss: 0.8159043192863464 Epoch: 12\n",
            "Accuracy 83.96666666666667\n",
            "Loss: 0.7838156223297119 Epoch: 13\n",
            "Accuracy 84.46666666666667\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-68-3b0fce78b6ef>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m   \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menviron\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"CUBLAS_WORKSPACE_CONFIG\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\":4096:2\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m   \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNeural_Network\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m30000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearning_rate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m   \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mCUDA\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'Accuracy'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstop_epoch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m   \u001b[0mdatadf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m   \u001b[0mdatadf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Epoch'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-67-e4d1ca42b740>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(a, suppressLog, CUDA, percentage, criterion, stop_epoch)\u001b[0m\n\u001b[1;32m     21\u001b[0m     \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mdatap\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m \u001b[0;32min\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_loader\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mCUDA\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# move the model to GPU\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    626\u001b[0m                 \u001b[0;31m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    627\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[call-arg]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 628\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    629\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    630\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    669\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    670\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 671\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    672\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    673\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory_device\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     56\u001b[0m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getitems__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     56\u001b[0m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getitems__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/utils/data/dataset.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, idx)\u001b[0m\n\u001b[1;32m    293\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindices\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 295\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindices\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    296\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    297\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torchvision/datasets/mnist.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, index)\u001b[0m\n\u001b[1;32m    143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m             \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    146\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtarget_transform\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torchvision/transforms/transforms.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, pic)\u001b[0m\n\u001b[1;32m    133\u001b[0m             \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mConverted\u001b[0m \u001b[0mimage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m         \"\"\"\n\u001b[0;32m--> 135\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    136\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__repr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torchvision/transforms/functional.py\u001b[0m in \u001b[0;36mto_tensor\u001b[0;34m(pic)\u001b[0m\n\u001b[1;32m    169\u001b[0m     \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpermute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontiguous\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    170\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mByteTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 171\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdefault_float_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdiv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m255\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    172\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    173\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "#TESTING\n",
        "for i in range(2):\n",
        "  os.environ[\"CUBLAS_WORKSPACE_CONFIG\"]=\":4096:2\"  \n",
        "  m = Neural_Network(batch_size = 30000, learning_rate = 1).to(0)\n",
        "  train(m, CUDA = True, criterion = 'Accuracy', stop_epoch = 100)\n",
        "  datadf = pd.DataFrame(data = m.data)\n",
        "  datadf.set_index('Epoch', inplace = True)\n",
        "  print(datadf)\n",
        "  datadf.to_csv(f'batch_{m.batch_size}.lr_{m.my_lr}.{i}.csv')\n",
        "  os.environ[\"CUBLAS_WORKSPACE_CONFIG\"]=\":4096:2\" \n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}