load_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    },
    "colab": {
      "name": "load_data.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "wtbjSpzCOsU9"
      },
      "source": [
        "import numpy as np\n",
        "import torch\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import time\n",
        "cuda = torch.cuda.is_available()\n",
        "\n",
        "BATCH_SIZE = 64"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zu1s28xJ7OOn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aa769932-ea26-44dc-c9e1-e357866b0967"
      },
      "source": [
        "# from google.colab import drive\n",
        "# drive.mount(\"/content/drive\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BYHQRcey8zqI"
      },
      "source": [
        "#TODO - replace all the following parameters\n",
        "train_X_path = \"/content/drive/My Drive/train_X.npy\"\n",
        "train_Y_path = '/content/drive/My Drive/train_Y.npy'\n",
        "test_X_path = '/content/drive/My Drive/test_X.npy'\n",
        "train_results_path = \"/content/drive/My Drive/train_results\"\n",
        "\n",
        "#loss_functions: XEntropy, MSE\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "#optimizer: Adam, SGD, SGD+momentum ; lr, weight_decay\n",
        "optimizer = optim.Adam(model.parameters(), lr = 3e-4, weight_decay=5e-6)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "co4fCKS8OsVC"
      },
      "source": [
        "# len(train_X) = 154684\n",
        "train_X = np.load(train_X_path, allow_pickle = True)[:110000]\n",
        "train_Y = np.load(train_Y_path, allow_pickle = True)[:110000]\n",
        "val_X = np.load(train_X_path, allow_pickle = True)[110000:]\n",
        "val_Y = np.load(train_Y_path, allow_pickle = True)[110000:]\n",
        "test_X = np.load(test_X_path, allow_pickle = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u8Cpz_bdOsVH"
      },
      "source": [
        "# custom Dataset class\n",
        "class MovieReviewsData(Dataset):\n",
        "    def __init__(self, X, Y = None):\n",
        "        self.maxlen = max(len(x) for x in X)\n",
        "        self.X = [F.pad(torch.FloatTensor(x), (0, 0, 0, self.maxlen - len(x))) for x in X]\n",
        "        if Y is not None:\n",
        "            self.Y = torch.LongTensor(Y)\n",
        "        else:\n",
        "            self.Y = None\n",
        "    \n",
        "    def __len__(self):\n",
        "        return len(self.X)\n",
        "    \n",
        "    def __getitem__(self, idx):\n",
        "        if self.Y is not None:\n",
        "            return self.X[idx], self.Y[idx]\n",
        "        else:\n",
        "            return self.X[idx]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R6ZcdaotOsVK"
      },
      "source": [
        "train_dataset = MovieReviewsData(train_X, train_Y)\n",
        "val_dataset = MovieReviewsData(val_X, val_Y)\n",
        "test_dataset = MovieReviewsData(test_X)\n",
        "train_loader = DataLoader(train_dataset, shuffle = True, batch_size = BATCH_SIZE)\n",
        "val_loader = DataLoader(val_dataset, shuffle = False, batch_size = BATCH_SIZE)\n",
        "test_loader = DataLoader(test_dataset, shuffle = False, batch_size = BATCH_SIZE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P5iolhO4OsVN"
      },
      "source": [
        "class MyModel(nn.Module):\n",
        "    def __init__(self, input_channel, out_channel, kernel_sizes, output_dim):\n",
        "        super().__init__()\n",
        "        self.conv1 = nn.Conv1d(input_channel, out_channel, kernel_size = kernel_sizes[0])\n",
        "        self.conv2 = nn.Conv1d(input_channel, out_channel, kernel_size = kernel_sizes[1])\n",
        "        self.conv3 = nn.Conv1d(input_channel, out_channel, kernel_size = kernel_sizes[2])\n",
        "        \n",
        "        self.linear = nn.Linear(3 * out_channel, output_dim)\n",
        "        self.dropout = nn.Dropout(0.5)\n",
        "        \n",
        "    def forward(self, embedded):     \n",
        "        embedded = embedded.permute(0, 2, 1)\n",
        "        conved = F.relu(self.conv1(embedded))\n",
        "        conved =  F.relu(self.conv2(conved))   \n",
        "        conved =  F.relu(self.conv3(conved))   \n",
        "\n",
        "        pooled = []\n",
        "        for conv in conved:\n",
        "          pooled.append(F.max_pool1d(conv, conv.shape[2]).squeeze(2))\n",
        "        \n",
        "        pooled = torch.cat(pooled, dim = 1)\n",
        "        drop = self.dropout(pooled)\n",
        "        return self.linear(drop)\n",
        "input_channel = 100\n",
        "out_channel = 50\n",
        "kernel_sizes = [2,3,4]\n",
        "output_dim = 5\n",
        "model = MyModel(input_channel, out_channel, kernel_sizes, output_dim)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tL6bbVZyZDw0"
      },
      "source": [
        "device = torch.device(\"cuda\" if cuda else \"cpu\")\n",
        "model.to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VNxhDR-8aTWf"
      },
      "source": [
        "def train_epoch(model, train_loader, criterion, optimizer):\n",
        "    model.train()\n",
        "\n",
        "    running_loss = 0.0\n",
        "    \n",
        "    start_time = time.time()\n",
        "    for batch_idx, (X, Y) in enumerate(train_loader):   \n",
        "        optimizer.zero_grad()   \n",
        "        X = X.to(device)\n",
        "        Y = Y.to(device) \n",
        "\n",
        "        outputs = model(X)\n",
        "        loss = criterion(outputs, Y)\n",
        "        running_loss += loss.item()\n",
        "\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "    \n",
        "    end_time = time.time()\n",
        "    \n",
        "    running_loss /= len(train_loader)\n",
        "    print('Training Loss: ', running_loss, 'Time: ',end_time - start_time, 's')\n",
        "    return running_loss\n",
        "\n",
        "def test_model(model, test_loader, criterion):\n",
        "    with torch.no_grad():\n",
        "        model.eval()\n",
        "\n",
        "        running_loss = 0.0\n",
        "        total_predictions = 0.0\n",
        "        correct_predictions = 0.0\n",
        "\n",
        "        for batch_idx, (X, Y) in enumerate(test_loader):   \n",
        "            X = X.to(device)\n",
        "            Y = Y.to(device) \n",
        "\n",
        "            outputs = model(X)\n",
        "            _, predicted = torch.max(outputs.data, 1)\n",
        "            total_predictions += Y.size(0)\n",
        "            correct_predictions += (predicted == Y).sum().item()\n",
        "\n",
        "            loss = criterion(outputs, Y)\n",
        "            running_loss += loss.item()\n",
        "\n",
        "\n",
        "        running_loss /= len(test_loader)\n",
        "        acc = (correct_predictions/total_predictions)*100.0\n",
        "        print('Testing Loss: ', running_loss)\n",
        "        print('Testing Accuracy: ', acc, '%')\n",
        "        return running_loss, acc\n",
        "\n",
        "#TODO: predict requires tokenize input sentence and embeddings\n",
        "\n",
        "# def predict(model, test_loader, criterion):\n",
        "#     with torch.no_grad():\n",
        "#         model.eval()\n",
        "\n",
        "#         for batch_idx, (X, Y) in enumerate(test_loader):   \n",
        "#             X = X.to(device)\n",
        "\n",
        "#             outputs = model(X)\n",
        "#             _, predicted = torch.max(outputs.data, 1)\n",
        "\n",
        "#             loss = criterion(outputs, Y)\n",
        "#             running_loss += loss.item()\n",
        "\n",
        "\n",
        "#         running_loss /= len(test_loader)\n",
        "#         acc = (correct_predictions/total_predictions)*100.0\n",
        "#         print('Testing Loss: ', running_loss)\n",
        "#         print('Testing Accuracy: ', acc, '%')\n",
        "#         return running_loss, acc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SUoeuWi_b8a8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "77ad8b03-fe7b-4da3-c8b2-ac6d6d7e6bdb"
      },
      "source": [
        "n_epochs = 40\n",
        "Train_loss = []\n",
        "Train_acc = []\n",
        "Test_loss = []\n",
        "Test_acc = []\n",
        "\n",
        "for i in range(n_epochs):\n",
        "    print(\"epoch\" + str(i+1))\n",
        "    train_epoch(model, train_loader, criterion, optimizer)\n",
        "    train_loss, train_acc = test_model(model, train_loader, criterion)\n",
        "    test_loss, test_acc = test_model(model, val_loader, criterion)\n",
        "    Train_loss.append(train_loss)\n",
        "    Train_acc.append(train_acc)\n",
        "    Test_loss.append(test_loss)\n",
        "    Test_acc.append(test_acc)\n",
        "    print('='*20)\n",
        "\n",
        "#save intermediate results\n",
        "torch.save({\n",
        "        'train_loss': Train_loss,\n",
        "        'train_acc': Train_acc,\n",
        "        'test_loss' : Test_loss,\n",
        "        'test_acc' : Test_acc,\n",
        "      }, train_results_path)\n",
        "\n",
        "#read intermediate results\n",
        "train_results = torch.load(train_results_path)\n",
        "train_acc =  train_results[\"train_acc\"]\n",
        "print(train_acc)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1.6413330933325092, 1.6413362220389125, 1.641331154846604, 1.6413342099747317, 1.6413343142048435]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lkVlU56HcBjr"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}