Question-answering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecc44557-75e9-42e4-a647-4b23e2f01546",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip3 install evaluate\n",
    "# !pip3 install rouge\n",
    "# !pip3 install --upgrade simplet5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "867f6875-c1f2-421c-a7fb-7a3ab2ad9d13",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 42\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "import scipy.spatial\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "\n",
    "import evaluate  # Bleu\n",
    "import transformers\n",
    "from transformers import T5Tokenizer, T5Model, TFT5ForConditionalGeneration, T5TokenizerFast\n",
    "from simplet5 import SimpleT5\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "52f05311-4968-418b-8188-f1cbe60b43d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...</td>\n",
       "      <td>When did Beyonce start becoming popular?</td>\n",
       "      <td>in the late 1990s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...</td>\n",
       "      <td>What areas did Beyonce compete in when she was...</td>\n",
       "      <td>singing and dancing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...</td>\n",
       "      <td>When did Beyonce leave Destiny's Child and bec...</td>\n",
       "      <td>2003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...</td>\n",
       "      <td>In what city and state did Beyonce  grow up?</td>\n",
       "      <td>Houston, Texas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...</td>\n",
       "      <td>In which decade did Beyonce become famous?</td>\n",
       "      <td>late 1990s</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             context  \\\n",
       "0  Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...   \n",
       "1  Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...   \n",
       "2  Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...   \n",
       "3  Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...   \n",
       "4  Beyoncé Giselle Knowles-Carter (/biːˈjɒnseɪ/ b...   \n",
       "\n",
       "                                            question               answer  \n",
       "0           When did Beyonce start becoming popular?    in the late 1990s  \n",
       "1  What areas did Beyonce compete in when she was...  singing and dancing  \n",
       "2  When did Beyonce leave Destiny's Child and bec...                 2003  \n",
       "3      In what city and state did Beyonce  grow up?        Houston, Texas  \n",
       "4         In which decade did Beyonce become famous?           late 1990s  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Loading the data\n",
    "\n",
    "with open('data/train-v2.0.json') as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "\n",
    "# Extracting context, question, and answers from the dataset\n",
    "\n",
    "def prepare_data(data):\n",
    "    articles = []\n",
    "    \n",
    "    for article in data[\"data\"]:\n",
    "        for paragraph in article[\"paragraphs\"]:\n",
    "            for qa in paragraph[\"qas\"]:\n",
    "                question = qa[\"question\"]\n",
    "\n",
    "                if not qa[\"is_impossible\"]:\n",
    "                  answer = qa[\"answers\"][0][\"text\"]\n",
    "                \n",
    "                inputs = {\"context\": paragraph[\"context\"], \"question\": question, \"answer\": answer}\n",
    "\n",
    "            \n",
    "                articles.append(inputs)\n",
    "\n",
    "    return articles\n",
    "\n",
    "data = prepare_data(data)\n",
    "\n",
    "# Create a Dataframe\n",
    "data = pd.DataFrame(data)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "65a4de1f-e746-4ec8-870b-3e426c5b127e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>130309</th>\n",
       "      <td>These quarks and leptons interact through four...</td>\n",
       "      <td>How many quarks and leptons are there?</td>\n",
       "      <td>Kathmandu Metropolitan City</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130310</th>\n",
       "      <td>These quarks and leptons interact through four...</td>\n",
       "      <td>What model satisfactorily explains gravity?</td>\n",
       "      <td>Kathmandu Metropolitan City</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130311</th>\n",
       "      <td>These quarks and leptons interact through four...</td>\n",
       "      <td>Interactions between quarks and leptons are th...</td>\n",
       "      <td>Kathmandu Metropolitan City</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130312</th>\n",
       "      <td>These quarks and leptons interact through four...</td>\n",
       "      <td>Mass and energy can always be compared to what?</td>\n",
       "      <td>Kathmandu Metropolitan City</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130313</th>\n",
       "      <td>These quarks and leptons interact through four...</td>\n",
       "      <td>What relation explains the carriers of the ele...</td>\n",
       "      <td>Kathmandu Metropolitan City</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130314</th>\n",
       "      <td>The term \"matter\" is used throughout physics i...</td>\n",
       "      <td>Physics has broadly agreed on the definition o...</td>\n",
       "      <td>Kathmandu Metropolitan City</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130315</th>\n",
       "      <td>The term \"matter\" is used throughout physics i...</td>\n",
       "      <td>Who coined the term partonic matter?</td>\n",
       "      <td>Kathmandu Metropolitan City</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130316</th>\n",
       "      <td>The term \"matter\" is used throughout physics i...</td>\n",
       "      <td>What is another name for anti-matter?</td>\n",
       "      <td>Kathmandu Metropolitan City</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130317</th>\n",
       "      <td>The term \"matter\" is used throughout physics i...</td>\n",
       "      <td>Matter usually does not need to be used in con...</td>\n",
       "      <td>Kathmandu Metropolitan City</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130318</th>\n",
       "      <td>The term \"matter\" is used throughout physics i...</td>\n",
       "      <td>What field of study has a variety of unusual c...</td>\n",
       "      <td>Kathmandu Metropolitan City</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  context  \\\n",
       "130309  These quarks and leptons interact through four...   \n",
       "130310  These quarks and leptons interact through four...   \n",
       "130311  These quarks and leptons interact through four...   \n",
       "130312  These quarks and leptons interact through four...   \n",
       "130313  These quarks and leptons interact through four...   \n",
       "130314  The term \"matter\" is used throughout physics i...   \n",
       "130315  The term \"matter\" is used throughout physics i...   \n",
       "130316  The term \"matter\" is used throughout physics i...   \n",
       "130317  The term \"matter\" is used throughout physics i...   \n",
       "130318  The term \"matter\" is used throughout physics i...   \n",
       "\n",
       "                                                 question  \\\n",
       "130309             How many quarks and leptons are there?   \n",
       "130310        What model satisfactorily explains gravity?   \n",
       "130311  Interactions between quarks and leptons are th...   \n",
       "130312    Mass and energy can always be compared to what?   \n",
       "130313  What relation explains the carriers of the ele...   \n",
       "130314  Physics has broadly agreed on the definition o...   \n",
       "130315               Who coined the term partonic matter?   \n",
       "130316              What is another name for anti-matter?   \n",
       "130317  Matter usually does not need to be used in con...   \n",
       "130318  What field of study has a variety of unusual c...   \n",
       "\n",
       "                             answer  \n",
       "130309  Kathmandu Metropolitan City  \n",
       "130310  Kathmandu Metropolitan City  \n",
       "130311  Kathmandu Metropolitan City  \n",
       "130312  Kathmandu Metropolitan City  \n",
       "130313  Kathmandu Metropolitan City  \n",
       "130314  Kathmandu Metropolitan City  \n",
       "130315  Kathmandu Metropolitan City  \n",
       "130316  Kathmandu Metropolitan City  \n",
       "130317  Kathmandu Metropolitan City  \n",
       "130318  Kathmandu Metropolitan City  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "51bfb46d-aa80-4782-bf02-034a67bd0150",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_text</th>\n",
       "      <th>target_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>129949</th>\n",
       "      <td>question: What stupa is close to the National ...</td>\n",
       "      <td>Swayambhunath &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129950</th>\n",
       "      <td>question: Who built the building that contains...</td>\n",
       "      <td>Bhimsen Thapa &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129951</th>\n",
       "      <td>question: When was the National Museum founded?</td>\n",
       "      <td>1928 &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129952</th>\n",
       "      <td>question: What does Chhauni Silkhana mean?</td>\n",
       "      <td>the stone house of arms and ammunition &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129953</th>\n",
       "      <td>question: When did Tribhuvan die?</td>\n",
       "      <td>1955 &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130044</th>\n",
       "      <td>question: From what city does Arkefly offer no...</td>\n",
       "      <td>Amsterdam &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130045</th>\n",
       "      <td>question: Who operates flights between Kathman...</td>\n",
       "      <td>Turkish Airlines &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130046</th>\n",
       "      <td>question: In what US state did Kathmandu first...</td>\n",
       "      <td>Oregon &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130047</th>\n",
       "      <td>question: What was Yangon previously known as?</td>\n",
       "      <td>Rangoon &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130048</th>\n",
       "      <td>question: With what Belorussian city does Kath...</td>\n",
       "      <td>Minsk &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              source_text  \\\n",
       "129949  question: What stupa is close to the National ...   \n",
       "129950  question: Who built the building that contains...   \n",
       "129951    question: When was the National Museum founded?   \n",
       "129952         question: What does Chhauni Silkhana mean?   \n",
       "129953                  question: When did Tribhuvan die?   \n",
       "...                                                   ...   \n",
       "130044  question: From what city does Arkefly offer no...   \n",
       "130045  question: Who operates flights between Kathman...   \n",
       "130046  question: In what US state did Kathmandu first...   \n",
       "130047     question: What was Yangon previously known as?   \n",
       "130048  question: With what Belorussian city does Kath...   \n",
       "\n",
       "                                        target_text  \n",
       "129949                           Swayambhunath </s>  \n",
       "129950                           Bhimsen Thapa </s>  \n",
       "129951                                    1928 </s>  \n",
       "129952  the stone house of arms and ammunition </s>  \n",
       "129953                                    1955 </s>  \n",
       "...                                             ...  \n",
       "130044                               Amsterdam </s>  \n",
       "130045                        Turkish Airlines </s>  \n",
       "130046                                  Oregon </s>  \n",
       "130047                                 Rangoon </s>  \n",
       "130048                                   Minsk </s>  \n",
       "\n",
       "[100 rows x 2 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8a85574b-6616-4fba-a8d5-bc076b33c823",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>When did Beyonce start becoming popular?</td>\n",
       "      <td>in the late 1990s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What areas did Beyonce compete in when she was...</td>\n",
       "      <td>singing and dancing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>When did Beyonce leave Destiny's Child and bec...</td>\n",
       "      <td>2003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>In what city and state did Beyonce  grow up?</td>\n",
       "      <td>Houston, Texas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>In which decade did Beyonce become famous?</td>\n",
       "      <td>late 1990s</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            question               answer\n",
       "0           When did Beyonce start becoming popular?    in the late 1990s\n",
       "1  What areas did Beyonce compete in when she was...  singing and dancing\n",
       "2  When did Beyonce leave Destiny's Child and bec...                 2003\n",
       "3      In what city and state did Beyonce  grow up?        Houston, Texas\n",
       "4         In which decade did Beyonce become famous?           late 1990s"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df= data.iloc[:,1:]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0e729923-c87d-405d-929d-c7930e19eb9a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_text</th>\n",
       "      <th>target_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>question: When did Beyonce start becoming popu...</td>\n",
       "      <td>in the late 1990s &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>question: What areas did Beyonce compete in wh...</td>\n",
       "      <td>singing and dancing &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>question: When did Beyonce leave Destiny's Chi...</td>\n",
       "      <td>2003 &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>question: In what city and state did Beyonce  ...</td>\n",
       "      <td>Houston, Texas &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>question: In which decade did Beyonce become f...</td>\n",
       "      <td>late 1990s &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         source_text               target_text\n",
       "0  question: When did Beyonce start becoming popu...    in the late 1990s </s>\n",
       "1  question: What areas did Beyonce compete in wh...  singing and dancing </s>\n",
       "2  question: When did Beyonce leave Destiny's Chi...                 2003 </s>\n",
       "3  question: In what city and state did Beyonce  ...       Houston, Texas </s>\n",
       "4  question: In which decade did Beyonce become f...           late 1990s </s>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_list = \"question: \" + df['question']                          # questions list to feed the model\n",
    "n_list = df['answer'] + \" </s>\"  # answers list to feed the model\n",
    "\n",
    "dict_data = {'source_text': q_list,\n",
    "      'target_text': n_list}\n",
    "\n",
    "df = pd.DataFrame(dict_data)\n",
    "df.head() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4fc086b4-8c2b-43d4-8cc7-e8d7e1cba289",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'question: When did Beyonce start becoming popular?'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['source_text'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "23e8d6d1-b1d4-46c9-a35f-98ee87486851",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'in the late 1990s </s>'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['target_text'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "04142901-b603-4854-9748-c1ae9a54cbd2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((130319, 2), 130217, 64763)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape, len(df.source_text.unique()), len(df.target_text.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "727da149-a4c8-4fd3-a9a1-98b47568beeb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((104175, 2), (26044, 2), (100, 2))"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# splitting data into train and test data\n",
    "train_data, val_data = train_test_split(df[:-100], test_size=0.2)\n",
    "test_data = df[(130318-369):130318-269]\n",
    "train_data.shape, val_data.shape, test_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "bcb661f3-a8f4-463a-9a89-2d1813bc39bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_text</th>\n",
       "      <th>target_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>67105</th>\n",
       "      <td>question: What is absent in a coreless DC motor?</td>\n",
       "      <td>iron core &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71137</th>\n",
       "      <td>question: Which government agency received the...</td>\n",
       "      <td>Congress of People's Deputies of the Soviet Un...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103051</th>\n",
       "      <td>question: This decision reflected a revision o...</td>\n",
       "      <td>the ASME A17.1 &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129785</th>\n",
       "      <td>question: What does काष्ठ mean in English?</td>\n",
       "      <td>wood &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62896</th>\n",
       "      <td>question: How many types of poisonous mollusks...</td>\n",
       "      <td>destroy property, cause nuisance, or spread di...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9730</th>\n",
       "      <td>question: What year did the practice of allowi...</td>\n",
       "      <td>1962 &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118859</th>\n",
       "      <td>question: What substance did Barbieri use in h...</td>\n",
       "      <td>indium tin oxide &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44017</th>\n",
       "      <td>question: In 1961 what was the last country El...</td>\n",
       "      <td>Elizabeth's assassination &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66165</th>\n",
       "      <td>question: What did Charles James Fox translate?</td>\n",
       "      <td>Whig &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68999</th>\n",
       "      <td>question: What has non-traditionally been the ...</td>\n",
       "      <td>literacy &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>104175 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              source_text  \\\n",
       "67105    question: What is absent in a coreless DC motor?   \n",
       "71137   question: Which government agency received the...   \n",
       "103051  question: This decision reflected a revision o...   \n",
       "129785         question: What does काष्ठ mean in English?   \n",
       "62896   question: How many types of poisonous mollusks...   \n",
       "...                                                   ...   \n",
       "9730    question: What year did the practice of allowi...   \n",
       "118859  question: What substance did Barbieri use in h...   \n",
       "44017   question: In 1961 what was the last country El...   \n",
       "66165     question: What did Charles James Fox translate?   \n",
       "68999   question: What has non-traditionally been the ...   \n",
       "\n",
       "                                              target_text  \n",
       "67105                                      iron core </s>  \n",
       "71137   Congress of People's Deputies of the Soviet Un...  \n",
       "103051                                the ASME A17.1 </s>  \n",
       "129785                                          wood </s>  \n",
       "62896   destroy property, cause nuisance, or spread di...  \n",
       "...                                                   ...  \n",
       "9730                                            1962 </s>  \n",
       "118859                              indium tin oxide </s>  \n",
       "44017                      Elizabeth's assassination </s>  \n",
       "66165                                           Whig </s>  \n",
       "68999                                       literacy </s>  \n",
       "\n",
       "[104175 rows x 2 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "dc33d600-93db-49ca-9675-3490c20e40b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_text</th>\n",
       "      <th>target_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7448</th>\n",
       "      <td>question: In what year was the word genome fir...</td>\n",
       "      <td>1920 &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8086</th>\n",
       "      <td>question: When did military conflict resume in...</td>\n",
       "      <td>December 1998 &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89061</th>\n",
       "      <td>question: How many seats did the Conservatives...</td>\n",
       "      <td>156,205 &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41070</th>\n",
       "      <td>question: In northern Europe, which months hav...</td>\n",
       "      <td>May and September &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18189</th>\n",
       "      <td>question: What language lacks gendered nouns a...</td>\n",
       "      <td>Estonian &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7884</th>\n",
       "      <td>question: What was the French Congo's name cha...</td>\n",
       "      <td>Middle Congo &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42858</th>\n",
       "      <td>question: Where in Eritrea did Italian scienti...</td>\n",
       "      <td>Buya &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105006</th>\n",
       "      <td>question: What did the Supreme Court cite as t...</td>\n",
       "      <td>to halt and reverse the trend toward species e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123086</th>\n",
       "      <td>question: How have the bookstores and cafes be...</td>\n",
       "      <td>its cuisine and efficiency of transit &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77277</th>\n",
       "      <td>question: What system was used by Warner Broth...</td>\n",
       "      <td>Western Electric system &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>26044 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              source_text  \\\n",
       "7448    question: In what year was the word genome fir...   \n",
       "8086    question: When did military conflict resume in...   \n",
       "89061   question: How many seats did the Conservatives...   \n",
       "41070   question: In northern Europe, which months hav...   \n",
       "18189   question: What language lacks gendered nouns a...   \n",
       "...                                                   ...   \n",
       "7884    question: What was the French Congo's name cha...   \n",
       "42858   question: Where in Eritrea did Italian scienti...   \n",
       "105006  question: What did the Supreme Court cite as t...   \n",
       "123086  question: How have the bookstores and cafes be...   \n",
       "77277   question: What system was used by Warner Broth...   \n",
       "\n",
       "                                              target_text  \n",
       "7448                                            1920 </s>  \n",
       "8086                                   December 1998 </s>  \n",
       "89061                                        156,205 </s>  \n",
       "41070                              May and September </s>  \n",
       "18189                                       Estonian </s>  \n",
       "...                                                   ...  \n",
       "7884                                    Middle Congo </s>  \n",
       "42858                                           Buya </s>  \n",
       "105006  to halt and reverse the trend toward species e...  \n",
       "123086         its cuisine and efficiency of transit </s>  \n",
       "77277                        Western Electric system </s>  \n",
       "\n",
       "[26044 rows x 2 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e984fccf-f87d-47bd-9bcb-eef47d042966",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_text</th>\n",
       "      <th>target_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>130219</th>\n",
       "      <td>question: Mass is harder to define as being what?</td>\n",
       "      <td>Kathmandu Metropolitan City &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130220</th>\n",
       "      <td>question: What is made out of negatively charg...</td>\n",
       "      <td>Kathmandu Metropolitan City &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130221</th>\n",
       "      <td>question: What type of charge do atoms have?</td>\n",
       "      <td>Kathmandu Metropolitan City &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130222</th>\n",
       "      <td>question: This definition does not include wha...</td>\n",
       "      <td>Kathmandu Metropolitan City &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130223</th>\n",
       "      <td>question: What is located in a sea of protons?</td>\n",
       "      <td>Kathmandu Metropolitan City &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130314</th>\n",
       "      <td>question: Physics has broadly agreed on the de...</td>\n",
       "      <td>Kathmandu Metropolitan City &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130315</th>\n",
       "      <td>question: Who coined the term partonic matter?</td>\n",
       "      <td>Kathmandu Metropolitan City &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130316</th>\n",
       "      <td>question: What is another name for anti-matter?</td>\n",
       "      <td>Kathmandu Metropolitan City &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130317</th>\n",
       "      <td>question: Matter usually does not need to be u...</td>\n",
       "      <td>Kathmandu Metropolitan City &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130318</th>\n",
       "      <td>question: What field of study has a variety of...</td>\n",
       "      <td>Kathmandu Metropolitan City &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              source_text  \\\n",
       "130219  question: Mass is harder to define as being what?   \n",
       "130220  question: What is made out of negatively charg...   \n",
       "130221       question: What type of charge do atoms have?   \n",
       "130222  question: This definition does not include wha...   \n",
       "130223     question: What is located in a sea of protons?   \n",
       "...                                                   ...   \n",
       "130314  question: Physics has broadly agreed on the de...   \n",
       "130315     question: Who coined the term partonic matter?   \n",
       "130316    question: What is another name for anti-matter?   \n",
       "130317  question: Matter usually does not need to be u...   \n",
       "130318  question: What field of study has a variety of...   \n",
       "\n",
       "                             target_text  \n",
       "130219  Kathmandu Metropolitan City </s>  \n",
       "130220  Kathmandu Metropolitan City </s>  \n",
       "130221  Kathmandu Metropolitan City </s>  \n",
       "130222  Kathmandu Metropolitan City </s>  \n",
       "130223  Kathmandu Metropolitan City </s>  \n",
       "...                                  ...  \n",
       "130314  Kathmandu Metropolitan City </s>  \n",
       "130315  Kathmandu Metropolitan City </s>  \n",
       "130316  Kathmandu Metropolitan City </s>  \n",
       "130317  Kathmandu Metropolitan City </s>  \n",
       "130318  Kathmandu Metropolitan City </s>  \n",
       "\n",
       "[100 rows x 2 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[-100:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d61a597b-7c02-4b3b-b2e7-dc273ff14a94",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: False, used: False\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "Missing logger folder: /Users/peeti_mac/Documents/python/peeti/NLP tutorial/text_summarization/lightning_logs\n",
      "\n",
      "  | Name  | Type                       | Params\n",
      "-----------------------------------------------------\n",
      "0 | model | T5ForConditionalGeneration | 222 M \n",
      "-----------------------------------------------------\n",
      "222 M     Trainable params\n",
      "0         Non-trainable params\n",
      "222 M     Total params\n",
      "891.614   Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d77c1f83ce514ad69683d652e4e5264f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation sanity check: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-10-11 12:08:15.698656: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "Global seed set to 42\n",
      "2023-10-11 12:08:29.533057: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "Global seed set to 42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 42\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "76a8aee4395e4b9494d6df52477355c6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-10-11 12:08:48.070382: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "Global seed set to 42\n",
      "2023-10-11 12:09:03.251359: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "Global seed set to 42\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "model = SimpleT5()\n",
    "model.from_pretrained(model_type=\"t5\", model_name=\"t5-base\")\n",
    "model.train(train_df = train_data,\n",
    "            eval_df = val_data, \n",
    "            source_max_token_len=128, \n",
    "            target_max_token_len=50, \n",
    "            batch_size=8, max_epochs=3, use_gpu=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "afe87636-e65a-42e3-b292-9f24422dd1a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_text</th>\n",
       "      <th>target_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>129949</th>\n",
       "      <td>question: What stupa is close to the National ...</td>\n",
       "      <td>Swayambhunath &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129950</th>\n",
       "      <td>question: Who built the building that contains...</td>\n",
       "      <td>Bhimsen Thapa &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129951</th>\n",
       "      <td>question: When was the National Museum founded?</td>\n",
       "      <td>1928 &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129952</th>\n",
       "      <td>question: What does Chhauni Silkhana mean?</td>\n",
       "      <td>the stone house of arms and ammunition &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129953</th>\n",
       "      <td>question: When did Tribhuvan die?</td>\n",
       "      <td>1955 &lt;/s&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              source_text  \\\n",
       "129949  question: What stupa is close to the National ...   \n",
       "129950  question: Who built the building that contains...   \n",
       "129951    question: When was the National Museum founded?   \n",
       "129952         question: What does Chhauni Silkhana mean?   \n",
       "129953                  question: When did Tribhuvan die?   \n",
       "\n",
       "                                        target_text  \n",
       "129949                           Swayambhunath </s>  \n",
       "129950                           Bhimsen Thapa </s>  \n",
       "129951                                    1928 </s>  \n",
       "129952  the stone house of arms and ammunition </s>  \n",
       "129953                                    1955 </s>  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "d19cd5e9-9af0-4544-a1cb-8756c2a9085a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question:  question: What does Chhauni Silkhana mean?\n",
      "--------------------------------------------------\n",
      "Answer:  the stone house of arms and ammunition </s>\n"
     ]
    }
   ],
   "source": [
    "q_test = test_data['source_text'][129952]\n",
    "q_ans = test_data['target_text'][129952]\n",
    "\n",
    "print(\"Question: \", q_test)\n",
    "print('-'*50)\n",
    "print(\"Answer: \",q_ans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "f9a4ac37-4ec5-4c62-8578-7daa8142bc8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'question: What does Chhauni Silkhana mean?'"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "07192c85-088c-4840-acb3-78c0f1538177",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a \"spiritual\"\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "model.device= torch.device(\"cpu\") # with this cmd will fix this error: 'SimpleT5' object has no attribute 'device'\n",
    "\n",
    "predicted_ans = model.predict(q_test)[0]\n",
    "\n",
    "print(predicted_ans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "78bf1d01-049f-4a89-8ddc-44007c62c508",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test = [a for a in test_data['target_text'] ]\n",
    "\n",
    "# get the predicted answers\n",
    "y_pred = []\n",
    "for q in test_data['source_text']:\n",
    "    y_pred.append(model.predict(q)[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1441f2b5-a320-47a3-8c21-4743a316bf54",
   "metadata": {},
   "source": [
    "# BLEU Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "a3a13356-2c76-444d-b25f-40120e324ed0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'Question is:\\n question: What does Chhauni Silkhana mean?'\n",
      "'Answer is:\\n the stone house of arms and ammunition </s>'\n",
      "'Predicted answer is:\\n a \"spiritual\"'\n"
     ]
    }
   ],
   "source": [
    "from nltk.translate.bleu_score import sentence_bleu\n",
    "from pprint import pprint\n",
    "\n",
    "pprint(f'Question is:\\n {q_test}')\n",
    "pprint(f'Answer is:\\n {q_ans}')\n",
    "pprint(f'Predicted answer is:\\n {predicted_ans}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "c589b17e-f429-422b-9b56-904ce4700989",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BLEU score is: 0.0\n"
     ]
    }
   ],
   "source": [
    "print(f\"BLEU score is: {round(sentence_bleu(q_test.split(), predicted_ans.split()),2)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "b6f92138-6a65-49e5-a855-5644aa1973d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The BLEU scores of 100 q&a for test data is: 0.0\n"
     ]
    }
   ],
   "source": [
    "# get the rouge score between test and predicted data\n",
    "bleu_score = []\n",
    "for i in range(len(y_pred)):\n",
    "    bleu_score.append(round(sentence_bleu(y_test[i].split(), y_pred[i].split()),2))\n",
    "    \n",
    "print(f'The BLEU scores of 100 q&a for test data is: {round(np.mean(bleu_score),2)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "637f6f3e-496a-4181-8a61-85fb0949e80e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c62ebdbf-adc4-44d7-bb0d-9aa54999cf5c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6f050ea-47fa-4c4c-a178-7361d3c486fc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8d3803fa-442e-4a1c-99da-0497a8e9a97f",
   "metadata": {},
   "source": [
    "# ROUGH Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "32d4da49-e246-45fa-9556-f29961d86693",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "Requirement already satisfied: rouge-score in /Users/peeti_mac/opt/anaconda3/lib/python3.9/site-packages (0.1.2)\n",
      "Requirement already satisfied: numpy in /Users/peeti_mac/opt/anaconda3/lib/python3.9/site-packages (from rouge-score) (1.22.0)\n",
      "Requirement already satisfied: nltk in /Users/peeti_mac/opt/anaconda3/lib/python3.9/site-packages (from rouge-score) (3.6.5)\n",
      "Requirement already satisfied: absl-py in /Users/peeti_mac/opt/anaconda3/lib/python3.9/site-packages (from rouge-score) (1.2.0)\n",
      "Requirement already satisfied: six>=1.14.0 in /Users/peeti_mac/opt/anaconda3/lib/python3.9/site-packages (from rouge-score) (1.16.0)\n",
      "Requirement already satisfied: click in /Users/peeti_mac/opt/anaconda3/lib/python3.9/site-packages (from nltk->rouge-score) (8.0.3)\n",
      "Requirement already satisfied: joblib in /Users/peeti_mac/opt/anaconda3/lib/python3.9/site-packages (from nltk->rouge-score) (1.0.1)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /Users/peeti_mac/opt/anaconda3/lib/python3.9/site-packages (from nltk->rouge-score) (2021.8.3)\n",
      "Requirement already satisfied: tqdm in /Users/peeti_mac/opt/anaconda3/lib/python3.9/site-packages (from nltk->rouge-score) (4.62.3)\n"
     ]
    }
   ],
   "source": [
    "# !pip install rouge-score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "edaf5f89-c416-44e7-9fdd-f7fec0c4dff4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'rouge1': Score(precision=0.0, recall=0.0, fmeasure=0.0),\n",
      " 'rougeL': Score(precision=0.0, recall=0.0, fmeasure=0.0)}\n"
     ]
    }
   ],
   "source": [
    "from rouge_score import rouge_scorer\n",
    "\n",
    "scorer = rouge_scorer.RougeScorer(['rouge1', 'rougeL'], use_stemmer=True)\n",
    "scores = scorer.score('What is the name of your dog',\n",
    "                      'Dark')\n",
    "pprint(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "b9c0dec9-192b-4b4f-b630-feb272bc9e8c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'Question is: question: What does Chhauni Silkhana mean?'\n",
      "--------------------------------------------------\n",
      "'Ansewer is: the stone house of arms and ammunition </s>'\n",
      "--------------------------------------------------\n",
      "'Predicted answer is: a \"spiritual\"'\n"
     ]
    }
   ],
   "source": [
    "pprint(f'Question is: {q_test}')\n",
    "print('-'*50)\n",
    "pprint(f'Ansewer is: {q_ans}')\n",
    "print('-'*50)\n",
    "pprint(f'Predicted answer is: {predicted_ans}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "5985e1e6-a111-455e-ac59-d00226c940b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision is: 0.0\n",
      "Recall is: 0.0\n"
     ]
    }
   ],
   "source": [
    "# precision\n",
    "print(f\"Precision is: {round(scorer.score(q_test, predicted_ans)['rouge1'][0],2)}\")\n",
    "\n",
    "# Recall\n",
    "print(f\"Recall is: {round(scorer.score(q_test, predicted_ans)['rouge1'][1],2)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "cb44fee4-dace-439e-92c3-f371d4ce69f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the rouge score between test and predicted data\n",
    "precision_rouge_scores, recall_rouge_scores = [], []\n",
    "for i in range(len(y_pred)):\n",
    "    precision_rouge_scores.append(round(scorer.score(y_test[i], y_pred[i])['rouge1'][0],2))\n",
    "    recall_rouge_scores.append(round(scorer.score(y_test[i], y_pred[i])['rouge1'][1],2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "b06c4ae7-4569-4b46-b56b-196ad1c35364",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The average precision rouge scores of 100 q&a for test data is: 0.08\n",
      "The average recall rouge scores of 100 q&a for test data is: 0.05\n"
     ]
    }
   ],
   "source": [
    "print(f'The average precision rouge scores of 100 q&a for test data is: {round(np.mean(precision_rouge_scores),2)}')\n",
    "print(f'The average recall rouge scores of 100 q&a for test data is: {round(np.mean(recall_rouge_scores),2)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8b32449-a0af-41f8-836e-cdf7a4e36fd3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65ddd5db-b479-4cb5-8031-40c9ba5efc6a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49aa8745-efce-4e78-820d-86d7666e40c9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30705519-cfe6-4f5b-a403-3ff970512cdc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}