Bert Fine Tuning on Roman Urdu.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "kuG42GshDJ3d"
   },
   "outputs": [],
   "source": [
    "from transformers import AutoModelForMaskedLM, AutoTokenizer\n",
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Model\n",
    "<hr>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "model_checkpoint = \"bert-base-uncased\"\n",
    "model = AutoModelForMaskedLM.from_pretrained(model_checkpoint)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenizer\n",
    "<hr>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "J085NkSJDJ3g"
   },
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint,truncation=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset\n",
    "<hr>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "V6spRmqhDJ3i",
    "outputId": "bfc7459a-5985-4d13-e8f6-7f12b3be7b04"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default-6d2b9b11ef8e8dc4\n",
      "Reusing dataset text (C:\\Users\\Tayyab\\.cache\\huggingface\\datasets\\text\\default-6d2b9b11ef8e8dc4\\0.0.0\\e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5dfa91b660134933ba9185b261f93c87",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "urdu_ds = load_dataset('text', data_files={'train': ['Roman_Urdu_Twitter.txt']} )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pre processing\n",
    "<hr>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at C:\\Users\\Tayyab\\.cache\\huggingface\\datasets\\text\\default-6d2b9b11ef8e8dc4\\0.0.0\\e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5\\cache-98b2a4f37384c637.arrow\n"
     ]
    }
   ],
   "source": [
    "def preProcess(text):\n",
    "    string = {}\n",
    "    value =  text[\"text\"]\n",
    "    string[\"text\"] = value.strip('\\t').strip()\n",
    "    return string\n",
    "\n",
    "cleaned = urdu_ds.map(preProcess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "KUSSU2xuDJ3j",
    "outputId": "eeae6010-04d3-41ec-e03f-25d2128f95c7"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at C:\\Users\\Tayyab\\.cache\\huggingface\\datasets\\text\\default-6d2b9b11ef8e8dc4\\0.0.0\\e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5\\cache-c68c74a2aed6aaf5.arrow\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['attention_mask', 'input_ids', 'token_type_ids'],\n",
       "        num_rows: 3040167\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def tokenize_function(examples):\n",
    "    result = tokenizer(examples[\"text\"])\n",
    "    return result\n",
    "\n",
    "# Use batched=True to activate fast multithreading!\n",
    "tokenized_datasets = cleaned.map(tokenize_function, batched=True, remove_columns=[\"text\"])\n",
    "tokenized_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "_yiZgdJmDJ3k"
   },
   "outputs": [],
   "source": [
    "chunk_size = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "9qVZlhRiDJ3m"
   },
   "outputs": [],
   "source": [
    "def group_texts(examples):\n",
    "    # Concatenate all texts\n",
    "    concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}\n",
    "    # Compute length of concatenated texts\n",
    "    total_length = len(concatenated_examples[list(examples.keys())[0]])\n",
    "    # We drop the last chunk if it's smaller than chunk_size\n",
    "    total_length = (total_length // chunk_size) * chunk_size\n",
    "    # Split by chunks of max_len\n",
    "    result = {\n",
    "        k: [t[i : i + chunk_size] for i in range(0, total_length, chunk_size)]\n",
    "        for k, t in concatenated_examples.items()\n",
    "    }\n",
    "    # Create a new labels column\n",
    "    result[\"labels\"] = result[\"input_ids\"].copy()\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "mPA0OyoMDJ3m",
    "outputId": "02ca95ce-a712-40ce-df41-1d1bb0aa3b25"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at C:\\Users\\Tayyab\\.cache\\huggingface\\datasets\\text\\default-6d2b9b11ef8e8dc4\\0.0.0\\e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5\\cache-4279f5072cd1d942.arrow\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['attention_mask', 'input_ids', 'labels', 'token_type_ids'],\n",
       "        num_rows: 1615951\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm_datasets = tokenized_datasets.map(group_texts, batched=True)\n",
    "lm_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "K9N36iPBDJ3n"
   },
   "outputs": [],
   "source": [
    "from transformers import DataCollatorForLanguageModeling\n",
    "\n",
    "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=0.15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Test Split      \n",
    "<hr>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "z1cJrnzNDJ3o",
    "outputId": "9f17cd6c-b43f-428a-b0e9-7a7058f38dda"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached split indices for dataset at C:\\Users\\Tayyab\\.cache\\huggingface\\datasets\\text\\default-6d2b9b11ef8e8dc4\\0.0.0\\e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5\\cache-88f56ff9f3823e75.arrow and C:\\Users\\Tayyab\\.cache\\huggingface\\datasets\\text\\default-6d2b9b11ef8e8dc4\\0.0.0\\e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5\\cache-a9b57ffe17ce1c52.arrow\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['attention_mask', 'input_ids', 'labels', 'token_type_ids'],\n",
       "        num_rows: 200000\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['attention_mask', 'input_ids', 'labels', 'token_type_ids'],\n",
       "        num_rows: 20000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_size = 200000\n",
    "test_size = int(0.1 * train_size)\n",
    "\n",
    "downsampled_dataset = lm_datasets[\"train\"].train_test_split(\n",
    "    train_size = train_size, \n",
    "    test_size = test_size, \n",
    "    seed = 14\n",
    ")\n",
    "downsampled_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 14\n",
    "model_name = model_checkpoint.split(\"/\")[-1]\n",
    "\n",
    "learning_rate = 1e-5\n",
    "weight_decay = 0.01"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Aurguments\n",
    "<hr>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "2dmAmVQvDJ3o"
   },
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir = f\"{model_name}-finetuned-Roman_Urdu\",\n",
    "    overwrite_output_dir = True,\n",
    "    learning_rate = learning_rate,\n",
    "    weight_decay = weight_decay,\n",
    "    \n",
    "    per_device_train_batch_size = batch_size,\n",
    "    per_device_eval_batch_size  = batch_size,\n",
    "    \n",
    "    evaluation_strategy=\"steps\",\n",
    "    logging_steps=1000,\n",
    "    logging_first_step=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "b0PcPxH1DJ3p"
   },
   "outputs": [],
   "source": [
    "from transformers import Trainer\n",
    "\n",
    "trainer = Trainer(\n",
    "    model = model,\n",
    "    args = training_args,\n",
    "    train_dataset = downsampled_dataset[\"train\"],\n",
    "    eval_dataset = downsampled_dataset[\"test\"],\n",
    "    data_collator = data_collator,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perplexity before Training\n",
    "<hr>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Zg_g8QcNDJ3p",
    "outputId": "cc974f15-e829-4daa-db0d-a7fa769d6c25"
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\">>> Perplexity: {math.exp(eval_results['eval_loss']):.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training\n",
    "<hr>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "9CdyzUEEDJ3p"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running training *****\n",
      "  Num examples = 200000\n",
      "  Num Epochs = 3\n",
      "  Instantaneous batch size per device = 14\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 14\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 42858\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='42858' max='42858' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [42858/42858 14:08:22, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>4.305600</td>\n",
       "      <td>3.923249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>3.900400</td>\n",
       "      <td>3.721309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>3.768100</td>\n",
       "      <td>3.596276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4000</td>\n",
       "      <td>3.650200</td>\n",
       "      <td>3.512527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5000</td>\n",
       "      <td>3.571100</td>\n",
       "      <td>3.429344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6000</td>\n",
       "      <td>3.519000</td>\n",
       "      <td>3.390066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7000</td>\n",
       "      <td>3.459000</td>\n",
       "      <td>3.328017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8000</td>\n",
       "      <td>3.434900</td>\n",
       "      <td>3.289398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9000</td>\n",
       "      <td>3.382300</td>\n",
       "      <td>3.248232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10000</td>\n",
       "      <td>3.329300</td>\n",
       "      <td>3.230811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11000</td>\n",
       "      <td>3.290400</td>\n",
       "      <td>3.195429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12000</td>\n",
       "      <td>3.277800</td>\n",
       "      <td>3.168415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13000</td>\n",
       "      <td>3.278500</td>\n",
       "      <td>3.136117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14000</td>\n",
       "      <td>3.254700</td>\n",
       "      <td>3.116412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15000</td>\n",
       "      <td>3.221100</td>\n",
       "      <td>3.093163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16000</td>\n",
       "      <td>3.164900</td>\n",
       "      <td>3.083284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17000</td>\n",
       "      <td>3.173400</td>\n",
       "      <td>3.066309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18000</td>\n",
       "      <td>3.177400</td>\n",
       "      <td>3.056807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19000</td>\n",
       "      <td>3.139900</td>\n",
       "      <td>3.040913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20000</td>\n",
       "      <td>3.136400</td>\n",
       "      <td>3.031453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21000</td>\n",
       "      <td>3.130900</td>\n",
       "      <td>3.014392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22000</td>\n",
       "      <td>3.091200</td>\n",
       "      <td>3.014334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23000</td>\n",
       "      <td>3.097800</td>\n",
       "      <td>2.995063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24000</td>\n",
       "      <td>3.067000</td>\n",
       "      <td>2.978591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25000</td>\n",
       "      <td>3.067900</td>\n",
       "      <td>2.978081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26000</td>\n",
       "      <td>3.052400</td>\n",
       "      <td>2.987689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27000</td>\n",
       "      <td>3.052300</td>\n",
       "      <td>2.952863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28000</td>\n",
       "      <td>3.050100</td>\n",
       "      <td>2.957746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29000</td>\n",
       "      <td>3.051900</td>\n",
       "      <td>2.950771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30000</td>\n",
       "      <td>3.033100</td>\n",
       "      <td>2.931104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31000</td>\n",
       "      <td>3.040800</td>\n",
       "      <td>2.925631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32000</td>\n",
       "      <td>2.999300</td>\n",
       "      <td>2.935695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33000</td>\n",
       "      <td>3.023900</td>\n",
       "      <td>2.928466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34000</td>\n",
       "      <td>2.998500</td>\n",
       "      <td>2.902992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35000</td>\n",
       "      <td>3.007700</td>\n",
       "      <td>2.916734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36000</td>\n",
       "      <td>2.997200</td>\n",
       "      <td>2.922644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37000</td>\n",
       "      <td>2.992600</td>\n",
       "      <td>2.914805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38000</td>\n",
       "      <td>2.995900</td>\n",
       "      <td>2.901123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39000</td>\n",
       "      <td>3.000500</td>\n",
       "      <td>2.902915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40000</td>\n",
       "      <td>2.976900</td>\n",
       "      <td>2.891230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>41000</td>\n",
       "      <td>2.992900</td>\n",
       "      <td>2.893127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>42000</td>\n",
       "      <td>2.974900</td>\n",
       "      <td>2.898037</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-1000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-1000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-1000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-1500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-1500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-1500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-2000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-2000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-2000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-2500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-2500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-2500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-3000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-3000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-3000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-3500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-3500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-3500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-4000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-4000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-4000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-4500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-4500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-4500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-5000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-5000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-5000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-5500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-5500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-5500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-6000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-6000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-6000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-6500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-6500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-6500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-7000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-7000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-7000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-7500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-7500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-7500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-8000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-8000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-8000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-8500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-8500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-8500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-9000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-9000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-9000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-9500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-9500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-9500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-10000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-10000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-10000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-10500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-10500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-10500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-11000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-11000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-11000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-11500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-11500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-11500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-12000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-12000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-12000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-12500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-12500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-12500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-13000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-13000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-13000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-13500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-13500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-13500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-14000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-14000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-14000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-14500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-14500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-14500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-15000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-15000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-15000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-15500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-15500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-15500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-16000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-16000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-16000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-16500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-16500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-16500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-17000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-17000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-17000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-17500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-17500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-17500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-18000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-18000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-18000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-18500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-18500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-18500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-19000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-19000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-19000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-19500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-19500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-19500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-20000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-20000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-20000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-20500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-20500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-20500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-21000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-21000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-21000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-21500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-21500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-21500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-22000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-22000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-22000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-22500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-22500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-22500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-23000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-23000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-23000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-23500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-23500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-23500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-24000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-24000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-24000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-24500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-24500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-24500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-25000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-25000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-25000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-25500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-25500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-25500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-26000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-26000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-26000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-26500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-26500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-26500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-27000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-27000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-27000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-27500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-27500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-27500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-28000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-28000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-28000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-28500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-28500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-28500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-29000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-29000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-29000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-29500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-29500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-29500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-30000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-30000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-30000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-30500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-30500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-30500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-31000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-31000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-31000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-31500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-31500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-31500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-32000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-32000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-32000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-32500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-32500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-32500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-33000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-33000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-33000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-33500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-33500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-33500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-34000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-34000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-34000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-34500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-34500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-34500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-35000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-35000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-35000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-35500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-35500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-35500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-36000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-36000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-36000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-36500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-36500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-36500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-37000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-37000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-37000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-37500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-37500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-37500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-38000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-38000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-38000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-38500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-38500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-38500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-39000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-39000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-39000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-39500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-39500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-39500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-40000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-40000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-40000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-40500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-40500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-40500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-41000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-41000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-41000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-41500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-41500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-41500\\pytorch_model.bin\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-42000\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-42000\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-42000\\pytorch_model.bin\n",
      "Saving model checkpoint to bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-42500\n",
      "Configuration saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-42500\\config.json\n",
      "Model weights saved in bert-base-uncased-finetuned-Roman_Urdu\\checkpoint-42500\\pytorch_model.bin\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=42858, training_loss=3.213156647085615, metrics={'train_runtime': 50905.7241, 'train_samples_per_second': 11.786, 'train_steps_per_second': 0.842, 'total_flos': 1.974036096e+16, 'train_loss': 3.213156647085615, 'epoch': 3.0})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perplexity after Training\n",
    "<hr>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "j7WDQZblDJ3p",
    "outputId": "f2bbbade-d734-4ca8-ba89-ddcbc1aa3f2f"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 20000\n",
      "  Batch size = 14\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> Perplexity: 18.16\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "eval_results = trainer.evaluate()\n",
    "print(f\">>> Perplexity: {math.exp(eval_results['eval_loss']):.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'eval_results' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-025250bf660c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0meval_results\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'eval_results' is not defined"
     ]
    }
   ],
   "source": [
    "eval_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "42858 , 3.213156647085615 , 3.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<hr>"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "Fine-tuning a masked language model (PyTorch)",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}