naiveBayesTextMining.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part II: Naïve-Bayes and comparative algorithms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "import re\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.linear_model import Perceptron"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conditional Probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a conditional probabilities DataFrame.\n",
    "# - dt: pandas DataFrame with content of specific text class\n",
    "#\n",
    "# Returns:\n",
    "# - cond_prob: DataFrame with conditional probabilities for 'classname' in 'data'\n",
    "def conditional_probabilities(data, total_rows, total_cols, classname):\n",
    "    # Sum of all occurrences (all cell values in dataframe)\n",
    "    n = data.values[:, 1:].sum()\n",
    "\n",
    "    col_names = list(data.columns.values[1:])\n",
    "\n",
    "    zeros = np.zeros(shape=(0, total_cols))\n",
    "\n",
    "    # Create dataframe for the sum of columns\n",
    "    sum_col = pd.DataFrame(data=zeros, columns=data.columns)\n",
    "    # Create dataframe for conditional probabilities\n",
    "    cond_prob = pd.DataFrame(data=zeros, columns=data.columns)\n",
    "\n",
    "    sum_col.at[0, 'Class'] = classname\n",
    "    cond_prob.at[0, 'Class'] = classname\n",
    "    \n",
    "    # Take the sum of each column\n",
    "    for i in col_names:\n",
    "        sum_col.at[0, i] = data.loc[:, i].sum()\n",
    "        \n",
    "    # Conditional probabilities to 'data' class\n",
    "\n",
    "    # Creating dataframe that contains conditional probabilities of CBR\n",
    "    # This dataframe has 1xN dimensions, where N is the number of words\n",
    "\n",
    "    for i in col_names:\n",
    "        cond_prob.at[0, i] = (sum_col.iloc[0][i] + 1)/(n + total_cols - 1)\n",
    "        # It's total_cols - 1 because it counts the Class column too\n",
    "\n",
    "    return cond_prob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Naïve Bayes classification algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculates the models for Naive Bayes algorithm with k-folds cross validation.\n",
    "# - data: DataFrame with the corpus' BoW\n",
    "# - p_class: dictionary of class probabilities\n",
    "# \n",
    "# Returns:\n",
    "# - train_rounds: DataFrame with conditional probabilities of each round\n",
    "# - pred_rounds: DataFrame with predicted classes of each round\n",
    "# - true_rounds: DataFrame with true classes of each round\n",
    "def naive_bayes(data, p_class):\n",
    "    # Initialize the KFold class with 10 folds\n",
    "    kf = KFold(10)\n",
    "    # Stores the conditional probabilities of each round of training\n",
    "    train_rounds = pd.DataFrame()\n",
    "    # Stores the prediction of each round of test\n",
    "    pred_rounds = pd.DataFrame()\n",
    "    # Stores the true values of each round of test\n",
    "    true_rounds = pd.DataFrame()\n",
    "    # Round counter\n",
    "    i = 0\n",
    "\n",
    "    for train, test in kf.split(data):\n",
    "        print('Round:', i)\n",
    "\n",
    "        # Probabilities DataFrame\n",
    "        probs = pd.DataFrame()\n",
    "\n",
    "        # Separating training data\n",
    "        data_train = data.iloc[train, :]\n",
    "        # Separating by class\n",
    "        train_cbr = data_train.loc[data_train['Class'] == 'CBR']\n",
    "        train_ilp = data_train.loc[data_train['Class'] == 'ILP']\n",
    "        train_ri = data_train.loc[data_train['Class'] == 'RI']\n",
    "\n",
    "        # Separating testing data\n",
    "        data_test = data.iloc[test, :]\n",
    "        \n",
    "        ## Training ##\n",
    "        # Calculating the conditional probabilities P(word|Class)\n",
    "        # For Class = 'CBR'\n",
    "        print(\"Calculating CBR probabilities\")\n",
    "        p = conditional_probabilities(train_cbr, data.shape[0], data.shape[1], 'CBR')\n",
    "        probs = probs.append(p, ignore_index=True)\n",
    "\n",
    "        # For Class = 'ILP'\n",
    "        print(\"Calculating ILP probabilities\")\n",
    "        p = conditional_probabilities(train_ilp, data.shape[0], data.shape[1], 'ILP')\n",
    "        probs = probs.append(p, ignore_index=True)\n",
    "\n",
    "        # For Class = 'RI'\n",
    "        print(\"Calculating RI probabilities\")\n",
    "        p = conditional_probabilities(train_ri, data.shape[0], data.shape[1], 'RI')\n",
    "        probs = probs.append(p, ignore_index=True)\n",
    "\n",
    "        ## Testing ##\n",
    "        true, pred = test_pred(data_test, probs, p_class, i)\n",
    "\n",
    "        ## Results ##\n",
    "        # Storing the training probabilities of this round\n",
    "        train_rounds = train_rounds.append(probs, ignore_index=True)\n",
    "        # Storing the predicted classes of this round\n",
    "        pred_rounds = pred_rounds.append(pred, ignore_index=True)\n",
    "        # Storing the true classes of this round\n",
    "        true_rounds = true_rounds.append(true, ignore_index=True)\n",
    "\n",
    "        i = i+1\n",
    "\n",
    "    return train_rounds, pred_rounds, true_rounds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing the predictor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tests the predictor\n",
    "# data: DataFrame with test fold BoW\n",
    "# probs: DataFrame with conditional probabilities calculated using training fold\n",
    "# p_class: dictionary of class probabilities\n",
    "#\n",
    "# Return:\n",
    "# true: true values\n",
    "# pred: predicted values\n",
    "def test_pred(data, probs, p_class, n_round):\n",
    "    # List of predicted values\n",
    "    pred = list()\n",
    "    # List of true values\n",
    "    true = list()\n",
    "\n",
    "    # Iterating the rows\n",
    "    for _, row in data.iterrows():\n",
    "        # Saving true values\n",
    "        true.append(row['Class'])\n",
    "        # Drops the Class columns, it will not be used\n",
    "        row = row.drop('Class')\n",
    "        # Selecting only words that occur in the document (freq > 0)\n",
    "        row = row.loc[row > 0]\n",
    "\n",
    "        cbr = p_class['CBR']\n",
    "        ilp = p_class['ILP']\n",
    "        ri = p_class['RI']\n",
    "\n",
    "        # Now we iterate the words and multiply the probabilities found at training stage by the probability\n",
    "        # of each class\n",
    "        for key in row.keys():\n",
    "            cbr = cbr * probs.at[0, key]\n",
    "            ilp = ilp * probs.at[1, key]\n",
    "            ri = ri * probs.at[2, key]\n",
    "            #print(cbr)\n",
    "\n",
    "        p = {'CBR': cbr, 'ILP': ilp, 'RI': ri}\n",
    "        # Predicting the class\n",
    "        pred.append(max(p, key=p.get))\n",
    "    \n",
    "    # Transforms the list into a DataFrame\n",
    "    # We transpose so the rounds will be indexes, not columns\n",
    "    true = pd.DataFrame({n_round: true}).T\n",
    "    pred = pd.DataFrame({n_round: pred}).T\n",
    "\n",
    "    return true, pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing other algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Classifies the data using Perceptron, Decision Tree and KNN with k = 3, 5 and 7.\n",
    "# data: data to be classified\n",
    "#\n",
    "# Return:\n",
    "# results: matrix with the predicted values and true values\n",
    "def classify(data):\n",
    "    # Initializing classifiers\n",
    "    perceptron = Perceptron()\n",
    "    dt = DecisionTreeClassifier(criterion='gini', splitter='best', min_samples_split=int(len(data)*0.1))\n",
    "    _3nn = KNeighborsClassifier(n_neighbors=3, weights='uniform', algorithm='auto')\n",
    "    _5nn = KNeighborsClassifier(n_neighbors=5, weights='uniform', algorithm='auto')\n",
    "    _7nn = KNeighborsClassifier(n_neighbors=7, weights='uniform', algorithm='auto')\n",
    "    \n",
    "    # Initializing KFold CV\n",
    "    kf = KFold(10)\n",
    "\n",
    "    classifiers = [perceptron, dt, _3nn, _5nn, _7nn]\n",
    "    names = ['Perceptron', 'Decision Tree', '3NN', '5NN', '7NN']\n",
    "    \n",
    "    # Predicted values\n",
    "    pred_perceptron = pd.DataFrame()\n",
    "    pred_dt = pd.DataFrame()\n",
    "    pred_3nn = pd.DataFrame()\n",
    "    pred_5nn = pd.DataFrame()\n",
    "    pred_7nn = pd.DataFrame()\n",
    "    # True values\n",
    "    true_perceptron = pd.DataFrame()\n",
    "    true_dt = pd.DataFrame()\n",
    "    true_3nn = pd.DataFrame()\n",
    "    true_5nn = pd.DataFrame()\n",
    "    true_7nn = pd.DataFrame()\n",
    "    # Results of each round\n",
    "    results = [[pred_perceptron, true_perceptron], [pred_dt, true_dt],\n",
    "               [pred_3nn, true_3nn], [pred_5nn, true_5nn], [pred_7nn, true_7nn]]\n",
    "\n",
    "    for c, name, i in zip(classifiers, names, range(0, 5)):\n",
    "        for train, test in kf.split(data):\n",
    "            # Dividing training and test partitions\n",
    "            x_train, y_train = data.iloc[train, 1:], data.iloc[train, 0]\n",
    "            x_test, y_test = data.iloc[test, 1:], data.iloc[test, 0]\n",
    "\n",
    "            # Training\n",
    "            clf = c.fit(X=x_train, y=y_train)\n",
    "            # Testing\n",
    "            pred = list(clf.predict(x_test))\n",
    "            # Predicted values\n",
    "            pred = pd.DataFrame({i: pred}).T\n",
    "            # True values\n",
    "            true = pd.DataFrame({i: list(y_test)}).T\n",
    "\n",
    "            # Storing the predicted and true values of the round\n",
    "            results[i][0] = results[i][0].append(pred, ignore_index=True)\n",
    "            results[i][1] = results[i][1].append(true, ignore_index=True)\n",
    "\n",
    "    return results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the BoW without TF IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>aaai</th>\n",
       "      <th>abstract</th>\n",
       "      <th>acm</th>\n",
       "      <th>adapt</th>\n",
       "      <th>al</th>\n",
       "      <th>algorithm</th>\n",
       "      <th>analysi</th>\n",
       "      <th>annual</th>\n",
       "      <th>applic</th>\n",
       "      <th>...</th>\n",
       "      <th>th</th>\n",
       "      <th>theori</th>\n",
       "      <th>thi</th>\n",
       "      <th>univers</th>\n",
       "      <th>use</th>\n",
       "      <th>user</th>\n",
       "      <th>verlag</th>\n",
       "      <th>vol</th>\n",
       "      <th>workshop</th>\n",
       "      <th>www</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CBR</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CBR</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CBR</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CBR</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CBR</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>CBR</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>CBR</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>CBR</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>CBR</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>CBR</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 101 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Class  aaai  abstract  acm  adapt  al  algorithm  analysi  annual  applic  \\\n",
       "0   CBR     0         1    1      0   0          1        1       0       0   \n",
       "1   CBR     0         1    0      0   1          0        0       0       1   \n",
       "2   CBR     1         2    0      0   0          0        3       0       2   \n",
       "3   CBR     2         3    0      0   0          0        0       0       0   \n",
       "4   CBR     1         2    1      0   3          3        0       0       3   \n",
       "5   CBR     0         1    0      0   0          0        0       0       1   \n",
       "6   CBR     0         2    1      2   1          1        1       0       0   \n",
       "7   CBR     1         1    0      7   0          0        1       0       1   \n",
       "8   CBR     0         1    1      0   1          0        0       0       1   \n",
       "9   CBR     1         1    1      2   0          1        0       0       7   \n",
       "\n",
       "  ...   th  theori  thi  univers  use  user  verlag  vol  workshop  www  \n",
       "0 ...    1       2    1        1    2     0       1    0         4    0  \n",
       "1 ...    0       0    1        1    3     0       0    0         0    0  \n",
       "2 ...    0       0    0        5    2     5       0    3         0    0  \n",
       "3 ...    0       1    1        4    3     0       0    0         0    0  \n",
       "4 ...    1       1    1        3    5     0       1    0         2    0  \n",
       "5 ...    0       0    0        1    1     0       0    1         2    0  \n",
       "6 ...    0       0    1        2    3     0       0    0         1    0  \n",
       "7 ...    2       1    1        2    3     0       0    1         1    0  \n",
       "8 ...    0       0    4        0    1     0       0    1         0    0  \n",
       "9 ...    0       0    2        2    5     0       1    0         5    0  \n",
       "\n",
       "[10 rows x 101 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bow = pd.read_csv('bow.csv', index_col=0)\n",
    "bow.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the BoW with TF IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>aaai</th>\n",
       "      <th>abstract</th>\n",
       "      <th>acm</th>\n",
       "      <th>adapt</th>\n",
       "      <th>al</th>\n",
       "      <th>algorithm</th>\n",
       "      <th>analysi</th>\n",
       "      <th>annual</th>\n",
       "      <th>applic</th>\n",
       "      <th>...</th>\n",
       "      <th>th</th>\n",
       "      <th>theori</th>\n",
       "      <th>thi</th>\n",
       "      <th>univers</th>\n",
       "      <th>use</th>\n",
       "      <th>user</th>\n",
       "      <th>verlag</th>\n",
       "      <th>vol</th>\n",
       "      <th>workshop</th>\n",
       "      <th>www</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CBR</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.020676</td>\n",
       "      <td>0.039820</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.034163</td>\n",
       "      <td>0.036633</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.029141</td>\n",
       "      <td>0.067765</td>\n",
       "      <td>0.023532</td>\n",
       "      <td>0.024078</td>\n",
       "      <td>0.045838</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.034734</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.111672</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CBR</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.037334</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.085438</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.054762</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.042491</td>\n",
       "      <td>0.043478</td>\n",
       "      <td>0.124154</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CBR</td>\n",
       "      <td>0.047975</td>\n",
       "      <td>0.048885</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.129920</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.071705</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.142324</td>\n",
       "      <td>0.054189</td>\n",
       "      <td>0.295068</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.153873</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CBR</td>\n",
       "      <td>0.099534</td>\n",
       "      <td>0.076067</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.041552</td>\n",
       "      <td>0.028858</td>\n",
       "      <td>0.118113</td>\n",
       "      <td>0.084320</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CBR</td>\n",
       "      <td>0.033174</td>\n",
       "      <td>0.033804</td>\n",
       "      <td>0.032552</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.116040</td>\n",
       "      <td>0.083782</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.074376</td>\n",
       "      <td>...</td>\n",
       "      <td>0.023822</td>\n",
       "      <td>0.027698</td>\n",
       "      <td>0.019237</td>\n",
       "      <td>0.059051</td>\n",
       "      <td>0.093679</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.028394</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.045645</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>CBR</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.027591</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.040471</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.032132</td>\n",
       "      <td>0.030585</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.057899</td>\n",
       "      <td>0.074511</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>CBR</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.038497</td>\n",
       "      <td>0.037072</td>\n",
       "      <td>0.079558</td>\n",
       "      <td>0.044050</td>\n",
       "      <td>0.031804</td>\n",
       "      <td>0.034104</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.021907</td>\n",
       "      <td>0.044832</td>\n",
       "      <td>0.064011</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.025991</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>CBR</td>\n",
       "      <td>0.047006</td>\n",
       "      <td>0.023949</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.346452</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.042432</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.035129</td>\n",
       "      <td>...</td>\n",
       "      <td>0.067509</td>\n",
       "      <td>0.039247</td>\n",
       "      <td>0.027257</td>\n",
       "      <td>0.055780</td>\n",
       "      <td>0.079642</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.050256</td>\n",
       "      <td>0.032338</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>CBR</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.053205</td>\n",
       "      <td>0.102468</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.121757</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.078041</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.242214</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.058977</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.111646</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>CBR</td>\n",
       "      <td>0.039056</td>\n",
       "      <td>0.019899</td>\n",
       "      <td>0.038323</td>\n",
       "      <td>0.082245</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.032878</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.204312</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.045294</td>\n",
       "      <td>0.046346</td>\n",
       "      <td>0.110288</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.033428</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.134342</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 101 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Class      aaai  abstract       acm     adapt        al  algorithm  \\\n",
       "0   CBR  0.000000  0.020676  0.039820  0.000000  0.000000   0.034163   \n",
       "1   CBR  0.000000  0.037334  0.000000  0.000000  0.085438   0.000000   \n",
       "2   CBR  0.047975  0.048885  0.000000  0.000000  0.000000   0.000000   \n",
       "3   CBR  0.099534  0.076067  0.000000  0.000000  0.000000   0.000000   \n",
       "4   CBR  0.033174  0.033804  0.032552  0.000000  0.116040   0.083782   \n",
       "5   CBR  0.000000  0.027591  0.000000  0.000000  0.000000   0.000000   \n",
       "6   CBR  0.000000  0.038497  0.037072  0.079558  0.044050   0.031804   \n",
       "7   CBR  0.047006  0.023949  0.000000  0.346452  0.000000   0.000000   \n",
       "8   CBR  0.000000  0.053205  0.102468  0.000000  0.121757   0.000000   \n",
       "9   CBR  0.039056  0.019899  0.038323  0.082245  0.000000   0.032878   \n",
       "\n",
       "    analysi  annual    applic ...         th    theori       thi   univers  \\\n",
       "0  0.036633     0.0  0.000000 ...   0.029141  0.067765  0.023532  0.024078   \n",
       "1  0.000000     0.0  0.054762 ...   0.000000  0.000000  0.042491  0.043478   \n",
       "2  0.129920     0.0  0.071705 ...   0.000000  0.000000  0.000000  0.142324   \n",
       "3  0.000000     0.0  0.000000 ...   0.000000  0.041552  0.028858  0.118113   \n",
       "4  0.000000     0.0  0.074376 ...   0.023822  0.027698  0.019237  0.059051   \n",
       "5  0.000000     0.0  0.040471 ...   0.000000  0.000000  0.000000  0.032132   \n",
       "6  0.034104     0.0  0.000000 ...   0.000000  0.000000  0.021907  0.044832   \n",
       "7  0.042432     0.0  0.035129 ...   0.067509  0.039247  0.027257  0.055780   \n",
       "8  0.000000     0.0  0.078041 ...   0.000000  0.000000  0.242214  0.000000   \n",
       "9  0.000000     0.0  0.204312 ...   0.000000  0.000000  0.045294  0.046346   \n",
       "\n",
       "        use      user    verlag       vol  workshop  www  \n",
       "0  0.045838  0.000000  0.034734  0.000000  0.111672  0.0  \n",
       "1  0.124154  0.000000  0.000000  0.000000  0.000000  0.0  \n",
       "2  0.054189  0.295068  0.000000  0.153873  0.000000  0.0  \n",
       "3  0.084320  0.000000  0.000000  0.000000  0.000000  0.0  \n",
       "4  0.093679  0.000000  0.028394  0.000000  0.045645  0.0  \n",
       "5  0.030585  0.000000  0.000000  0.057899  0.074511  0.0  \n",
       "6  0.064011  0.000000  0.000000  0.000000  0.025991  0.0  \n",
       "7  0.079642  0.000000  0.000000  0.050256  0.032338  0.0  \n",
       "8  0.058977  0.000000  0.000000  0.111646  0.000000  0.0  \n",
       "9  0.110288  0.000000  0.033428  0.000000  0.134342  0.0  \n",
       "\n",
       "[10 rows x 101 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bow_tfidf = pd.read_csv('bow_tfidf.csv', index_col=0)\n",
    "bow_tfidf.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## P(Class) calculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'CBR': 0.4808362369337979,\n",
       " 'ILP': 0.2073170731707317,\n",
       " 'RI': 0.3118466898954704}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculating number of documents of each class\n",
    "n_cbr = bow.loc[bow['Class'] == 'CBR'].shape[0]\n",
    "n_ilp = bow.loc[bow['Class'] == 'ILP'].shape[0]\n",
    "n_ri = bow.loc[bow['Class'] == 'RI'].shape[0]\n",
    "\n",
    "# Calculating P(Class)\n",
    "p_class = dict()\n",
    "p_class['CBR'] = n_cbr/bow.shape[0]\n",
    "p_class['ILP'] = n_ilp/bow.shape[0]\n",
    "p_class['RI'] = n_ri/bow.shape[0]\n",
    "\n",
    "p_class"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Round: 0\n",
      "Calculating CBR probabilities\n",
      "Calculating ILP probabilities\n",
      "Calculating RI probabilities\n",
      "Round: 1\n",
      "Calculating CBR probabilities\n",
      "Calculating ILP probabilities\n",
      "Calculating RI probabilities\n",
      "Round: 2\n",
      "Calculating CBR probabilities\n",
      "Calculating ILP probabilities\n",
      "Calculating RI probabilities\n",
      "Round: 3\n",
      "Calculating CBR probabilities\n",
      "Calculating ILP probabilities\n",
      "Calculating RI probabilities\n",
      "Round: 4\n",
      "Calculating CBR probabilities\n",
      "Calculating ILP probabilities\n",
      "Calculating RI probabilities\n",
      "Round: 5\n",
      "Calculating CBR probabilities\n",
      "Calculating ILP probabilities\n",
      "Calculating RI probabilities\n",
      "Round: 6\n",
      "Calculating CBR probabilities\n",
      "Calculating ILP probabilities\n",
      "Calculating RI probabilities\n",
      "Round: 7\n",
      "Calculating CBR probabilities\n",
      "Calculating ILP probabilities\n",
      "Calculating RI probabilities\n",
      "Round: 8\n",
      "Calculating CBR probabilities\n",
      "Calculating ILP probabilities\n",
      "Calculating RI probabilities\n",
      "Round: 9\n",
      "Calculating CBR probabilities\n",
      "Calculating ILP probabilities\n",
      "Calculating RI probabilities\n",
      "Train probs:    Class      aaai  abstract       acm     adapt        al  algorithm  \\\n",
      "0    CBR  0.008278  0.008118  0.002821  0.012218  0.004658   0.006628   \n",
      "1    ILP  0.003459  0.008224  0.002807  0.000587  0.001110   0.012858   \n",
      "2     RI  0.002975  0.006927  0.023416  0.001487  0.004292   0.013557   \n",
      "3    CBR  0.008750  0.007758  0.003004  0.013217  0.005329   0.006478   \n",
      "4    ILP  0.003699  0.008339  0.002488  0.000673  0.001211   0.012778   \n",
      "5     RI  0.002833  0.007147  0.023471  0.001692  0.003045   0.014252   \n",
      "6    CBR  0.008152  0.008179  0.002806  0.012414  0.005109   0.006591   \n",
      "7    ILP  0.003401  0.008335  0.003001  0.000600  0.001067   0.012869   \n",
      "8     RI  0.003069  0.007102  0.022884  0.001622  0.004296   0.014905   \n",
      "9    CBR  0.008007  0.008115  0.002768  0.012736  0.005374   0.005858   \n",
      "10   ILP  0.003541  0.008401  0.003055  0.000694  0.001250   0.012359   \n",
      "11    RI  0.003206  0.007278  0.022571  0.001690  0.003769   0.013776   \n",
      "12   CBR  0.008397  0.008235  0.002889  0.012015  0.004914   0.006318   \n",
      "13   ILP  0.003667  0.008041  0.002637  0.000579  0.001094   0.012415   \n",
      "14    RI  0.003130  0.006979  0.022967  0.001565  0.003976   0.012816   \n",
      "15   CBR  0.008448  0.008186  0.002965  0.012278  0.005063   0.006559   \n",
      "16   ILP  0.003984  0.008103  0.002903  0.000608  0.001148   0.013032   \n",
      "17    RI  0.002898  0.006983  0.022660  0.001537  0.004567   0.014141   \n",
      "18   CBR  0.008376  0.008117  0.002740  0.013339  0.004860   0.006463   \n",
      "19   ILP  0.003448  0.008413  0.002896  0.000621  0.001241   0.012137   \n",
      "20    RI  0.002981  0.007298  0.023228  0.001513  0.004005   0.014106   \n",
      "21   CBR  0.008086  0.008217  0.002993  0.012811  0.004936   0.006432   \n",
      "22   ILP  0.003364  0.008021  0.002846  0.000647  0.001164   0.013196   \n",
      "23    RI  0.003107  0.007249  0.022153  0.001576  0.004458   0.012292   \n",
      "24   CBR  0.008232  0.008153  0.002948  0.012800  0.005152   0.006373   \n",
      "25   ILP  0.003698  0.008473  0.002690  0.000471  0.001210   0.012777   \n",
      "26    RI  0.003315  0.006885  0.023118  0.001572  0.004292   0.014449   \n",
      "27   CBR  0.008234  0.007992  0.002691  0.012970  0.004924   0.006189   \n",
      "28   ILP  0.003656  0.008194  0.002836  0.000567  0.000945   0.012290   \n",
      "29    RI  0.002990  0.007300  0.022384  0.001407  0.004398   0.014776   \n",
      "\n",
      "     analysi    annual    applic    ...           th    theori       thi  \\\n",
      "0   0.004631  0.002289  0.008731    ...     0.009210  0.004525  0.010248   \n",
      "1   0.003916  0.001305  0.007898    ...     0.017819  0.015730  0.010313   \n",
      "2   0.013089  0.011304  0.007097    ...     0.018529  0.011857  0.011474   \n",
      "3   0.004676  0.002194  0.008463    ...     0.008777  0.004414  0.010187   \n",
      "4   0.003497  0.001412  0.008205    ...     0.018495  0.015670  0.009752   \n",
      "5   0.012137  0.011757  0.007147    ...     0.019580  0.012856  0.011545   \n",
      "6   0.004685  0.001959  0.008576    ...     0.009211  0.004526  0.009952   \n",
      "7   0.003734  0.001067  0.008202    ...     0.018137  0.014936  0.010335   \n",
      "8   0.013239  0.010434  0.007628    ...     0.019026  0.012845  0.011705   \n",
      "9   0.004380  0.002096  0.009163    ...     0.008787  0.004622  0.010264   \n",
      "10  0.003471  0.001250  0.008193    ...     0.018260  0.015552  0.010623   \n",
      "11  0.012390  0.011437  0.006975    ...     0.019192  0.012130  0.011870   \n",
      "12  0.004644  0.002025  0.008721    ...     0.008424  0.004806  0.010233   \n",
      "13  0.003281  0.001287  0.007397    ...     0.018912  0.015245  0.009970   \n",
      "14  0.012266  0.010955  0.007402    ...     0.018991  0.011716  0.011885   \n",
      "15  0.004932  0.002151  0.008579    ...     0.008632  0.004617  0.010153   \n",
      "16  0.003714  0.001013  0.008238    ...     0.017623  0.015395  0.010601   \n",
      "17  0.013482  0.011242  0.007466    ...     0.017522  0.011857  0.011725   \n",
      "18  0.004550  0.002171  0.008867    ...     0.008764  0.004602  0.010237   \n",
      "19  0.003862  0.001310  0.008413    ...     0.018413  0.015447  0.010482   \n",
      "20  0.013349  0.010902  0.007075    ...     0.018600  0.012815  0.011525   \n",
      "21  0.004384  0.002100  0.008637    ...     0.008716  0.004279  0.010002   \n",
      "22  0.003105  0.001358  0.008280    ...     0.018695  0.015202  0.010091   \n",
      "23  0.012968  0.011167  0.007249    ...     0.018776  0.010671  0.011887   \n",
      "24  0.004860  0.002204  0.008764    ...     0.009003  0.004700  0.010251   \n",
      "25  0.003698  0.001278  0.008338    ...     0.018761  0.014525  0.009885   \n",
      "26  0.012749  0.011049  0.007140    ...     0.018529  0.012664  0.012112   \n",
      "27  0.004763  0.002045  0.008772    ...     0.008933  0.004467  0.010279   \n",
      "28  0.003656  0.001324  0.008194    ...     0.018908  0.015316  0.010211   \n",
      "29  0.012929  0.009895  0.007520    ...     0.018339  0.012841  0.012270   \n",
      "\n",
      "     univers       use      user    verlag       vol  workshop       www  \n",
      "0   0.011818  0.016476  0.004046  0.009795  0.005457  0.014693  0.001251  \n",
      "1   0.015012  0.018276  0.000783  0.011553  0.005548  0.014882  0.001566  \n",
      "2   0.013174  0.022056  0.006332  0.003825  0.007607  0.007352  0.011899  \n",
      "3   0.011806  0.016143  0.004205  0.009952  0.005224  0.014497  0.001175  \n",
      "4   0.014796  0.017957  0.000538  0.011299  0.005044  0.015132  0.001614  \n",
      "5   0.013195  0.021188  0.005752  0.004060  0.006809  0.007612  0.011461  \n",
      "6   0.011805  0.016358  0.004129  0.010349  0.005029  0.014452  0.001297  \n",
      "7   0.014203  0.018404  0.000800  0.011469  0.004801  0.014270  0.001734  \n",
      "8   0.013152  0.022182  0.006094  0.004077  0.007146  0.007847  0.012143  \n",
      "9   0.011903  0.016552  0.003842  0.009566  0.005589  0.014241  0.001209  \n",
      "10  0.014789  0.017705  0.000833  0.010831  0.005902  0.014441  0.001666  \n",
      "11  0.013776  0.021834  0.005675  0.003466  0.006542  0.007755  0.011870  \n",
      "12  0.011826  0.016632  0.004050  0.009882  0.005211  0.014229  0.001350  \n",
      "13  0.014345  0.018461  0.000708  0.011000  0.004696  0.015245  0.001672  \n",
      "14  0.012604  0.021825  0.006471  0.003976  0.007656  0.007571  0.012054  \n",
      "15  0.011963  0.016607  0.003883  0.009786  0.005247  0.014298  0.001207  \n",
      "16  0.015192  0.018771  0.000878  0.011546  0.005739  0.013707  0.001958  \n",
      "17  0.013262  0.022529  0.005973  0.003689  0.007070  0.008080  0.012516  \n",
      "18  0.011530  0.016545  0.003852  0.009901  0.004834  0.014347  0.001241  \n",
      "19  0.014206  0.018964  0.000828  0.011241  0.005655  0.015516  0.001793  \n",
      "20  0.013127  0.022382  0.006141  0.003916  0.006986  0.007876  0.012415  \n",
      "21  0.011945  0.016434  0.004043  0.010081  0.005356  0.014387  0.001286  \n",
      "22  0.015007  0.018371  0.000841  0.010997  0.005304  0.015202  0.001747  \n",
      "23  0.012878  0.022694  0.006709  0.003197  0.007474  0.007384  0.012698  \n",
      "24  0.011711  0.016598  0.004090  0.009666  0.005152  0.013783  0.001115  \n",
      "25  0.015063  0.018896  0.000807  0.011297  0.005111  0.014794  0.001883  \n",
      "26  0.013174  0.022481  0.006120  0.004080  0.007267  0.007649  0.005652  \n",
      "27  0.011463  0.016871  0.004171  0.009956  0.005382  0.014342  0.001292  \n",
      "28  0.015001  0.018845  0.000819  0.011156  0.004916  0.015001  0.001765  \n",
      "29  0.013721  0.022384  0.006113  0.004002  0.007344  0.007476  0.011962  \n",
      "\n",
      "[30 rows x 101 columns]\n",
      "Predicted labels:     0    1    2    3    4    5    6    7    8    9  ...    48   49   50   51  \\\n",
      "0  ILP  CBR  ILP  CBR  ILP   RI  ILP   RI   RI  ILP ...   CBR  CBR  CBR  ILP   \n",
      "1  CBR  CBR   RI  CBR  CBR  CBR   RI  ILP   RI  CBR ...   ILP  CBR  CBR   RI   \n",
      "2  ILP  CBR   RI   RI   RI   RI   RI  CBR  ILP  CBR ...   CBR  CBR   RI   RI   \n",
      "3   RI  CBR  CBR  CBR  CBR  CBR   RI  CBR   RI   RI ...    RI  ILP  CBR  CBR   \n",
      "4  CBR  CBR  ILP   RI   RI   RI   RI  ILP   RI  CBR ...   CBR  CBR   RI  CBR   \n",
      "5  CBR  CBR   RI   RI  CBR  ILP  ILP   RI  ILP  CBR ...   CBR  CBR   RI   RI   \n",
      "6  CBR   RI   RI  CBR  CBR  CBR  CBR   RI  CBR  CBR ...   CBR   RI  ILP   RI   \n",
      "7  CBR   RI  CBR  ILP  ILP  ILP  CBR  ILP   RI   RI ...    RI  CBR  CBR   RI   \n",
      "8  CBR  CBR  ILP  CBR   RI   RI  CBR  CBR   RI   RI ...    RI  CBR   RI  CBR   \n",
      "9  CBR  CBR   RI   RI  CBR  CBR  CBR  CBR  CBR  CBR ...    RI   RI  CBR   RI   \n",
      "\n",
      "    52   53   54   55   56   57  \n",
      "0   RI  CBR   RI  CBR   RI   RI  \n",
      "1  CBR  CBR  ILP  ILP  ILP   RI  \n",
      "2   RI  CBR  CBR  CBR   RI   RI  \n",
      "3  CBR  ILP  ILP  CBR  CBR   RI  \n",
      "4  CBR   RI  ILP  CBR   RI  NaN  \n",
      "5  CBR   RI  CBR  ILP  CBR  NaN  \n",
      "6  CBR   RI  CBR   RI   RI  NaN  \n",
      "7   RI   RI   RI  CBR   RI  NaN  \n",
      "8  CBR  ILP  CBR   RI  ILP  NaN  \n",
      "9   RI  CBR  CBR  CBR  ILP  NaN  \n",
      "\n",
      "[10 rows x 58 columns]\n",
      "True labels:     0    1    2    3    4    5    6    7    8    9  ...    48   49   50   51  \\\n",
      "0  ILP  CBR  ILP  CBR  ILP   RI  ILP   RI   RI  ILP ...   CBR  CBR  CBR  ILP   \n",
      "1  CBR  CBR   RI  CBR  CBR  CBR   RI  ILP   RI  CBR ...   ILP  CBR  CBR   RI   \n",
      "2  CBR  CBR   RI   RI   RI   RI   RI  CBR  ILP  CBR ...   CBR  CBR   RI   RI   \n",
      "3   RI  CBR  CBR  CBR  CBR  CBR   RI  CBR   RI   RI ...    RI  ILP  CBR  CBR   \n",
      "4  CBR  CBR  ILP   RI   RI   RI   RI  ILP   RI  CBR ...   CBR  CBR   RI  CBR   \n",
      "5  CBR  CBR   RI   RI  CBR   RI  ILP   RI  ILP  CBR ...   CBR  CBR   RI   RI   \n",
      "6  CBR   RI   RI  CBR  CBR  CBR  CBR   RI  CBR  CBR ...   CBR   RI  ILP   RI   \n",
      "7  CBR   RI  CBR  ILP  ILP  ILP  CBR  ILP   RI   RI ...    RI  CBR  CBR   RI   \n",
      "8  CBR  CBR  ILP  CBR   RI   RI  CBR  CBR   RI   RI ...    RI  CBR   RI  CBR   \n",
      "9  CBR  CBR   RI   RI  CBR  CBR  CBR  CBR  CBR  CBR ...    RI   RI  CBR   RI   \n",
      "\n",
      "    52   53   54   55   56   57  \n",
      "0   RI  CBR   RI  CBR   RI   RI  \n",
      "1  CBR  CBR  ILP  ILP  ILP   RI  \n",
      "2   RI  CBR  CBR  CBR   RI  CBR  \n",
      "3  CBR  ILP  ILP  CBR  CBR   RI  \n",
      "4  CBR   RI  ILP  CBR   RI  NaN  \n",
      "5  CBR   RI  CBR  ILP  CBR  NaN  \n",
      "6  CBR   RI  CBR   RI   RI  NaN  \n",
      "7   RI   RI   RI  CBR   RI  NaN  \n",
      "8  CBR  ILP  CBR   RI  ILP  NaN  \n",
      "9   RI  CBR  CBR  CBR  ILP  NaN  \n",
      "\n",
      "[10 rows x 58 columns]\n"
     ]
    }
   ],
   "source": [
    "# We need to shuffle the DataFrame because the split() function returns consecutive indexes, like\n",
    "# range(0, 50). If the DataFrame is ordered, we might receive folds with only 1 of the classes.\n",
    "rd = bow.sample(frac=1)\n",
    "train_rounds, pred_rounds, true_rounds = naive_bayes(rd, p_class)\n",
    "print(\"Train probs:\", train_rounds)\n",
    "print(\"Predicted labels:\", pred_rounds)\n",
    "print(\"True labels:\", true_rounds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving the results in CSV\n",
    "pred_rounds.to_csv('pred_rounds_bayes.csv')\n",
    "true_rounds.to_csv('true_rounds_bayes.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running Decision Tree, KNN and Perceptron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Eduardo\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.perceptron.Perceptron'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "rd = bow_tfidf.sample(frac=1)\n",
    "results = classify(rd)\n",
    "\n",
    "# Writing the results in CSV\n",
    "# Perceptron\n",
    "results[0][0].to_csv('pred_rounds_perceptron.csv')\n",
    "results[0][1].to_csv('true_rounds_perceptron.csv')\n",
    "\n",
    "# Decision tree\n",
    "results[1][0].to_csv('pred_rounds_dt.csv')\n",
    "results[1][1].to_csv('true_rounds_dt.csv')\n",
    "\n",
    "# 3NN\n",
    "results[2][0].to_csv('pred_rounds_3nn.csv')\n",
    "results[2][1].to_csv('true_rounds_3nn.csv')\n",
    "\n",
    "# 5NN\n",
    "results[3][0].to_csv('pred_rounds_5nn.csv')\n",
    "results[3][1].to_csv('true_rounds_5nn.csv')\n",
    "\n",
    "# 7NN\n",
    "results[4][0].to_csv('pred_rounds_7nn.csv')\n",
    "results[4][1].to_csv('true_rounds_7nn.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}