2020-07-05.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I want to do a quick pearson's chi squared independence test between the independent variables and dependent variable, just after still up to this point not producing much benefit from the last models. This should at least in a super rudimentary way help understand whether the variables are at all useful.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import xgboost as xgb\n",
    "from xgboost import XGBClassifier\n",
    "import datetime; import pytz\n",
    "import matplotlib.pyplot as plt\n",
    "# from scipy.special import softmax\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import train_test_split # (*arrays, **options)\n",
    "import numpy as np\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.preprocessing import OneHotEncoder, LabelEncoder, LabelBinarizer\n",
    "from sklearn.datasets import load_svmlight_file\n",
    "from joblib import dump, load\n",
    "import joblib\n",
    "import os\n",
    "from sklearn.metrics import confusion_matrix, mean_squared_error\n",
    "from sklearn.model_selection import KFold, train_test_split, GridSearchCV\n",
    "from sklearn.metrics import accuracy_score, balanced_accuracy_score\n",
    "import fresh.utils as fu\n",
    "\n",
    "from importlib import reload\n",
    "from collections import Counter\n",
    "from tqdm.notebook import tqdm\n",
    "import fresh.preproc.v1 as pv1\n",
    "import fresh.preproc.v2 as pv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "localdir = '/home/ec2-user/SageMaker/learn-citibike'  # sagemaker\n",
    "datadir = f'{localdir}/artifacts/2020-07-03T171842Z'\n",
    "\n",
    "train_loc = f'{datadir}/train.libsvm'\n",
    "\n",
    "# Convert the dtrain to numpy  ( nice advice from https://stackoverflow.com/a/40430328  )\n",
    "train_data = load_svmlight_file(train_loc)\n",
    "X_train = train_data[0].toarray()\n",
    "y_train = train_data[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "rawdatadir = f'{localdir}/local_datas'\n",
    "\n",
    "tripsdf = pd.read_csv(f'{rawdatadir}/2013-07 - Citi Bike trip data.csv'\n",
    "                     )\n",
    "stationsdf = pd.read_csv(f'{localdir}/datas/stations/stations-2018-12-04-c.csv',\n",
    "                        index_col=0)\n",
    "X, y, neighborhoods = fu.prepare_data(tripsdf, stationsdf)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chi2= 1.0724852071005921, p= 0.300384770390566, dof= 1\n",
      "Ho, our null hypothesis is the two variables are independent.\n",
      "p value > 0.05 so we likely cannot reject the null hypothesis \n",
      "=> independent\n",
      "\n",
      "\n",
      "chi2= 19.911071844194957, p= 8.11291182934436e-06, dof= 1\n",
      "Ho, our null hypothesis is the two variables are independent.\n",
      "p value <= 0.05 so can lean towards rejecting the null hypothesis.\n",
      "=> some dependency likely\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# For my own sanity checking... \n",
    "# gender vs handedness , example from https://en.wikipedia.org/wiki/Contingency_table\n",
    "# Also some inspiration on which statistic is good in my situation, \n",
    "#      from here , https://stackabuse.com/statistical-hypothesis-analysis-in-python-with-anovas-chi-square-and-pearson-correlation/\n",
    "# one more reference that helped me : https://www.ling.upenn.edu/~clight/chisquared.htm\n",
    "def chi_sq_test(table):\n",
    "    chi2, p, dof, expected = chi2_contingency(table)\n",
    "    print(f'chi2= {chi2}, p= {p}, dof= {dof}')\n",
    "    # print(expected)\n",
    "    print('Ho, our null hypothesis is the two variables are independent.')\n",
    "    if p <= 0.05:\n",
    "        print('p value <= 0.05 so can lean towards rejecting the null hypothesis.\\n'\n",
    "             '=> some dependency likely')\n",
    "    else:\n",
    "        print('p value > 0.05 so we likely cannot reject the null hypothesis \\n'\n",
    "             '=> independent')\n",
    "    print('\\n')\n",
    "    \n",
    "table = np.array([[43, 9], \n",
    "                  [44, 4]])\n",
    "chi_sq_test(table)\n",
    "\n",
    "\n",
    "table = np.array([[43, 40], \n",
    "                  [44, 4]])\n",
    "chi_sq_test(table)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chi2 960.8880653984245, p 5.473221121152702e-165, dof 56\n",
      "col_0  Chelsea  East Village  Garment District  Gramercy Park  \\\n",
      "row_0                                                           \n",
      "0           69           123                25             49   \n",
      "1           36           128                49             44   \n",
      "2           89           118                83             42   \n",
      "3           84           268               109            105   \n",
      "4            9            61                28             38   \n",
      "\n",
      "col_0  Lower East Side  Midtown  Midtown East  Midtown West  Murray Hill  \\\n",
      "row_0                                                                      \n",
      "0                   12      136           612           407          235   \n",
      "1                   22      198           318           282          144   \n",
      "2                   20      206           296           199          112   \n",
      "3                   70      175           536           264          261   \n",
      "4                   28       34           130            48           63   \n",
      "\n",
      "col_0  Nolita  Rose Hill  Stuyvesant Town  Theater District  \\\n",
      "row_0                                                         \n",
      "0          88         64               51               342   \n",
      "1         130         54               89               160   \n",
      "2         126         51               96               132   \n",
      "3         188        116              314               219   \n",
      "4          69         32               84                78   \n",
      "\n",
      "col_0  Ukrainian Village  Union Square  \n",
      "row_0                                   \n",
      "0                     14           154  \n",
      "1                     44           171  \n",
      "2                     80           170  \n",
      "3                    189           248  \n",
      "4                     46            36  \n",
      "[[ 68.3347 166.1938  70.0014  66.1918  36.1912 178.3369 450.4852 285.72\n",
      "  194.0515 143.0981  75.4777 150.9554 221.6711  88.8113 185.4799]\n",
      " [ 53.6403 130.4562  54.9486  51.9582  28.4088 139.9881 353.6148 224.28\n",
      "  152.3235 112.3269  59.2473 118.4946 174.0039  69.7137 145.5951]\n",
      " [ 52.234  127.036   53.508   50.596   27.664  136.318  344.344  218.4\n",
      "  148.33   109.382   57.694  115.388  169.442   67.886  141.778 ]\n",
      " [ 90.2902 219.5908  92.4924  87.4588  47.8192 235.6354 595.2232 377.52\n",
      "  256.399  189.0746  99.7282 199.4564 292.8926 117.3458 245.0734]\n",
      " [ 22.5008  54.7232  23.0496  21.7952  11.9168  58.7216 148.3328  94.08\n",
      "   63.896   47.1184  24.8528  49.7056  72.9904  29.2432  61.0736]]\n"
     ]
    }
   ],
   "source": [
    "# Some quick playing around first ...  using column 2 which is the \"time of day\" feature.\n",
    "indices = np.random.choice(range(100000), replace=False, size=10000)\n",
    "# X[0] # array(['Midtown East', 0, 4, 'Customer', 1], dtype=object)\n",
    "table = pd.crosstab(X[indices, 2], y[indices])\n",
    "\n",
    "#X[indices, 4]#.shape, y[indices].shape\n",
    "#Counter(X[indices, 2])\n",
    "# from scipy.stats import chi2_contingency\n",
    "\n",
    "chi2, p, dof, expected = chi2_contingency(table)\n",
    "print(f'chi2 {chi2}, p {p}, dof {dof}')\n",
    "print(table)\n",
    "print(expected)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>col_0</th>\n",
       "      <th>Chelsea</th>\n",
       "      <th>East Village</th>\n",
       "      <th>Garment District</th>\n",
       "      <th>Gramercy Park</th>\n",
       "      <th>Lower East Side</th>\n",
       "      <th>Midtown</th>\n",
       "      <th>Midtown East</th>\n",
       "      <th>Midtown West</th>\n",
       "      <th>Murray Hill</th>\n",
       "      <th>Nolita</th>\n",
       "      <th>Rose Hill</th>\n",
       "      <th>Stuyvesant Town</th>\n",
       "      <th>Theater District</th>\n",
       "      <th>Ukrainian Village</th>\n",
       "      <th>Union Square</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row_0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>59</td>\n",
       "      <td>124</td>\n",
       "      <td>23</td>\n",
       "      <td>51</td>\n",
       "      <td>11</td>\n",
       "      <td>161</td>\n",
       "      <td>594</td>\n",
       "      <td>435</td>\n",
       "      <td>214</td>\n",
       "      <td>104</td>\n",
       "      <td>65</td>\n",
       "      <td>48</td>\n",
       "      <td>387</td>\n",
       "      <td>16</td>\n",
       "      <td>166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>40</td>\n",
       "      <td>112</td>\n",
       "      <td>40</td>\n",
       "      <td>50</td>\n",
       "      <td>20</td>\n",
       "      <td>178</td>\n",
       "      <td>348</td>\n",
       "      <td>238</td>\n",
       "      <td>143</td>\n",
       "      <td>103</td>\n",
       "      <td>59</td>\n",
       "      <td>96</td>\n",
       "      <td>169</td>\n",
       "      <td>59</td>\n",
       "      <td>194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>78</td>\n",
       "      <td>120</td>\n",
       "      <td>71</td>\n",
       "      <td>68</td>\n",
       "      <td>28</td>\n",
       "      <td>161</td>\n",
       "      <td>294</td>\n",
       "      <td>196</td>\n",
       "      <td>113</td>\n",
       "      <td>106</td>\n",
       "      <td>54</td>\n",
       "      <td>98</td>\n",
       "      <td>153</td>\n",
       "      <td>70</td>\n",
       "      <td>171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101</td>\n",
       "      <td>248</td>\n",
       "      <td>114</td>\n",
       "      <td>117</td>\n",
       "      <td>80</td>\n",
       "      <td>185</td>\n",
       "      <td>511</td>\n",
       "      <td>258</td>\n",
       "      <td>240</td>\n",
       "      <td>209</td>\n",
       "      <td>124</td>\n",
       "      <td>304</td>\n",
       "      <td>207</td>\n",
       "      <td>188</td>\n",
       "      <td>257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9</td>\n",
       "      <td>64</td>\n",
       "      <td>34</td>\n",
       "      <td>34</td>\n",
       "      <td>26</td>\n",
       "      <td>30</td>\n",
       "      <td>122</td>\n",
       "      <td>70</td>\n",
       "      <td>61</td>\n",
       "      <td>57</td>\n",
       "      <td>33</td>\n",
       "      <td>81</td>\n",
       "      <td>64</td>\n",
       "      <td>42</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "col_0  Chelsea  East Village  Garment District  Gramercy Park  \\\n",
       "row_0                                                           \n",
       "0           59           124                23             51   \n",
       "1           40           112                40             50   \n",
       "2           78           120                71             68   \n",
       "3          101           248               114            117   \n",
       "4            9            64                34             34   \n",
       "\n",
       "col_0  Lower East Side  Midtown  Midtown East  Midtown West  Murray Hill  \\\n",
       "row_0                                                                      \n",
       "0                   11      161           594           435          214   \n",
       "1                   20      178           348           238          143   \n",
       "2                   28      161           294           196          113   \n",
       "3                   80      185           511           258          240   \n",
       "4                   26       30           122            70           61   \n",
       "\n",
       "col_0  Nolita  Rose Hill  Stuyvesant Town  Theater District  \\\n",
       "row_0                                                         \n",
       "0         104         65               48               387   \n",
       "1         103         59               96               169   \n",
       "2         106         54               98               153   \n",
       "3         209        124              304               207   \n",
       "4          57         33               81                64   \n",
       "\n",
       "col_0  Ukrainian Village  Union Square  \n",
       "row_0                                   \n",
       "0                     16           166  \n",
       "1                     59           194  \n",
       "2                     70           171  \n",
       "3                    188           257  \n",
       "4                     42            42  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================================================\n",
      "Building crosstab on start_neighborhood\n",
      "chi2= 1101939.4755814166, p= 0.0, dof= 2809\n",
      "Ho, our null hypothesis is the two variables are independent.\n",
      "p value <= 0.05 so can lean towards rejecting the null hypothesis.\n",
      "=> some dependency likely\n",
      "\n",
      "\n",
      "====================================================\n",
      "Building crosstab on gender\n",
      "chi2= 28181.371173591728, p= 0.0, dof= 106\n",
      "Ho, our null hypothesis is the two variables are independent.\n",
      "p value <= 0.05 so can lean towards rejecting the null hypothesis.\n",
      "=> some dependency likely\n",
      "\n",
      "\n",
      "====================================================\n",
      "Building crosstab on time_of_day\n",
      "chi2= 43677.30494407689, p= 0.0, dof= 212\n",
      "Ho, our null hypothesis is the two variables are independent.\n",
      "p value <= 0.05 so can lean towards rejecting the null hypothesis.\n",
      "=> some dependency likely\n",
      "\n",
      "\n",
      "====================================================\n",
      "Building crosstab on usertype\n",
      "chi2= 23638.087048050013, p= 0.0, dof= 53\n",
      "Ho, our null hypothesis is the two variables are independent.\n",
      "p value <= 0.05 so can lean towards rejecting the null hypothesis.\n",
      "=> some dependency likely\n",
      "\n",
      "\n",
      "====================================================\n",
      "Building crosstab on weekday\n",
      "chi2= 9183.135943620315, p= 0.0, dof= 53\n",
      "Ho, our null hypothesis is the two variables are independent.\n",
      "p value <= 0.05 so can lean towards rejecting the null hypothesis.\n",
      "=> some dependency likely\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Ok.. got for all of it... compare independent variable  at a time.. \n",
    "# indices = np.random.choice(range(100000), replace=False, size=10000)\n",
    "# X[0] # array(['Midtown East', 0, 4, 'Customer', 1], dtype=object)\n",
    "independent_variables = {0: 'start_neighborhood',\n",
    "                        1: 'gender',\n",
    "                        2: 'time_of_day',\n",
    "                        3: 'usertype',\n",
    "                        4: 'weekday'}\n",
    "for col_index, var in independent_variables.items():\n",
    "    # Build a cross-tab \n",
    "    print('====================================================')\n",
    "    print(f'Building crosstab on {var}')\n",
    "    table = pd.crosstab(X[:, col_index], y)\n",
    "    #print(table)\n",
    "    chi_sq_test(table)\n",
    "    #print()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There can be much more to be said and done here, but for now I think I will casually use this as proof that I should keep going ! There's yet a model that can capture the entropy in this data!."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}