2020-06-12.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import xgboost as xgb\n",
    "from xgboost import XGBClassifier\n",
    "import datetime; import pytz\n",
    "import matplotlib as plt\n",
    "from scipy.special import softmax\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import train_test_split # (*arrays, **options)\n",
    "import numpy as np\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.preprocessing import OneHotEncoder, LabelEncoder\n",
    "\n",
    "from joblib import dump, load\n",
    "import joblib\n",
    "import os\n",
    "from sklearn.metrics import confusion_matrix, mean_squared_error\n",
    "from sklearn.model_selection import KFold, train_test_split, GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "datadir = '/opt/data'\n",
    "localdir = '/opt/program'\n",
    "tripsdf = pd.read_csv(f'{datadir}/2013-07 - Citi Bike trip data.csv')\n",
    "stationsdf = pd.read_csv(f'{localdir}/datas/stations/stations-2018-12-04-c.csv',\n",
    "                        index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/opt/program'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load model from 2020-06-10 notebook\n",
    "bundle = joblib.load('/opt/program/artifacts/2020-06-11T041641Z/bundle.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'notebook': '2020-06-10-again',\n",
       " 'model': XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "               colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
       "               learning_rate=0.1, max_delta_step=0, max_depth=3,\n",
       "               min_child_weight=1, missing=nan, n_estimators=100, n_jobs=1,\n",
       "               nthread=None, objective='multi:softprob', random_state=0,\n",
       "               reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "               silent=None, subsample=1, verbosity=1),\n",
       " 'actuals': array([60, 60, 60, ...,  2,  2,  2]),\n",
       " 'predictions': array([46, 46, 46, ...,  2,  2,  2]),\n",
       " 'confusion_matrix': array([[ 193,   64,   14, ...,    0,    0,  150],\n",
       "        [  27, 1167,   16, ...,    0,    0,   28],\n",
       "        [   4,    6,  379, ...,    0,    0,  236],\n",
       "        ...,\n",
       "        [   0,    0,   17, ...,    0,    0,   46],\n",
       "        [  67,  599,   24, ...,    0,    0,   78],\n",
       "        [  19,    9,  196, ...,    0,    0, 2017]]),\n",
       " 'walltime_train': '19min 45s',\n",
       " 'preproc': {'le': LabelEncoder(),\n",
       "  'one_hot_enc': OneHotEncoder(categories=[['Alphabet City', 'Battery Park City',\n",
       "                             'Bedford-Stuyvesant', 'Bloomingdale', 'Boerum Hill',\n",
       "                             'Bowery', 'Broadway Triangle', 'Brooklyn Heights',\n",
       "                             'Brooklyn Navy Yard', 'Carnegie Hill',\n",
       "                             'Carroll Gardens', 'Central Park', 'Chelsea',\n",
       "                             'Chinatown', 'Civic Center', 'Clinton Hill',\n",
       "                             'Cobble Hill', 'Columbia Street Waterfront District',\n",
       "                             'Downtown Brooklyn', 'Dumbo', 'East Harlem',\n",
       "                             'East Village', 'East Williamsburg',\n",
       "                             'Financial District', 'Flatiron District',\n",
       "                             'Fort Greene', 'Fulton Ferry District',\n",
       "                             'Garment District', 'Governors Island', 'Gowanus', ...],\n",
       "                            [0, 1, 2]],\n",
       "                drop=None, dtype=<class 'numpy.float64'>, handle_unknown='error',\n",
       "                sparse=True)}}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bundle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'overall accuracy': 0.1541161182619253, 'num_correct': 64992, 'total': 421708}\n",
      "(54, 54)\n",
      "[193, 1167, 379, 0, 0, 1406, 0, 1039, 30505, 0, 685, 557, 67, 62, 49, 0, 4534, 0, 2583, 109, 0, 0, 2766, 2003, 0, 0, 0, 236, 0, 2722, 0, 0, 612, 6064, 581, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 4632, 0, 0, 0, 0, 0, 0, 2017]\n",
      "confusion, sum, diagonal 64992\n"
     ]
    }
   ],
   "source": [
    "# quick manual accuracy computation\n",
    "bundle['actuals'].shape, bundle['predictions'].shape\n",
    "correct = len([i for i, _ in enumerate(bundle['actuals'])\n",
    "              if bundle['actuals'][i] == bundle['predictions'][i]\n",
    "              ])\n",
    "print({'overall accuracy': correct/len(bundle['actuals']), \n",
    "       'num_correct': correct, 'total': len(bundle['actuals'])})\n",
    "print(bundle['confusion_matrix'].shape)\n",
    "\n",
    "confusion_diagonal = [bundle['confusion_matrix'][i][j]\n",
    "                                  for i in range(54)\n",
    "                                  for j in range(54)\n",
    "                                  if i == j]\n",
    "print(confusion_diagonal)\n",
    "print('confusion, sum, diagonal', sum(confusion_diagonal))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "78\n",
      "[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00057195826, 0.00096085493, 0.0019978876, 0.0028608162, 0.003014325, 0.0031509278, 0.005542396, 0.0055443724, 0.0056226645, 0.0064963517, 0.0066661313, 0.007307959, 0.007578683, 0.0077737863, 0.008284873, 0.00866166, 0.009315344, 0.010423137, 0.010517005, 0.010585468, 0.010727769, 0.012014797, 0.012143584, 0.012632706, 0.01298739, 0.013684955, 0.014033192, 0.014308239, 0.014311185, 0.016608043, 0.017402876, 0.017734889, 0.017809916, 0.017948298, 0.018543119, 0.01898507, 0.019225849, 0.02069202, 0.020770036, 0.021067692, 0.02152036, 0.022673236, 0.024081945, 0.02418218, 0.02421692, 0.026871085, 0.027934289, 0.02844202, 0.030187974, 0.032351416, 0.033994444, 0.035514593, 0.03653184, 0.036576174, 0.044999287, 0.04772858, 0.05568539]\n"
     ]
    }
   ],
   "source": [
    "model = bundle['model']\n",
    "print(len(model.feature_importances_))\n",
    "print(sorted(model.feature_importances_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  1,  2,  4,  5,  7,  8, 11, 12, 13, 14, 15, 17, 18, 19, 21, 23,\n",
       "       24, 25, 26, 27, 30, 32, 33, 34, 36, 37, 39, 40, 42, 43, 44, 45, 46,\n",
       "       47, 48, 49, 50, 51, 52, 53, 57, 58, 60, 62, 63, 64, 65, 66, 67, 68,\n",
       "       71, 72, 73])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 78 features\n",
    "model.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'fresh.utils' from '/opt/program/fresh/utils.py'>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import fresh.utils as fu\n",
    "from importlib import reload\n",
    "reload(fu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y, neighborhoods = fu.prepare_data(tripsdf, stationsdf)\n",
    "# = sorted(stationsdf.neighborhood.unique().tolist())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75, 3)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(bundle['preproc']['one_hot_enc'].categories[0]), len(bundle['preproc']['one_hot_enc'].categories[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['feature_start_neighborhood=Alphabet City', 'feature_start_neighborhood=Battery Park City', 'feature_start_neighborhood=Bedford-Stuyvesant', 'feature_start_neighborhood=Bloomingdale', 'feature_start_neighborhood=Boerum Hill', 'feature_start_neighborhood=Bowery', 'feature_start_neighborhood=Broadway Triangle', 'feature_start_neighborhood=Brooklyn Heights', 'feature_start_neighborhood=Brooklyn Navy Yard', 'feature_start_neighborhood=Carnegie Hill', 'feature_start_neighborhood=Carroll Gardens', 'feature_start_neighborhood=Central Park', 'feature_start_neighborhood=Chelsea', 'feature_start_neighborhood=Chinatown', 'feature_start_neighborhood=Civic Center', 'feature_start_neighborhood=Clinton Hill', 'feature_start_neighborhood=Cobble Hill', 'feature_start_neighborhood=Columbia Street Waterfront District', 'feature_start_neighborhood=Downtown Brooklyn', 'feature_start_neighborhood=Dumbo', 'feature_start_neighborhood=East Harlem', 'feature_start_neighborhood=East Village', 'feature_start_neighborhood=East Williamsburg', 'feature_start_neighborhood=Financial District', 'feature_start_neighborhood=Flatiron District', 'feature_start_neighborhood=Fort Greene', 'feature_start_neighborhood=Fulton Ferry District', 'feature_start_neighborhood=Garment District', 'feature_start_neighborhood=Governors Island', 'feature_start_neighborhood=Gowanus', 'feature_start_neighborhood=Gramercy Park', 'feature_start_neighborhood=Greenpoint', 'feature_start_neighborhood=Greenwich Village', \"feature_start_neighborhood=Hell's Kitchen\", 'feature_start_neighborhood=Hudson Square', 'feature_start_neighborhood=Hunters Point', 'feature_start_neighborhood=Kips Bay', 'feature_start_neighborhood=Korea Town', 'feature_start_neighborhood=Lenox Hill', 'feature_start_neighborhood=Lincoln Square', 'feature_start_neighborhood=Little Italy', 'feature_start_neighborhood=Long Island City', 'feature_start_neighborhood=Lower East Side', 'feature_start_neighborhood=Lower Manhattan', 'feature_start_neighborhood=Meatpacking District', 'feature_start_neighborhood=Midtown', 'feature_start_neighborhood=Midtown East', 'feature_start_neighborhood=Midtown West', 'feature_start_neighborhood=Murray Hill', 'feature_start_neighborhood=NoHo', 'feature_start_neighborhood=NoMad', 'feature_start_neighborhood=Nolita', 'feature_start_neighborhood=Park Slope', 'feature_start_neighborhood=Peter Cooper Village', 'feature_start_neighborhood=Prospect Heights', 'feature_start_neighborhood=Prospect Park', 'feature_start_neighborhood=Red Hook', 'feature_start_neighborhood=Rose Hill', 'feature_start_neighborhood=SoHo', 'feature_start_neighborhood=Stuyvesant Heights', 'feature_start_neighborhood=Stuyvesant Town', 'feature_start_neighborhood=Sunset Park', 'feature_start_neighborhood=Sutton Place', 'feature_start_neighborhood=Theater District', 'feature_start_neighborhood=Tribeca', 'feature_start_neighborhood=Tudor City', 'feature_start_neighborhood=Two Bridges', 'feature_start_neighborhood=Ukrainian Village', 'feature_start_neighborhood=Union Square', 'feature_start_neighborhood=Upper East Side', 'feature_start_neighborhood=Upper West Side', 'feature_start_neighborhood=Vinegar Hill', 'feature_start_neighborhood=West Village', 'feature_start_neighborhood=Williamsburg', 'feature_start_neighborhood=Yorkville', 'feature_gender=0', 'feature_gender=1', 'feature_gender=2']\n"
     ]
    }
   ],
   "source": [
    "features = ([f'feature_start_neighborhood={x}' for x in bundle['preproc']['one_hot_enc'].categories[0]]\n",
    "           + [f'feature_gender={x}' for x in bundle['preproc']['one_hot_enc'].categories[1]])\n",
    "print(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['feature_start_neighborhood=Bloomingdale', 0.0],\n",
       " ['feature_start_neighborhood=Broadway Triangle', 0.0],\n",
       " ['feature_start_neighborhood=Carnegie Hill', 0.0],\n",
       " ['feature_start_neighborhood=Carroll Gardens', 0.0],\n",
       " ['feature_start_neighborhood=Cobble Hill', 0.0],\n",
       " ['feature_start_neighborhood=East Harlem', 0.0],\n",
       " ['feature_start_neighborhood=East Williamsburg', 0.0],\n",
       " ['feature_start_neighborhood=Governors Island', 0.0],\n",
       " ['feature_start_neighborhood=Gowanus', 0.0],\n",
       " ['feature_start_neighborhood=Greenpoint', 0.0],\n",
       " ['feature_start_neighborhood=Hunters Point', 0.0],\n",
       " ['feature_start_neighborhood=Lenox Hill', 0.0],\n",
       " ['feature_start_neighborhood=Long Island City', 0.0],\n",
       " ['feature_start_neighborhood=Prospect Heights', 0.0],\n",
       " ['feature_start_neighborhood=Prospect Park', 0.0],\n",
       " ['feature_start_neighborhood=Red Hook', 0.0],\n",
       " ['feature_start_neighborhood=Stuyvesant Heights', 0.0],\n",
       " ['feature_start_neighborhood=Sunset Park', 0.0],\n",
       " ['feature_start_neighborhood=Upper East Side', 0.0],\n",
       " ['feature_start_neighborhood=Upper West Side', 0.0],\n",
       " ['feature_start_neighborhood=Yorkville', 0.0],\n",
       " ['feature_gender=2', 0.00057195826],\n",
       " ['feature_gender=1', 0.00096085493],\n",
       " ['feature_start_neighborhood=Tudor City', 0.0019978876],\n",
       " ['feature_gender=0', 0.0028608162],\n",
       " ['feature_start_neighborhood=Korea Town', 0.003014325],\n",
       " ['feature_start_neighborhood=Sutton Place', 0.0031509278],\n",
       " ['feature_start_neighborhood=Rose Hill', 0.005542396],\n",
       " ['feature_start_neighborhood=NoHo', 0.0055443724],\n",
       " ['feature_start_neighborhood=NoMad', 0.0056226645],\n",
       " ['feature_start_neighborhood=Union Square', 0.0064963517],\n",
       " ['feature_start_neighborhood=Little Italy', 0.0066661313],\n",
       " ['feature_start_neighborhood=Gramercy Park', 0.007307959],\n",
       " ['feature_start_neighborhood=Peter Cooper Village', 0.007578683],\n",
       " ['feature_start_neighborhood=Flatiron District', 0.0077737863],\n",
       " ['feature_start_neighborhood=Nolita', 0.008284873],\n",
       " ['feature_start_neighborhood=Lower Manhattan', 0.00866166],\n",
       " ['feature_start_neighborhood=SoHo', 0.009315344],\n",
       " ['feature_start_neighborhood=Stuyvesant Town', 0.010423137],\n",
       " ['feature_start_neighborhood=Garment District', 0.010517005],\n",
       " ['feature_start_neighborhood=Meatpacking District', 0.010585468],\n",
       " ['feature_start_neighborhood=Murray Hill', 0.010727769],\n",
       " ['feature_start_neighborhood=Kips Bay', 0.012014797],\n",
       " ['feature_start_neighborhood=Hudson Square', 0.012143584],\n",
       " ['feature_start_neighborhood=Ukrainian Village', 0.012632706],\n",
       " ['feature_start_neighborhood=Midtown', 0.01298739],\n",
       " ['feature_start_neighborhood=East Village', 0.013684955],\n",
       " ['feature_start_neighborhood=Park Slope', 0.014033192],\n",
       " ['feature_start_neighborhood=Vinegar Hill', 0.014308239],\n",
       " ['feature_start_neighborhood=Bowery', 0.014311185],\n",
       " ['feature_start_neighborhood=Chinatown', 0.016608043],\n",
       " ['feature_start_neighborhood=Alphabet City', 0.017402876],\n",
       " ['feature_start_neighborhood=Lincoln Square', 0.017734889],\n",
       " ['feature_start_neighborhood=Dumbo', 0.017809916],\n",
       " ['feature_start_neighborhood=Brooklyn Navy Yard', 0.017948298],\n",
       " ['feature_start_neighborhood=Civic Center', 0.018543119],\n",
       " ['feature_start_neighborhood=Theater District', 0.01898507],\n",
       " ['feature_start_neighborhood=Fulton Ferry District', 0.019225849],\n",
       " ['feature_start_neighborhood=West Village', 0.02069202],\n",
       " ['feature_start_neighborhood=Midtown West', 0.020770036],\n",
       " ['feature_start_neighborhood=Midtown East', 0.021067692],\n",
       " ['feature_start_neighborhood=Boerum Hill', 0.02152036],\n",
       " ['feature_start_neighborhood=Two Bridges', 0.022673236],\n",
       " [\"feature_start_neighborhood=Hell's Kitchen\", 0.024081945],\n",
       " ['feature_start_neighborhood=Greenwich Village', 0.02418218],\n",
       " ['feature_start_neighborhood=Columbia Street Waterfront District',\n",
       "  0.02421692],\n",
       " ['feature_start_neighborhood=Battery Park City', 0.026871085],\n",
       " ['feature_start_neighborhood=Chelsea', 0.027934289],\n",
       " ['feature_start_neighborhood=Lower East Side', 0.02844202],\n",
       " ['feature_start_neighborhood=Downtown Brooklyn', 0.030187974],\n",
       " ['feature_start_neighborhood=Tribeca', 0.032351416],\n",
       " ['feature_start_neighborhood=Clinton Hill', 0.033994444],\n",
       " ['feature_start_neighborhood=Brooklyn Heights', 0.035514593],\n",
       " ['feature_start_neighborhood=Bedford-Stuyvesant', 0.03653184],\n",
       " ['feature_start_neighborhood=Financial District', 0.036576174],\n",
       " ['feature_start_neighborhood=Fort Greene', 0.044999287],\n",
       " ['feature_start_neighborhood=Williamsburg', 0.04772858],\n",
       " ['feature_start_neighborhood=Central Park', 0.05568539]]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "importances = model.feature_importances_\n",
    "labeled_importances = [[features[i], importances[i]] for i in range(78)]\n",
    "sorted(labeled_importances, key=lambda x:x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perhaps indeed the starting locations which are so un-important, basically have just way too many\n",
    "# destinations, or maybe there are just not enough trips involving those stations.\n",
    "# But anyway, I think ultimately the importances which are there \n",
    "#    across different cross validation folds are the ones to focus on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# retry that multi-logloss ...\n",
    "rng = np.random.RandomState(31337)\n",
    "indices1 = []\n",
    "kf = KFold(n_splits=2, shuffle=True, random_state=rng)\n",
    "for train_index, test_index in kf.split(X):\n",
    "    indices1.append([train_index, test_index])\n",
    "\n",
    "indices2 = []\n",
    "kf = KFold(n_splits=2, shuffle=True, random_state=rng)\n",
    "for train_index, test_index in kf.split(X):\n",
    "    indices2.append([train_index, test_index])\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([array([     0,      2,      3, ..., 843411, 843412, 843415]),\n",
       "  array([     1,      7,      8, ..., 843410, 843413, 843414])],\n",
       " [array([     5,      7,      9, ..., 843412, 843413, 843414]),\n",
       "  array([     0,      1,      2, ..., 843408, 843411, 843415])])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# hmm doesnt look the same though, guess that random seed doesnt work deterministically?\n",
    "indices1[0][:5] , indices2[0][:5]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75,)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#log_loss(y_true_enc, y_prob, labels=self.labels)\n",
    "#help(log_loss)\n",
    "bundle['preproc']['le'].classes_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_transformed = bundle['preproc']['one_hot_enc'].transform(X[:1000])\n",
    "y_prob = model.predict_proba(X_transformed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(54,)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_prob[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "75\n",
      "54\n"
     ]
    }
   ],
   "source": [
    "# ok got to clear up this inconsistency.. so, len(bundle['preproc']['one_hot_enc'].categories[0]) => 75\n",
    "# but there are I believe only 54 neighborhoods actually represented in the data itself,\n",
    "\n",
    "from collections import Counter\n",
    "print(len(bundle['preproc']['one_hot_enc'].categories[0]))\n",
    "print(len(dict(Counter([x[0] for x in X]))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Alphabet City',\n",
       " 'Battery Park City',\n",
       " 'Bedford-Stuyvesant',\n",
       " 'Boerum Hill',\n",
       " 'Bowery',\n",
       " 'Brooklyn Heights',\n",
       " 'Brooklyn Navy Yard',\n",
       " 'Central Park',\n",
       " 'Chelsea',\n",
       " 'Chinatown',\n",
       " 'Civic Center',\n",
       " 'Clinton Hill',\n",
       " 'Columbia Street Waterfront District',\n",
       " 'Downtown Brooklyn',\n",
       " 'Dumbo',\n",
       " 'East Village',\n",
       " 'Financial District',\n",
       " 'Flatiron District',\n",
       " 'Fort Greene',\n",
       " 'Fulton Ferry District',\n",
       " 'Garment District',\n",
       " 'Gramercy Park',\n",
       " 'Greenwich Village',\n",
       " \"Hell's Kitchen\",\n",
       " 'Hudson Square',\n",
       " 'Kips Bay',\n",
       " 'Korea Town',\n",
       " 'Lincoln Square',\n",
       " 'Little Italy',\n",
       " 'Lower East Side',\n",
       " 'Lower Manhattan',\n",
       " 'Meatpacking District',\n",
       " 'Midtown',\n",
       " 'Midtown East',\n",
       " 'Midtown West',\n",
       " 'Murray Hill',\n",
       " 'NoHo',\n",
       " 'NoMad',\n",
       " 'Nolita',\n",
       " 'Park Slope',\n",
       " 'Peter Cooper Village',\n",
       " 'Rose Hill',\n",
       " 'SoHo',\n",
       " 'Stuyvesant Town',\n",
       " 'Sutton Place',\n",
       " 'Theater District',\n",
       " 'Tribeca',\n",
       " 'Tudor City',\n",
       " 'Two Bridges',\n",
       " 'Ukrainian Village',\n",
       " 'Union Square',\n",
       " 'Vinegar Hill',\n",
       " 'West Village',\n",
       " 'Williamsburg']"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(list(dict(Counter(y)).keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# yea so 54 in X and y... but i used stationsdf w/ 75 neighborhoods, \n",
    "# so at the very least I should not be using that if it ends up indeed , \n",
    "# blowing up the one hot encoder to 75 , ... this might be why we have those start neighborhoods, \n",
    "# with  0 importance, since there was just no data for them \n",
    "\n",
    "# Anyhow since the predict_proba output is indeed only length 54, that tells me the classifier\n",
    "# ended up doing the rigiht thing anway. So I will keep this in mind for the next dataset but \n",
    "# in this case it is okay anyway."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020-06-13\n",
    "\n",
    "#### quick update, also just calculate a multiclass logloss as well\n",
    "- Hmm don't have the indices I used in notebook \"2020-06-10-again\" so I have to do that again\n",
    "- Also kernel died while trying to do this so numbers starting over \n",
    "- And wow happened twice. Turns out running log_loss() with 843416 rows did the trick!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import xgboost as xgb\n",
    "from xgboost import XGBClassifier\n",
    "import datetime; import pytz\n",
    "import matplotlib as plt\n",
    "from scipy.special import softmax\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import train_test_split # (*arrays, **options)\n",
    "import numpy as np\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.preprocessing import OneHotEncoder, LabelEncoder\n",
    "\n",
    "from joblib import dump, load\n",
    "import joblib\n",
    "import os\n",
    "from sklearn.metrics import confusion_matrix, mean_squared_error\n",
    "from sklearn.model_selection import KFold, train_test_split, GridSearchCV\n",
    "import fresh.utils as fu\n",
    "\n",
    "from importlib import reload\n",
    "from collections import Counter\n",
    "from tqdm.notebook import tqdm\n",
    "import fresh.preproc.v1 as pv1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "datadir = '/opt/data'\n",
    "localdir = '/opt/program'\n",
    "tripsdf = pd.read_csv(f'{datadir}/2013-07 - Citi Bike trip data.csv')\n",
    "stationsdf = pd.read_csv(f'{localdir}/datas/stations/stations-2018-12-04-c.csv',\n",
    "                        index_col=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load model from 2020-06-10 notebook\n",
    "# bundle = joblib.load('/opt/program/artifacts/2020-06-11T041641Z/bundle.joblib')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "example\n",
    "log_loss(y[:100], y_prob_vec[:100], labels=sorted(list(dict(Counter(y)).keys())))\n",
    "=> 4.133\n",
    "\n",
    "losses_vec = []\n",
    "for part in fu.get_partitions(list(range(len(y_prob_vec))), slice_size=1000):\n",
    "    i, j = part[0], part[-1]   \n",
    "    losses_vec.append(log_loss(y[i:j], y_prob_vec[i:j], labels=labels))\n",
    "\n",
    "fu.big_logloss(y, y_prob=y_prob_vec, labels=labels)\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### kernel keeps dying when I'm trying to do this\n",
    "* I finally started to log some breadcrumbs and I am seeing now where it is dying.. \n",
    "* I see that it is dying right before the second `xgb.XGBClassifier()` \n",
    "```\n",
    "2020-06-14 17:49:40Z, Starting\n",
    "2020-06-14 17:49:44Z, [0] Done preprocessing\n",
    "2020-06-14 18:01:09Z, [0] Done fit\n",
    "2020-06-14 18:01:09Z, [0] Done transforming test data\n",
    "2020-06-14 18:02:27Z, [0] Done predict()\n",
    "2020-06-14 18:03:48Z, [0] Done  predict_proba()\n",
    "2020-06-14 18:03:48Z, [0] Done  done fu.big_logloss()\n",
    "2020-06-14 18:03:48Z, [0] wrote bundle /opt/program/artifacts/2020-06-14T174940Z/bundle_0.joblib\n",
    "2020-06-14 18:03:50Z, [1] Done preprocessing\n",
    "```\n",
    "* The `xgb.XGBClassifier()` is indeed here the longest segment. I am now wondering about borrowing a technique from another project where I would just call a subprocess \n",
    "* Reading about using [external memory](https://xgboost.readthedocs.io/en/latest/tutorials/external_memory.html)\n",
    "* Maybe I can contribute to [this stackoverflow question](https://stackoverflow.com/questions/43972009/how-to-load-a-big-train-csv-for-xgboost) . And indeed [this np.memmap](https://stackoverflow.com/questions/16149803/working-with-big-data-in-python-and-numpy-not-enough-ram-how-to-save-partial-r/16633274#16633274) answer is interesting too.\n",
    "\n",
    "\n",
    "#### I ended up trying out the external memory approach\n",
    "- Based on what I read at least, I could only find this within the scope of the non-scikit learn implementation of XG boost. The \n",
    "- Or perhaps I can still keep trying, but for the below approach, it was the native `xgb.train()` usage.\n",
    "\n",
    "#### But this setup is using some different set of parameters and is doing terribly\n",
    "- My `('/opt/program/artifacts/2020-06-11T041641Z/bundle.joblib')` bundle being inspected at the head of this notebook has an accuracy of about `0.15` and I do not have a logloss estimate yet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'fresh.utils' from '/opt/program/fresh/utils.py'>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reload(fu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:14:46] SparsePageSource::CreateRowPage Finished writing to dtest.cache\n",
      "[23:14:47] 8000x78 matrix with 8000 entries loaded from /opt/program/artifacts/2020-06-14T225321Z/dtest.0.test#dtest.cache\n"
     ]
    }
   ],
   "source": [
    "dtest = xgb.DMatrix(f'{outpath}#dtest.cache')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:04:00] 1x78 matrix with 16000 entries loaded from /opt/program/artifacts/2020-06-14T225321Z/dtrain.0.train#dtrain.cache\n",
      "[23:04:00] WARNING: src/learner.cc:658: Tree method is automatically set to 'approx' since external-memory data matrix is used.\n",
      "[23:04:00] SparsePageSource: Finished writing to dtrain.cache\n",
      "[0]\ttrain-merror:1\n",
      "[1]\ttrain-merror:1\n"
     ]
    }
   ],
   "source": [
    "fu.save_libsvm(X_transformed.toarray(), y_enc, outpath)\n",
    "fu.log(workdir, f'[{i}] Saved train data: {outpath}')\n",
    "dtrain = xgb.DMatrix(f'{outpath}#dtrain.cache')\n",
    "\n",
    "param = {'max_depth':2, 'eta':1, 'objective':'multi:softprob', 'num_class': 54}\n",
    "\n",
    "\n",
    "watchlist = [ (dtrain, 'train')]\n",
    "num_round = 2\n",
    "bst = xgb.train(param, dtrain, num_round, watchlist)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_prob = bst.predict(dtest)\n",
    "y_prob_classes = np.argmax(y_prob, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_prob_classes[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Another go\n",
    "But store the intermediary results this time unlike inthe 2020-06-10 notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on method fit in module xgboost.sklearn:\n",
      "\n",
      "fit(X, y, sample_weight=None, eval_set=None, eval_metric=None, early_stopping_rounds=None, verbose=True, xgb_model=None, sample_weight_eval_set=None, callbacks=None) method of xgboost.sklearn.XGBClassifier instance\n",
      "    Fit gradient boosting classifier\n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    X : array_like\n",
      "        Feature matrix\n",
      "    y : array_like\n",
      "        Labels\n",
      "    sample_weight : array_like\n",
      "        Weight for each instance\n",
      "    eval_set : list, optional\n",
      "        A list of (X, y) pairs to use as a validation set for\n",
      "        early-stopping\n",
      "    sample_weight_eval_set : list, optional\n",
      "        A list of the form [L_1, L_2, ..., L_n], where each L_i is a list of\n",
      "        instance weights on the i-th validation set.\n",
      "    eval_metric : str, callable, optional\n",
      "        If a str, should be a built-in evaluation metric to use. See\n",
      "        doc/parameter.rst. If callable, a custom evaluation metric. The call\n",
      "        signature is func(y_predicted, y_true) where y_true will be a\n",
      "        DMatrix object such that you may need to call the get_label\n",
      "        method. It must return a str, value pair where the str is a name\n",
      "        for the evaluation and value is the value of the evaluation\n",
      "        function. This objective is always minimized.\n",
      "    early_stopping_rounds : int, optional\n",
      "        Activates early stopping. Validation error needs to decrease at\n",
      "        least every <early_stopping_rounds> round(s) to continue training.\n",
      "        Requires at least one item in evals. If there's more than one,\n",
      "        will use the last. If early stopping occurs, the model will have\n",
      "        three additional fields: bst.best_score, bst.best_iteration and\n",
      "        bst.best_ntree_limit (bst.best_ntree_limit is the ntree_limit parameter\n",
      "        default value in predict method if not any other value is specified).\n",
      "        (Use bst.best_ntree_limit to get the correct value if num_parallel_tree\n",
      "        and/or num_class appears in the parameters)\n",
      "    verbose : bool\n",
      "        If `verbose` and an evaluation set is used, writes the evaluation\n",
      "        metric measured on the validation set to stderr.\n",
      "    xgb_model : str\n",
      "        file name of stored xgb model or 'Booster' instance Xgb model to be\n",
      "        loaded before training (allows training continuation).\n",
      "    callbacks : list of callback functions\n",
      "        List of callback functions that are applied at end of each iteration.\n",
      "        It is possible to use predefined callbacks by using :ref:`callback_api`.\n",
      "        Example:\n",
      "    \n",
      "        .. code-block:: python\n",
      "    \n",
      "            [xgb.callback.reset_learning_rate(custom_rates)]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(xgb.XGBClassifier().fit) # feature_names=[], label=? \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "workdir:  /opt/program/artifacts/2020-06-15T003722Z\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6a66849e7e248c68a986f4ee35ead79",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='outer', max=1.0, style=ProgressStyle(de…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[00:38:24] 1x78 matrix with 16000 entries loaded from /opt/program/artifacts/2020-06-15T003722Z/dtrain.txt#dtrain.cache\n",
      "[00:38:24] WARNING: src/learner.cc:658: Tree method is automatically set to 'approx' since external-memory data matrix is used.\n",
      "[00:38:24] SparsePageSource: Finished writing to dtrain.cache\n",
      "[0]\ttrain-merror:1\n",
      "[1]\ttrain-merror:1\n",
      "[00:38:39] 168684x78 matrix with 168684 entries loaded from /opt/program/artifacts/2020-06-15T003722Z/dtest.txt\n",
      "[00:39:41] 1x78 matrix with 16000 entries loaded from /opt/program/artifacts/2020-06-15T003722Z/dtrain.txt#dtrain.cache\n",
      "[00:39:41] WARNING: src/learner.cc:658: Tree method is automatically set to 'approx' since external-memory data matrix is used.\n",
      "[00:39:41] SparsePageSource: Finished writing to dtrain.cache\n",
      "[0]\ttrain-merror:1\n",
      "[1]\ttrain-merror:1\n",
      "[00:39:56] 168683x78 matrix with 168683 entries loaded from /opt/program/artifacts/2020-06-15T003722Z/dtest.txt\n",
      "[00:40:56] 1x78 matrix with 16000 entries loaded from /opt/program/artifacts/2020-06-15T003722Z/dtrain.txt#dtrain.cache\n",
      "[00:40:56] WARNING: src/learner.cc:658: Tree method is automatically set to 'approx' since external-memory data matrix is used.\n",
      "[00:40:56] SparsePageSource: Finished writing to dtrain.cache\n",
      "[0]\ttrain-merror:1\n",
      "[1]\ttrain-merror:1\n",
      "[00:41:11] 168683x78 matrix with 168683 entries loaded from /opt/program/artifacts/2020-06-15T003722Z/dtest.txt\n",
      "[00:42:10] 1x78 matrix with 16000 entries loaded from /opt/program/artifacts/2020-06-15T003722Z/dtrain.txt#dtrain.cache\n",
      "[00:42:10] WARNING: src/learner.cc:658: Tree method is automatically set to 'approx' since external-memory data matrix is used.\n",
      "[00:42:11] SparsePageSource: Finished writing to dtrain.cache\n",
      "[0]\ttrain-merror:1\n",
      "[1]\ttrain-merror:1\n",
      "[00:42:25] 168683x78 matrix with 168683 entries loaded from /opt/program/artifacts/2020-06-15T003722Z/dtest.txt\n",
      "[00:43:27] 1x78 matrix with 16000 entries loaded from /opt/program/artifacts/2020-06-15T003722Z/dtrain.txt#dtrain.cache\n",
      "[00:43:27] WARNING: src/learner.cc:658: Tree method is automatically set to 'approx' since external-memory data matrix is used.\n",
      "[00:43:27] SparsePageSource: Finished writing to dtrain.cache\n",
      "[0]\ttrain-merror:1\n",
      "[1]\ttrain-merror:1\n",
      "[00:43:42] 168683x78 matrix with 168683 entries loaded from /opt/program/artifacts/2020-06-15T003722Z/dtest.txt\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#%%time\n",
    "\n",
    "def runner():\n",
    "\n",
    "    # New workdir\n",
    "    workdir = fu.make_work_dir(); print('workdir: ', workdir)\n",
    "    fu.log(workdir, 'Starting')\n",
    "\n",
    "    rng = np.random.RandomState(31337)\n",
    "    X, y, neighborhoods = fu.prepare_data(tripsdf, stationsdf)\n",
    "    neighborhoods = sorted(stationsdf.neighborhood.unique().tolist())\n",
    "    labels = sorted(list(dict(Counter(y)).keys()))\n",
    "\n",
    "    #X, y = X[:10000], y[:10000]\n",
    "\n",
    "    kf = KFold(n_splits=5, shuffle=True, random_state=rng)\n",
    "    for i, (train_index, test_index) in enumerate(tqdm(kf.split(X), desc='outer', leave=True)):\n",
    "\n",
    "        # preprocess\n",
    "        (X_transformed, one_hot_enc, le,\n",
    "             y_enc) = pv1.preprocess(X[train_index], y[train_index], \n",
    "                                 neighborhoods)\n",
    "        fu.log(workdir, f'[{i}] Done preprocessing')\n",
    "        labels = le.classes_\n",
    "        param = {'max_depth':3, 'eta': .1, 'objective':'multi:softprob', 'num_class': len(labels)}  # 54\n",
    "\n",
    "\n",
    "        #\n",
    "        outpath = f'{workdir}/dtrain.txt'\n",
    "        fu.save_libsvm(X_transformed.toarray(), y_enc, outpath)\n",
    "        fu.log(workdir, f'[{i}] Saved train data: {outpath}')\n",
    "        dtrain = xgb.DMatrix(f'{outpath}#dtrain.cache')\n",
    "        #xgb_model = xgb.XGBClassifier().fit(dtrain) #X_transformed, y_enc\n",
    "        watchlist = [ (dtrain, 'train')]\n",
    "        num_round = 2\n",
    "        \n",
    "        xgb_model = xgb.train(param, dtrain, num_round, watchlist)\n",
    "\n",
    "        fu.log(workdir, f'[{i}] Done fit')\n",
    "\n",
    "        X_test_transformed = one_hot_enc.transform(X[test_index]).toarray()\n",
    "        actuals = le.transform(y[test_index])\n",
    "        fu.log(workdir, f'[{i}] Done transforming test data')\n",
    "\n",
    "        outpath = f'{workdir}/dtest.txt'    \n",
    "        fu.save_libsvm(X_test_transformed, outpath=outpath)\n",
    "        fu.log(workdir, f'[{i}] Done saving to {outpath}')\n",
    "        dtest = xgb.DMatrix(f'{outpath}') # #dtest.cache\n",
    "\n",
    "\n",
    "        # predictions = xgb_model.predict(X_test_transformed)\n",
    "        y_prob_vec = xgb_model.predict(dtest)\n",
    "        predictions = np.argmax(y_prob_vec, axis=1)\n",
    "\n",
    "        fu.log(workdir, f'[{i}] Done predict()')\n",
    "        correct = len([i for i, _ in enumerate(actuals)\n",
    "                  if actuals[i] == predictions[i]])\n",
    "\n",
    "        #y_prob_vec = []\n",
    "        #X_parts = fu.get_partitions(X_test_transformed, slice_size=1000)\n",
    "        #for X_part in tqdm(X_parts, desc='inner', leave=False):\n",
    "        #    # X_transformed = one_hot_enc.transform(X_part)\n",
    "        #    y_prob = xgb_model.predict_proba(X_part)\n",
    "        #    y_prob_vec.extend(y_prob)\n",
    "        #fu.log(workdir, f'[{i}] Done  predict_proba()')\n",
    "        # y_prob_vec = xgb_model.predict_proba(X_test_transformed)\n",
    "\n",
    "        bundle_file = f'{workdir}/bundle_{i}.joblib'\n",
    "        loss = fu.big_logloss(actuals, y_prob=y_prob_vec, labels=labels)\n",
    "        fu.log(workdir, f'[{i}] Done  done fu.big_logloss()')\n",
    "\n",
    "        joblib.dump({'loss': loss,\n",
    "         'confusion_matrix': confusion_matrix(actuals, predictions),\n",
    "         'model': xgb_model,\n",
    "         'notebook': '2020-06-12.ipynb',\n",
    "         'accuracy': correct/len(actuals),\n",
    "         'timestamp': fu.utc_ts(),\n",
    "        }, bundle_file)\n",
    "        fu.log(workdir, f'[{i}] wrote bundle {bundle_file}')\n",
    "        \n",
    "import ipdb\n",
    "#ipdb.runcall(runner)\n",
    "runner()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'loss': 0.0,\n",
       " 'confusion_matrix': array([[3738,    0,    0, ...,    0,    0,    0],\n",
       "        [4807,    0,    0, ...,    0,    0,    0],\n",
       "        [ 931,    0,    0, ...,    0,    0,    0],\n",
       "        ...,\n",
       "        [ 170,    0,    0, ...,    0,    0,    0],\n",
       "        [7305,    0,    0, ...,    0,    0,    0],\n",
       "        [2473,    0,    0, ...,    0,    0,    0]]),\n",
       " 'model': <xgboost.core.Booster at 0x7f8b60a7b190>,\n",
       " 'notebook': '2020-06-12.ipynb',\n",
       " 'accuracy': 0.022159909415886605,\n",
       " 'timestamp': '2020-06-15T004112Z'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "workdir = '/opt/program/artifacts/2020-06-15T003722Z/' # artifacts/2020-06-15T003722Z/bundle_0.joblib\n",
    "bundle = joblib.load(f'{workdir}/bundle_2.joblib')\n",
    "bundle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'loss': 0.0,\n",
       " 'confusion_matrix': array([[3774,    0,    0, ...,    0,    0,    0],\n",
       "        [4818,    0,    0, ...,    0,    0,    0],\n",
       "        [ 913,    0,    0, ...,    0,    0,    0],\n",
       "        ...,\n",
       "        [ 187,    0,    0, ...,    0,    0,    0],\n",
       "        [7111,    0,    0, ...,    0,    0,    0],\n",
       "        [2364,    0,    0, ...,    0,    0,    0]]),\n",
       " 'model': <xgboost.core.Booster at 0x7f8b60adb410>,\n",
       " 'notebook': '2020-06-12.ipynb',\n",
       " 'accuracy': 0.022373194849541155,\n",
       " 'timestamp': '2020-06-15T003840Z'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bundle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bundle['model'].get_fscore()#(f'{workdir}/model_0.json', with_stats=True, dump_format='json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8000, 78)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtest.num_row(), dtest.num_col() #    y_prob_vec = xgb_model.predict(dtest)\n",
    "# This is too weird.\n",
    "'''\n",
    "ipdb> p xgb.DMatrix('/opt/program/artifacts/2020-06-14T235247Z/dtest.txt').num_row()\n",
    "[00:05:44] 2000x78 matrix with 2000 entries loaded from /opt/program/artifacts/2020-06-14T235247Z/dtest.txt\n",
    "2000\n",
    "ipdb> p xgb.DMatrix('/opt/program/artifacts/2020-06-14T235247Z/dtest.txt#dtest.cache').num_row()\n",
    "[00:06:31] 8000x78 matrix with 8000 entries loaded from /opt/program/artifacts/2020-06-14T235247Z/dtest.txt#dtest.cache\n",
    "8000\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 0:45 , starting \n",
    "# 1:00 ,  still going, but dang no sign of the inner tqdm yet (the predict proba)\n",
    "# 1:01 => TypeError, sparse matrix length is ambiguous; use getnnz() or shape[0]\n",
    "#          \n",
    "#/opt/program/fresh/utils.py in get_partitions(vec, slice_size, keep_remainder)\n",
    "#      39 def get_partitions(vec, slice_size, keep_remainder=True):\n",
    "#      40     assert slice_size > 0\n",
    "# ---> 41     num_slices = int(math.floor(len(vec)/slice_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fu.save_libsvm(X_transformed, y_enc, outpath)\n",
    "X_transformed.toarray()[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c56fe9a67d7417fb51587765df94de3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='outer', max=10.0, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='inner', max=10.0, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='inner', max=10.0, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='inner', max=10.0, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='inner', max=10.0, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='inner', max=10.0, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='inner', max=10.0, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='inner', max=10.0, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='inner', max=10.0, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='inner', max=10.0, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='inner', max=10.0, style=ProgressStyle(description_width='…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "foo\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "import time\n",
    "for i in tqdm(range(10), desc='outer', leave=True):\n",
    "    for j in tqdm(range(10), desc='inner', leave=False):\n",
    "        time.sleep(.5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3.993434763431549, 3.9934347634315492)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pretty sure since I had had to partition the logloss , I can still average it to get a full score.\n",
    "i, j = 0, 1000\n",
    "a = log_loss(y[i:j], y_prob_vec[i:j], labels=labels)\n",
    "\n",
    "i, j = 1000, 2000\n",
    "b = log_loss(y[i:j], y_prob_vec[i:j], labels=labels)\n",
    "\n",
    "i, j = 0, 2000\n",
    "c = log_loss(y[i:j], y_prob_vec[i:j], labels=labels)\n",
    "\n",
    "(a+b)/2, c\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Future food for thought\n",
    "- [here](https://medium.com/@gabrielziegler3/multiclass-multilabel-classification-with-xgboost-66195e4d9f2d) , a note about using one versus rest approach \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}