2020-06-14.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# try again  the external memory approach w/ sklearn style XGboost...!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import xgboost as xgb\n",
    "from xgboost import XGBClassifier\n",
    "import datetime; import pytz\n",
    "import matplotlib as plt\n",
    "from scipy.special import softmax\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import train_test_split # (*arrays, **options)\n",
    "import numpy as np\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.preprocessing import OneHotEncoder, LabelEncoder\n",
    "\n",
    "from joblib import dump, load\n",
    "import joblib\n",
    "import os\n",
    "from sklearn.metrics import confusion_matrix, mean_squared_error\n",
    "from sklearn.model_selection import KFold, train_test_split, GridSearchCV\n",
    "from sklearn.metrics import accuracy_score\n",
    "import fresh.utils as fu\n",
    "\n",
    "from importlib import reload\n",
    "from collections import Counter\n",
    "from tqdm.notebook import tqdm\n",
    "import fresh.preproc.v1 as pv1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "datadir = '/opt/data'\n",
    "localdir = '/opt/program'\n",
    "tripsdf = pd.read_csv(f'{datadir}/2013-07 - Citi Bike trip data.csv')\n",
    "stationsdf = pd.read_csv(f'{localdir}/datas/stations/stations-2018-12-04-c.csv',\n",
    "                        index_col=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[03:35:29] 168683x78 matrix with 168683 entries loaded from /opt/program/artifacts/2020-06-15T003722Z/dtest.txt\n"
     ]
    }
   ],
   "source": [
    "artifactdir = '/opt/program/artifacts/2020-06-15T003722Z' \n",
    "bundle = joblib.load(f'{artifactdir}/bundle_2.joblib')\n",
    "model = bundle['model']\n",
    "dtest = xgb.DMatrix(f'{artifactdir}/dtest.txt')\n",
    "\n",
    "y_preds = model.predict(dtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'base_score': 0.5, 'booster': 'gbtree', 'colsample_bylevel': 1, 'colsample_bynode': 1, 'colsample_bytree': 1, 'gamma': 0, 'learning_rate': 0.1, 'max_delta_step': 0, 'max_depth': 3, 'min_child_weight': 1, 'missing': nan, 'n_estimators': 100, 'nthread': 1, 'objective': 'multi:softprob', 'reg_alpha': 0, 'reg_lambda': 1, 'scale_pos_weight': 1, 'seed': 0, 'subsample': 1, 'verbosity': 1}\n",
      "100\n"
     ]
    }
   ],
   "source": [
    "bundle = joblib.load('/opt/program/artifacts/2020-06-11T041641Z/bundle.joblib')\n",
    "model = bundle['model']\n",
    "param = model.get_xgb_params()\n",
    "print(param)\n",
    "print(model.get_num_boosting_rounds()) \n",
    "# Oh hmm I was wondering in the \"2020-06-12.ipynb\" notebook why the results were so bad.\n",
    "# Perhaps since this model appears to have `100` boosting rounds (in the past I suppse..) maybe indeed\n",
    "# thiss is a matter of 10 vs 100 rounds."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[03:46:07] 674733x78 matrix with 1349466 entries loaded from /opt/program/artifacts/2020-06-15T003722Z/dtrain.txt\n"
     ]
    }
   ],
   "source": [
    "dtrain = xgb.DMatrix(f'{artifactdir}/dtrain.txt')\n",
    "booster = xgb.train({**param, 'num_class': 54}, dtrain, num_boost_round=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on class XGBClassifier in module xgboost.sklearn:\n",
      "\n",
      "class XGBClassifier(XGBModel, sklearn.base.ClassifierMixin)\n",
      " |  XGBClassifier(max_depth=3, learning_rate=0.1, n_estimators=100, verbosity=1, silent=None, objective='binary:logistic', booster='gbtree', n_jobs=1, nthread=None, gamma=0, min_child_weight=1, max_delta_step=0, subsample=1, colsample_bytree=1, colsample_bylevel=1, colsample_bynode=1, reg_alpha=0, reg_lambda=1, scale_pos_weight=1, base_score=0.5, random_state=0, seed=None, missing=None, **kwargs)\n",
      " |  \n",
      " |  Implementation of the scikit-learn API for XGBoost classification.\n",
      " |  \n",
      " |  Parameters\n",
      " |  ----------\n",
      " |  max_depth : int\n",
      " |      Maximum tree depth for base learners.\n",
      " |  learning_rate : float\n",
      " |      Boosting learning rate (xgb's \"eta\")\n",
      " |  n_estimators : int\n",
      " |      Number of trees to fit.\n",
      " |  verbosity : int\n",
      " |      The degree of verbosity. Valid values are 0 (silent) - 3 (debug).\n",
      " |  silent : boolean\n",
      " |      Whether to print messages while running boosting. Deprecated. Use verbosity instead.\n",
      " |  objective : string or callable\n",
      " |      Specify the learning task and the corresponding learning objective or\n",
      " |      a custom objective function to be used (see note below).\n",
      " |  booster: string\n",
      " |      Specify which booster to use: gbtree, gblinear or dart.\n",
      " |  nthread : int\n",
      " |      Number of parallel threads used to run xgboost.  (Deprecated, please use ``n_jobs``)\n",
      " |  n_jobs : int\n",
      " |      Number of parallel threads used to run xgboost.  (replaces ``nthread``)\n",
      " |  gamma : float\n",
      " |      Minimum loss reduction required to make a further partition on a leaf node of the tree.\n",
      " |  min_child_weight : int\n",
      " |      Minimum sum of instance weight(hessian) needed in a child.\n",
      " |  max_delta_step : int\n",
      " |      Maximum delta step we allow each tree's weight estimation to be.\n",
      " |  subsample : float\n",
      " |      Subsample ratio of the training instance.\n",
      " |  colsample_bytree : float\n",
      " |      Subsample ratio of columns when constructing each tree.\n",
      " |  colsample_bylevel : float\n",
      " |      Subsample ratio of columns for each level.\n",
      " |  colsample_bynode : float\n",
      " |      Subsample ratio of columns for each split.\n",
      " |  reg_alpha : float (xgb's alpha)\n",
      " |      L1 regularization term on weights\n",
      " |  reg_lambda : float (xgb's lambda)\n",
      " |      L2 regularization term on weights\n",
      " |  scale_pos_weight : float\n",
      " |      Balancing of positive and negative weights.\n",
      " |  base_score:\n",
      " |      The initial prediction score of all instances, global bias.\n",
      " |  seed : int\n",
      " |      Random number seed.  (Deprecated, please use random_state)\n",
      " |  random_state : int\n",
      " |      Random number seed.  (replaces seed)\n",
      " |  missing : float, optional\n",
      " |      Value in the data which needs to be present as a missing value. If\n",
      " |      None, defaults to np.nan.\n",
      " |  importance_type: string, default \"gain\"\n",
      " |      The feature importance type for the feature_importances_ property: either \"gain\",\n",
      " |      \"weight\", \"cover\", \"total_gain\" or \"total_cover\".\n",
      " |  \\*\\*kwargs : dict, optional\n",
      " |      Keyword arguments for XGBoost Booster object.  Full documentation of parameters can\n",
      " |      be found here: https://github.com/dmlc/xgboost/blob/master/doc/parameter.rst.\n",
      " |      Attempting to set a parameter via the constructor args and \\*\\*kwargs dict simultaneously\n",
      " |      will result in a TypeError.\n",
      " |  \n",
      " |      .. note:: \\*\\*kwargs unsupported by scikit-learn\n",
      " |  \n",
      " |          \\*\\*kwargs is unsupported by scikit-learn.  We do not guarantee that parameters\n",
      " |          passed via this argument will interact properly with scikit-learn.\n",
      " |  \n",
      " |  Note\n",
      " |  ----\n",
      " |  A custom objective function can be provided for the ``objective``\n",
      " |  parameter. In this case, it should have the signature\n",
      " |  ``objective(y_true, y_pred) -> grad, hess``:\n",
      " |  \n",
      " |  y_true: array_like of shape [n_samples]\n",
      " |      The target values\n",
      " |  y_pred: array_like of shape [n_samples]\n",
      " |      The predicted values\n",
      " |  \n",
      " |  grad: array_like of shape [n_samples]\n",
      " |      The value of the gradient for each sample point.\n",
      " |  hess: array_like of shape [n_samples]\n",
      " |      The value of the second derivative for each sample point\n",
      " |  \n",
      " |  Method resolution order:\n",
      " |      XGBClassifier\n",
      " |      XGBModel\n",
      " |      sklearn.base.BaseEstimator\n",
      " |      sklearn.base.ClassifierMixin\n",
      " |      builtins.object\n",
      " |  \n",
      " |  Methods defined here:\n",
      " |  \n",
      " |  __init__(self, max_depth=3, learning_rate=0.1, n_estimators=100, verbosity=1, silent=None, objective='binary:logistic', booster='gbtree', n_jobs=1, nthread=None, gamma=0, min_child_weight=1, max_delta_step=0, subsample=1, colsample_bytree=1, colsample_bylevel=1, colsample_bynode=1, reg_alpha=0, reg_lambda=1, scale_pos_weight=1, base_score=0.5, random_state=0, seed=None, missing=None, **kwargs)\n",
      " |      Initialize self.  See help(type(self)) for accurate signature.\n",
      " |  \n",
      " |  evals_result(self)\n",
      " |      Return the evaluation results.\n",
      " |      \n",
      " |      If **eval_set** is passed to the `fit` function, you can call\n",
      " |      ``evals_result()`` to get evaluation results for all passed **eval_sets**.\n",
      " |      When **eval_metric** is also passed to the `fit` function, the\n",
      " |      **evals_result** will contain the **eval_metrics** passed to the `fit` function.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      evals_result : dictionary\n",
      " |      \n",
      " |      Example\n",
      " |      -------\n",
      " |      \n",
      " |      .. code-block:: python\n",
      " |      \n",
      " |          param_dist = {'objective':'binary:logistic', 'n_estimators':2}\n",
      " |      \n",
      " |          clf = xgb.XGBClassifier(**param_dist)\n",
      " |      \n",
      " |          clf.fit(X_train, y_train,\n",
      " |                  eval_set=[(X_train, y_train), (X_test, y_test)],\n",
      " |                  eval_metric='logloss',\n",
      " |                  verbose=True)\n",
      " |      \n",
      " |          evals_result = clf.evals_result()\n",
      " |      \n",
      " |      The variable **evals_result** will contain\n",
      " |      \n",
      " |      .. code-block:: python\n",
      " |      \n",
      " |          {'validation_0': {'logloss': ['0.604835', '0.531479']},\n",
      " |          'validation_1': {'logloss': ['0.41965', '0.17686']}}\n",
      " |  \n",
      " |  fit(self, X, y, sample_weight=None, eval_set=None, eval_metric=None, early_stopping_rounds=None, verbose=True, xgb_model=None, sample_weight_eval_set=None, callbacks=None)\n",
      " |      Fit gradient boosting classifier\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      X : array_like\n",
      " |          Feature matrix\n",
      " |      y : array_like\n",
      " |          Labels\n",
      " |      sample_weight : array_like\n",
      " |          Weight for each instance\n",
      " |      eval_set : list, optional\n",
      " |          A list of (X, y) pairs to use as a validation set for\n",
      " |          early-stopping\n",
      " |      sample_weight_eval_set : list, optional\n",
      " |          A list of the form [L_1, L_2, ..., L_n], where each L_i is a list of\n",
      " |          instance weights on the i-th validation set.\n",
      " |      eval_metric : str, callable, optional\n",
      " |          If a str, should be a built-in evaluation metric to use. See\n",
      " |          doc/parameter.rst. If callable, a custom evaluation metric. The call\n",
      " |          signature is func(y_predicted, y_true) where y_true will be a\n",
      " |          DMatrix object such that you may need to call the get_label\n",
      " |          method. It must return a str, value pair where the str is a name\n",
      " |          for the evaluation and value is the value of the evaluation\n",
      " |          function. This objective is always minimized.\n",
      " |      early_stopping_rounds : int, optional\n",
      " |          Activates early stopping. Validation error needs to decrease at\n",
      " |          least every <early_stopping_rounds> round(s) to continue training.\n",
      " |          Requires at least one item in evals. If there's more than one,\n",
      " |          will use the last. If early stopping occurs, the model will have\n",
      " |          three additional fields: bst.best_score, bst.best_iteration and\n",
      " |          bst.best_ntree_limit (bst.best_ntree_limit is the ntree_limit parameter\n",
      " |          default value in predict method if not any other value is specified).\n",
      " |          (Use bst.best_ntree_limit to get the correct value if num_parallel_tree\n",
      " |          and/or num_class appears in the parameters)\n",
      " |      verbose : bool\n",
      " |          If `verbose` and an evaluation set is used, writes the evaluation\n",
      " |          metric measured on the validation set to stderr.\n",
      " |      xgb_model : str\n",
      " |          file name of stored xgb model or 'Booster' instance Xgb model to be\n",
      " |          loaded before training (allows training continuation).\n",
      " |      callbacks : list of callback functions\n",
      " |          List of callback functions that are applied at end of each iteration.\n",
      " |          It is possible to use predefined callbacks by using :ref:`callback_api`.\n",
      " |          Example:\n",
      " |      \n",
      " |          .. code-block:: python\n",
      " |      \n",
      " |              [xgb.callback.reset_learning_rate(custom_rates)]\n",
      " |  \n",
      " |  predict(self, data, output_margin=False, ntree_limit=None, validate_features=True)\n",
      " |      Predict with `data`.\n",
      " |      \n",
      " |      .. note:: This function is not thread safe.\n",
      " |      \n",
      " |        For each booster object, predict can only be called from one thread.\n",
      " |        If you want to run prediction using multiple thread, call ``xgb.copy()`` to make copies\n",
      " |        of model object and then call ``predict()``.\n",
      " |      \n",
      " |      .. note:: Using ``predict()`` with DART booster\n",
      " |      \n",
      " |        If the booster object is DART type, ``predict()`` will perform dropouts, i.e. only\n",
      " |        some of the trees will be evaluated. This will produce incorrect results if ``data`` is\n",
      " |        not the training data. To obtain correct results on test sets, set ``ntree_limit`` to\n",
      " |        a nonzero value, e.g.\n",
      " |      \n",
      " |        .. code-block:: python\n",
      " |      \n",
      " |          preds = bst.predict(dtest, ntree_limit=num_round)\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      data : DMatrix\n",
      " |          The dmatrix storing the input.\n",
      " |      output_margin : bool\n",
      " |          Whether to output the raw untransformed margin value.\n",
      " |      ntree_limit : int\n",
      " |          Limit number of trees in the prediction; defaults to best_ntree_limit if defined\n",
      " |          (i.e. it has been trained with early stopping), otherwise 0 (use all trees).\n",
      " |      validate_features : bool\n",
      " |          When this is True, validate that the Booster's and data's feature_names are identical.\n",
      " |          Otherwise, it is assumed that the feature_names are the same.\n",
      " |      Returns\n",
      " |      -------\n",
      " |      prediction : numpy array\n",
      " |  \n",
      " |  predict_proba(self, data, ntree_limit=None, validate_features=True)\n",
      " |      Predict the probability of each `data` example being of a given class.\n",
      " |      \n",
      " |      .. note:: This function is not thread safe\n",
      " |      \n",
      " |          For each booster object, predict can only be called from one thread.\n",
      " |          If you want to run prediction using multiple thread, call ``xgb.copy()`` to make copies\n",
      " |          of model object and then call predict\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      data : DMatrix\n",
      " |          The dmatrix storing the input.\n",
      " |      ntree_limit : int\n",
      " |          Limit number of trees in the prediction; defaults to best_ntree_limit if defined\n",
      " |          (i.e. it has been trained with early stopping), otherwise 0 (use all trees).\n",
      " |      validate_features : bool\n",
      " |          When this is True, validate that the Booster's and data's feature_names are identical.\n",
      " |          Otherwise, it is assumed that the feature_names are the same.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      prediction : numpy array\n",
      " |          a numpy array with the probability of each data example being of a given class.\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from XGBModel:\n",
      " |  \n",
      " |  __setstate__(self, state)\n",
      " |  \n",
      " |  apply(self, X, ntree_limit=0)\n",
      " |      Return the predicted leaf every tree for each sample.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      X : array_like, shape=[n_samples, n_features]\n",
      " |          Input features matrix.\n",
      " |      \n",
      " |      ntree_limit : int\n",
      " |          Limit number of trees in the prediction; defaults to 0 (use all trees).\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      X_leaves : array_like, shape=[n_samples, n_trees]\n",
      " |          For each datapoint x in X and for each tree, return the index of the\n",
      " |          leaf x ends up in. Leaves are numbered within\n",
      " |          ``[0; 2**(self.max_depth+1))``, possibly with gaps in the numbering.\n",
      " |  \n",
      " |  get_booster(self)\n",
      " |      Get the underlying xgboost Booster of this model.\n",
      " |      \n",
      " |      This will raise an exception when fit was not called\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      booster : a xgboost booster of underlying model\n",
      " |  \n",
      " |  get_num_boosting_rounds(self)\n",
      " |      Gets the number of xgboost boosting rounds.\n",
      " |  \n",
      " |  get_params(self, deep=False)\n",
      " |      Get parameters.\n",
      " |  \n",
      " |  get_xgb_params(self)\n",
      " |      Get xgboost type parameters.\n",
      " |  \n",
      " |  load_model(self, fname)\n",
      " |      Load the model from a file.\n",
      " |      \n",
      " |      The model is loaded from an XGBoost internal binary format which is\n",
      " |      universal among the various XGBoost interfaces. Auxiliary attributes of\n",
      " |      the Python Booster object (such as feature names) will not be loaded.\n",
      " |      Label encodings (text labels to numeric labels) will be also lost.\n",
      " |      **If you are using only the Python interface, we recommend pickling the\n",
      " |      model object for best results.**\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      fname : string or a memory buffer\n",
      " |          Input file name or memory buffer(see also save_raw)\n",
      " |  \n",
      " |  save_model(self, fname)\n",
      " |      Save the model to a file.\n",
      " |      \n",
      " |      The model is saved in an XGBoost internal binary format which is\n",
      " |      universal among the various XGBoost interfaces. Auxiliary attributes of\n",
      " |      the Python Booster object (such as feature names) will not be loaded.\n",
      " |      Label encodings (text labels to numeric labels) will be also lost.\n",
      " |      **If you are using only the Python interface, we recommend pickling the\n",
      " |      model object for best results.**\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      fname : string\n",
      " |          Output file name\n",
      " |  \n",
      " |  set_params(self, **params)\n",
      " |      Set the parameters of this estimator.\n",
      " |      Modification of the sklearn method to allow unknown kwargs. This allows using\n",
      " |      the full range of xgboost parameters that are not defined as member variables\n",
      " |      in sklearn grid search.\n",
      " |      Returns\n",
      " |      -------\n",
      " |      self\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors inherited from XGBModel:\n",
      " |  \n",
      " |  coef_\n",
      " |      Coefficients property\n",
      " |      \n",
      " |      .. note:: Coefficients are defined only for linear learners\n",
      " |      \n",
      " |          Coefficients are only defined when the linear model is chosen as base\n",
      " |          learner (`booster=gblinear`). It is not defined for other base learner types, such\n",
      " |          as tree learners (`booster=gbtree`).\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      coef_ : array of shape ``[n_features]`` or ``[n_classes, n_features]``\n",
      " |  \n",
      " |  feature_importances_\n",
      " |      Feature importances property\n",
      " |      \n",
      " |      .. note:: Feature importance is defined only for tree boosters\n",
      " |      \n",
      " |          Feature importance is only defined when the decision tree model is chosen as base\n",
      " |          learner (`booster=gbtree`). It is not defined for other base learner types, such\n",
      " |          as linear learners (`booster=gblinear`).\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      feature_importances_ : array of shape ``[n_features]``\n",
      " |  \n",
      " |  intercept_\n",
      " |      Intercept (bias) property\n",
      " |      \n",
      " |      .. note:: Intercept is defined only for linear learners\n",
      " |      \n",
      " |          Intercept (bias) is only defined when the linear model is chosen as base\n",
      " |          learner (`booster=gblinear`). It is not defined for other base learner types, such\n",
      " |          as tree learners (`booster=gbtree`).\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      intercept_ : array of shape ``(1,)`` or ``[n_classes]``\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from sklearn.base.BaseEstimator:\n",
      " |  \n",
      " |  __getstate__(self)\n",
      " |  \n",
      " |  __repr__(self, N_CHAR_MAX=700)\n",
      " |      Return repr(self).\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors inherited from sklearn.base.BaseEstimator:\n",
      " |  \n",
      " |  __dict__\n",
      " |      dictionary for instance variables (if defined)\n",
      " |  \n",
      " |  __weakref__\n",
      " |      list of weak references to the object (if defined)\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from sklearn.base.ClassifierMixin:\n",
      " |  \n",
      " |  score(self, X, y, sample_weight=None)\n",
      " |      Return the mean accuracy on the given test data and labels.\n",
      " |      \n",
      " |      In multi-label classification, this is the subset accuracy\n",
      " |      which is a harsh metric since you require for each sample that\n",
      " |      each label set be correctly predicted.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      X : array-like of shape (n_samples, n_features)\n",
      " |          Test samples.\n",
      " |      \n",
      " |      y : array-like of shape (n_samples,) or (n_samples, n_outputs)\n",
      " |          True labels for X.\n",
      " |      \n",
      " |      sample_weight : array-like of shape (n_samples,), default=None\n",
      " |          Sample weights.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      score : float\n",
      " |          Mean accuracy of self.predict(X) wrt. y.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(xgb.XGBClassifier)\n",
    "# clf.fit(xgb_model=) # loaded before training (allows training continuation).\n",
    "clf = xgb.XGBClassifier().load_model(fname='') # feature names not loaded \n",
    "# can also clf.save_model() # but they say in the doc... \n",
    "# **If you are using only the Python interface, we recommend pickling the\n",
    "#      model object for best results.**\n",
    "#\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y, neighborhoods = fu.prepare_data(tripsdf, stationsdf)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "clf = xgb.XGBClassifier()\n",
    "\n",
    "\n",
    "# preproc\n",
    "(X_transformed,\n",
    "     one_hot_enc, le,\n",
    "     y_enc) = pv1.preprocess(X_train, y_train, # X[train_index]\n",
    "                         neighborhoods)\n",
    "labels = le.classes_\n",
    "\n",
    "# Test set\n",
    "X_test_transformed = one_hot_enc.transform(X_test)\n",
    "y_test_enc = le.transform(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "54"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Counter(y_train)) # 54"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "XGBoostError",
     "evalue": "need to call fit or load_model beforehand",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mXGBoostError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-d3a12c5ce30a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# pre-evaluate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0my_prob_vec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test_transformed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfu\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbig_logloss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test_enc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_prob_vec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/xgboost/sklearn.py\u001b[0m in \u001b[0;36mpredict_proba\u001b[0;34m(self, data, ntree_limit, validate_features)\u001b[0m\n\u001b[1;32m    830\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mntree_limit\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    831\u001b[0m             \u001b[0mntree_limit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"best_ntree_limit\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 832\u001b[0;31m         class_probs = self.get_booster().predict(test_dmatrix,\n\u001b[0m\u001b[1;32m    833\u001b[0m                                                  \u001b[0mntree_limit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mntree_limit\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    834\u001b[0m                                                  validate_features=validate_features)\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/xgboost/sklearn.py\u001b[0m in \u001b[0;36mget_booster\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    191\u001b[0m         \"\"\"\n\u001b[1;32m    192\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_Booster\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 193\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mXGBoostError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'need to call fit or load_model beforehand'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    194\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_Booster\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mXGBoostError\u001b[0m: need to call fit or load_model beforehand"
     ]
    }
   ],
   "source": [
    "# pre-evaluate\n",
    "# y_prob_vec = clf.predict_proba(X_test_transformed)\n",
    "# loss = fu.big_logloss(y_test_enc, y_prob_vec, labels)\n",
    "#\n",
    "# ok, cannot call on raw model like TensorFlow. heh.\n",
    "# XGBoostError: need to call fit or load_model beforehand"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac3aa61865644fad865ccf75649541c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=633.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "The number of classes in labels is different from that in y_pred. Classes found in labels: ['Alphabet City' 'Battery Park City' 'Bedford-Stuyvesant' 'Boerum Hill'\n 'Bowery' 'Brooklyn Heights' 'Brooklyn Navy Yard' 'Central Park' 'Chelsea'\n 'Chinatown' 'Civic Center' 'Clinton Hill'\n 'Columbia Street Waterfront District' 'Downtown Brooklyn' 'Dumbo'\n 'East Village' 'Financial District' 'Flatiron District' 'Fort Greene'\n 'Fulton Ferry District' 'Garment District' 'Gramercy Park'\n 'Greenwich Village' \"Hell's Kitchen\" 'Hudson Square' 'Kips Bay'\n 'Korea Town' 'Lincoln Square' 'Little Italy' 'Lower East Side'\n 'Lower Manhattan' 'Meatpacking District' 'Midtown' 'Midtown East'\n 'Midtown West' 'Murray Hill' 'NoHo' 'NoMad' 'Nolita' 'Park Slope'\n 'Peter Cooper Village' 'Rose Hill' 'SoHo' 'Stuyvesant Town'\n 'Sutton Place' 'Theater District' 'Tribeca' 'Tudor City' 'Two Bridges'\n 'Ukrainian Village' 'Union Square' 'Vinegar Hill' 'West Village'\n 'Williamsburg']",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-499be10396ba>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0my_prob_vec\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test_transformed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfu\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbig_logloss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test_enc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_prob_vec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m     \u001b[0mloss_vec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/program/fresh/utils.py\u001b[0m in \u001b[0;36mbig_logloss\u001b[0;34m(y, y_prob, labels)\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mpart\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mget_partitions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_prob\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mslice_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpart\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpart\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m         \u001b[0mlosses_vec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlog_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_prob\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlosses_vec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/sklearn/metrics/_classification.py\u001b[0m in \u001b[0;36mlog_loss\u001b[0;34m(y_true, y_pred, eps, normalize, sample_weight, labels)\u001b[0m\n\u001b[1;32m   2289\u001b[0m             raise ValueError('The number of classes in labels is different '\n\u001b[1;32m   2290\u001b[0m                              \u001b[0;34m'from that in y_pred. Classes found in '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2291\u001b[0;31m                              'labels: {0}'.format(lb.classes_))\n\u001b[0m\u001b[1;32m   2292\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2293\u001b[0m     \u001b[0;31m# Renormalize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: The number of classes in labels is different from that in y_pred. Classes found in labels: ['Alphabet City' 'Battery Park City' 'Bedford-Stuyvesant' 'Boerum Hill'\n 'Bowery' 'Brooklyn Heights' 'Brooklyn Navy Yard' 'Central Park' 'Chelsea'\n 'Chinatown' 'Civic Center' 'Clinton Hill'\n 'Columbia Street Waterfront District' 'Downtown Brooklyn' 'Dumbo'\n 'East Village' 'Financial District' 'Flatiron District' 'Fort Greene'\n 'Fulton Ferry District' 'Garment District' 'Gramercy Park'\n 'Greenwich Village' \"Hell's Kitchen\" 'Hudson Square' 'Kips Bay'\n 'Korea Town' 'Lincoln Square' 'Little Italy' 'Lower East Side'\n 'Lower Manhattan' 'Meatpacking District' 'Midtown' 'Midtown East'\n 'Midtown West' 'Murray Hill' 'NoHo' 'NoMad' 'Nolita' 'Park Slope'\n 'Peter Cooper Village' 'Rose Hill' 'SoHo' 'Stuyvesant Town'\n 'Sutton Place' 'Theater District' 'Tribeca' 'Tudor City' 'Two Bridges'\n 'Ukrainian Village' 'Union Square' 'Vinegar Hill' 'West Village'\n 'Williamsburg']"
     ]
    }
   ],
   "source": [
    "parts = fu.get_partitions(list(range(X_transformed.shape[0])), slice_size=1000)\n",
    "loss_vec = []\n",
    "for part in tqdm(parts):\n",
    "    i, j = part[0], part[-1]   \n",
    "\n",
    "    clf.fit(X_transformed[i:j], y_enc[i:j])\n",
    "    \n",
    "    y_prob_vec = clf.predict_proba(X_test_transformed)\n",
    "    loss = fu.big_logloss(y_test_enc, y_prob_vec, labels)\n",
    "    loss_vec.append(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(54, (210854,), (210854, 53), 53)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#loss = fu.big_logloss(y_test_enc, y_prob_vec, labels)\n",
    "len(labels), y_test_enc.shape, y_prob_vec.shape, len(Counter(y_enc[i:j]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "632562\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "54"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Ok... going to try shuffling first instead.. \n",
    "# so the len(Counter(y_enc[i:j])) ,  hopefully , of the first fit , has 54 not 53.\n",
    "\n",
    "# Ok I shuffled about four times until I ended up where the first part below here was '54'\n",
    "\n",
    "size = X_transformed.shape[0]\n",
    "print(size)\n",
    "indices = np.random.choice(range(size), size=size, replace=False)\n",
    "# X_train_shfl = X_transformed[indices]\n",
    "# Ylabels_train_shfl = y_enc[indices]#.astype('int64')\n",
    "\n",
    "parts = fu.get_partitions(indices, slice_size=1000)\n",
    "len(parts[0])\n",
    "len(Counter(y_enc[parts[0]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "workdir = fu.make_work_dir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "workdir /opt/program/artifacts/2020-06-16T033314Z\n"
     ]
    }
   ],
   "source": [
    "print(\"workdir\", workdir); fu.log(workdir, 'First line')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13204aa955cf4dcea2a984f83fe084bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=633.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "The number of classes in labels is different from that in y_pred. Classes found in labels: ['Alphabet City' 'Battery Park City' 'Bedford-Stuyvesant' 'Boerum Hill'\n 'Bowery' 'Brooklyn Heights' 'Brooklyn Navy Yard' 'Central Park' 'Chelsea'\n 'Chinatown' 'Civic Center' 'Clinton Hill'\n 'Columbia Street Waterfront District' 'Downtown Brooklyn' 'Dumbo'\n 'East Village' 'Financial District' 'Flatiron District' 'Fort Greene'\n 'Fulton Ferry District' 'Garment District' 'Gramercy Park'\n 'Greenwich Village' \"Hell's Kitchen\" 'Hudson Square' 'Kips Bay'\n 'Korea Town' 'Lincoln Square' 'Little Italy' 'Lower East Side'\n 'Lower Manhattan' 'Meatpacking District' 'Midtown' 'Midtown East'\n 'Midtown West' 'Murray Hill' 'NoHo' 'NoMad' 'Nolita' 'Park Slope'\n 'Peter Cooper Village' 'Rose Hill' 'SoHo' 'Stuyvesant Town'\n 'Sutton Place' 'Theater District' 'Tribeca' 'Tudor City' 'Two Bridges'\n 'Ukrainian Village' 'Union Square' 'Vinegar Hill' 'West Village'\n 'Williamsburg']",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-33-5be8c779daa1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mfu\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mworkdir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf'[{i}] Done predict_proba'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfu\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbig_logloss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_test_enc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_prob_vec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m     \u001b[0mfu\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mworkdir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf'[{i}] Done big_logloss'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/program/fresh/utils.py\u001b[0m in \u001b[0;36mbig_logloss\u001b[0;34m(y, y_prob, labels)\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mpart\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mget_partitions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_prob\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mslice_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpart\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpart\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m         \u001b[0mlosses_vec\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlog_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_prob\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlosses_vec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/sklearn/metrics/_classification.py\u001b[0m in \u001b[0;36mlog_loss\u001b[0;34m(y_true, y_pred, eps, normalize, sample_weight, labels)\u001b[0m\n\u001b[1;32m   2289\u001b[0m             raise ValueError('The number of classes in labels is different '\n\u001b[1;32m   2290\u001b[0m                              \u001b[0;34m'from that in y_pred. Classes found in '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2291\u001b[0;31m                              'labels: {0}'.format(lb.classes_))\n\u001b[0m\u001b[1;32m   2292\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2293\u001b[0m     \u001b[0;31m# Renormalize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: The number of classes in labels is different from that in y_pred. Classes found in labels: ['Alphabet City' 'Battery Park City' 'Bedford-Stuyvesant' 'Boerum Hill'\n 'Bowery' 'Brooklyn Heights' 'Brooklyn Navy Yard' 'Central Park' 'Chelsea'\n 'Chinatown' 'Civic Center' 'Clinton Hill'\n 'Columbia Street Waterfront District' 'Downtown Brooklyn' 'Dumbo'\n 'East Village' 'Financial District' 'Flatiron District' 'Fort Greene'\n 'Fulton Ferry District' 'Garment District' 'Gramercy Park'\n 'Greenwich Village' \"Hell's Kitchen\" 'Hudson Square' 'Kips Bay'\n 'Korea Town' 'Lincoln Square' 'Little Italy' 'Lower East Side'\n 'Lower Manhattan' 'Meatpacking District' 'Midtown' 'Midtown East'\n 'Midtown West' 'Murray Hill' 'NoHo' 'NoMad' 'Nolita' 'Park Slope'\n 'Peter Cooper Village' 'Rose Hill' 'SoHo' 'Stuyvesant Town'\n 'Sutton Place' 'Theater District' 'Tribeca' 'Tudor City' 'Two Bridges'\n 'Ukrainian Village' 'Union Square' 'Vinegar Hill' 'West Village'\n 'Williamsburg']"
     ]
    }
   ],
   "source": [
    "fu.log(workdir, 'Starting')\n",
    "loss_vec = []\n",
    "for i, part in enumerate(tqdm(parts)):\n",
    "    clf.fit(X_transformed[part], y_enc[part])\n",
    "    fu.log(workdir, f'[{i}] Done fit')\n",
    "    \n",
    "    y_prob_vec = clf.predict_proba(X_test_transformed)\n",
    "    fu.log(workdir, f'[{i}] Done predict_proba')\n",
    "    \n",
    "    loss = fu.big_logloss(y_test_enc, y_prob_vec, labels)\n",
    "    fu.log(workdir, f'[{i}] Done big_logloss')\n",
    "    \n",
    "    loss_vec.append(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Observing here, this first log line from the file here , \"2020-06-16 03:36:20Z, Starting\" , \n",
    "# and I see that the big_logloss() first part  did not crash here. \n",
    "# ...   but loss vec ... hmm alll zeroes... not learning? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.0, 0.0]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((210854,), (210854, 52), (54,))"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_enc.shape, y_prob_vec.shape, labels.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Trying out that model save "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Since the output of y_prob_vec is showing a size of 52, this makes me think the fit() is starting from scratch\n",
    "# ... I can at least quickly try out that save ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/opt/program/artifacts/2020-06-16T035733Z\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c26c58199ccc4d3aa9631cc5730f7e82",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=633.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "workdir = fu.make_work_dir(); print(workdir)\n",
    "fu.log(workdir, 'Starting')\n",
    "prev_model = None\n",
    "loss_vec = []; acc_vec = []\n",
    "for i, part in enumerate(tqdm(parts)):\n",
    "    clf.fit(X_transformed[part], y_enc[part], xgb_model=prev_model)\n",
    "    fu.log(workdir, f'[{i}] Done fit')\n",
    "    \n",
    "    prev_model = f'{workdir}/model.xg'\n",
    "    clf.save_model(prev_model)\n",
    "    \n",
    "    y_prob_vec = clf.predict_proba(X_test_transformed)\n",
    "    fu.log(workdir, f'[{i}] Done predict_proba')\n",
    "    \n",
    "    loss = fu.big_logloss(y_test_enc, y_prob_vec, labels)\n",
    "    fu.log(workdir, f'[{i}] Done big_logloss, loss={loss}.')\n",
    "    \n",
    "    loss_vec.append(loss)\n",
    "    \n",
    "    acc = accuracy_score(y_test_enc, np.argmax(y_prob_vec, axis=1))\n",
    "    acc_vec.append(acc)\n",
    "    fu.log(workdir, f'[{i}] Done accuracy, acc={acc}.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### output logs\n",
    "My logs, below, are showing the predict_proba part is what was taking the bulk of the time!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "```\n",
    "(pandars3) $ tail -f artifacts/2020-06-16T035733Z/work.log \n",
    "2020-06-16 03:57:33Z, Starting\n",
    "2020-06-16 03:57:35Z, [0] Done fit\n",
    "2020-06-16 03:59:10Z, [0] Done predict_proba\n",
    "2020-06-16 03:59:10Z, [0] Done big_logloss, loss=0.0.\n",
    "2020-06-16 03:59:10Z, [0] Done accuracy, acc=0.11610403407096853.\n",
    "2020-06-16 03:59:13Z, [1] Done fit\n",
    "2020-06-16 04:02:27Z, [1] Done predict_proba\n",
    "2020-06-16 04:02:27Z, [1] Done big_logloss, loss=0.0.\n",
    "2020-06-16 04:02:27Z, [1] Done accuracy, acc=0.11933849962533317.\n",
    "2020-06-16 04:02:31Z, [2] Done fit\n",
    "2020-06-16 04:09:21Z, [2] Done predict_proba\n",
    "2020-06-16 04:09:21Z, [2] Done big_logloss, loss=0.0.\n",
    "2020-06-16 04:09:21Z, [2] Done accuracy, acc=0.11234313790584954.\n",
    "2020-06-16 04:09:27Z, [3] Done fit\n",
    "2020-06-16 04:19:43Z, [3] Done predict_proba\n",
    "2020-06-16 04:19:43Z, [3] Done big_logloss, loss=0.0.\n",
    "2020-06-16 04:19:43Z, [3] Done accuracy, acc=0.0981295114154818.\n",
    "2020-06-16 04:19:52Z, [4] Done fit\n",
    "2020-06-16 04:35:20Z, [4] Done predict_proba\n",
    "2020-06-16 04:35:20Z, [4] Done big_logloss, loss=0.0.\n",
    "2020-06-16 04:35:20Z, [4] Done accuracy, acc=0.015944682102307758.\n",
    "2020-06-16 04:35:31Z, [5] Done fit\n",
    "2020-06-16 04:56:32Z, [5] Done predict_proba\n",
    "2020-06-16 04:56:33Z, [5] Done big_logloss, loss=0.0.\n",
    "2020-06-16 04:56:33Z, [5] Done accuracy, acc=0.10582678061597124.\n",
    "2020-06-16 04:56:47Z, [6] Done fit\n",
    "2020-06-16 05:22:41Z, [6] Done predict_proba\n",
    "2020-06-16 05:22:41Z, [6] Done big_logloss, loss=0.0.\n",
    "2020-06-16 05:22:41Z, [6] Done accuracy, acc=0.109895946958559.\n",
    "2020-06-16 05:22:58Z, [7] Done fit\n",
    "2020-06-16 05:53:38Z, [7] Done predict_proba\n",
    "2020-06-16 05:53:38Z, [7] Done big_logloss, loss=0.0.\n",
    "2020-06-16 05:53:38Z, [7] Done accuracy, acc=0.1033795896686807.\n",
    "2020-06-16 05:53:58Z, [8] Done fit\n",
    "2020-06-16 06:29:45Z, [8] Done predict_proba\n",
    "2020-06-16 06:29:45Z, [8] Done big_logloss, loss=0.0.\n",
    "2020-06-16 06:29:45Z, [8] Done accuracy, acc=0.020260464586870537.\n",
    "2020-06-16 06:30:08Z, [9] Done fit\n",
    "2020-06-16 07:10:49Z, [9] Done predict_proba\n",
    "2020-06-16 07:10:49Z, [9] Done big_logloss, loss=0.0.\n",
    "2020-06-16 07:10:49Z, [9] Done accuracy, acc=0.12078499815037894.\n",
    "2020-06-16 07:11:14Z, [10] Done fit\n",
    "2020-06-16 07:57:28Z, [10] Done predict_proba\n",
    "2020-06-16 07:57:28Z, [10] Done big_logloss, loss=0.0.\n",
    "2020-06-16 07:57:28Z, [10] Done accuracy, acc=0.12298557295569446.\n",
    "2020-06-16 07:57:56Z, [11] Done fit\n",
    "2020-06-16 08:49:22Z, [11] Done predict_proba\n",
    "2020-06-16 08:49:23Z, [11] Done big_logloss, loss=0.0.\n",
    "2020-06-16 08:49:23Z, [11] Done accuracy, acc=0.1153736708812733.\n",
    "2020-06-16 08:49:54Z, [12] Done fit\n",
    "2020-06-16 09:47:09Z, [12] Done predict_proba\n",
    "2020-06-16 09:47:09Z, [12] Done big_logloss, loss=0.0.\n",
    "2020-06-16 09:47:09Z, [12] Done accuracy, acc=0.11813861724226242.\n",
    "2020-06-16 09:47:44Z, [13] Done fit\n",
    "2020-06-16 10:50:05Z, [13] Done predict_proba\n",
    "2020-06-16 10:50:06Z, [13] Done big_logloss, loss=0.0.\n",
    "2020-06-16 10:50:06Z, [13] Done accuracy, acc=0.11920570631811585.\n",
    "2020-06-16 10:50:45Z, [14] Done fit\n",
    "2020-06-16 12:02:35Z, [14] Done predict_proba\n",
    "2020-06-16 12:02:35Z, [14] Done big_logloss, loss=0.0.\n",
    "2020-06-16 12:02:35Z, [14] Done accuracy, acc=0.11395088544680206.\n",
    "2020-06-16 12:03:16Z, [15] Done fit\n",
    "2020-06-16 13:22:31Z, [15] Done predict_proba\n",
    "2020-06-16 13:22:31Z, [15] Done big_logloss, loss=0.0.\n",
    "2020-06-16 13:22:31Z, [15] Done accuracy, acc=0.11543532491676706.\n",
    "2020-06-16 13:23:15Z, [16] Done fit\n",
    "2020-06-16 14:47:58Z, [16] Done predict_proba\n",
    "2020-06-16 14:47:58Z, [16] Done big_logloss, loss=0.0.\n",
    "2020-06-16 14:47:58Z, [16] Done accuracy, acc=0.026767336640519033.\n",
    "2020-06-16 14:48:46Z, [17] Done fit\n",
    "2020-06-16 16:19:31Z, [17] Done predict_proba\n",
    "2020-06-16 16:19:32Z, [17] Done big_logloss, loss=0.0.\n",
    "2020-06-16 16:19:32Z, [17] Done accuracy, acc=0.11969419598395098.\n",
    "2020-06-16 16:20:23Z, [18] Done fit\n",
    "\n",
    "\n",
    "\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# still not sure why loss 0 tho. (EDIT: later learned this was because I was passing class name labels \n",
    "# and not encoded labels. Somehow instead of crashing that just produces a 0.0 log loss )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}