2020-06-04-pure-prior-probability-model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "repodir = os.getenv('REPO_DIR')\n",
    "tripsdf = pd.read_csv(f'{repodir}/data/citibike/2013-07 - Citi Bike trip data.csv')\n",
    "stationsdf = pd.read_csv(f'{repodir}/learn-citibike/datas/stations/stations-2018-12-04-c.csv',\n",
    "                        index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tripduration</th>\n",
       "      <th>starttime</th>\n",
       "      <th>stoptime</th>\n",
       "      <th>start station id</th>\n",
       "      <th>start station name</th>\n",
       "      <th>start station latitude</th>\n",
       "      <th>start station longitude</th>\n",
       "      <th>end station id</th>\n",
       "      <th>end station name</th>\n",
       "      <th>end station latitude</th>\n",
       "      <th>end station longitude</th>\n",
       "      <th>bikeid</th>\n",
       "      <th>usertype</th>\n",
       "      <th>birth year</th>\n",
       "      <th>gender</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>634</td>\n",
       "      <td>2013-07-01 00:00:00</td>\n",
       "      <td>2013-07-01 00:10:34</td>\n",
       "      <td>164</td>\n",
       "      <td>E 47 St &amp; 2 Ave</td>\n",
       "      <td>40.753231</td>\n",
       "      <td>-73.970325</td>\n",
       "      <td>504</td>\n",
       "      <td>1 Ave &amp; E 15 St</td>\n",
       "      <td>40.732219</td>\n",
       "      <td>-73.981656</td>\n",
       "      <td>16950</td>\n",
       "      <td>Customer</td>\n",
       "      <td>\\N</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1547</td>\n",
       "      <td>2013-07-01 00:00:02</td>\n",
       "      <td>2013-07-01 00:25:49</td>\n",
       "      <td>388</td>\n",
       "      <td>W 26 St &amp; 10 Ave</td>\n",
       "      <td>40.749718</td>\n",
       "      <td>-74.002950</td>\n",
       "      <td>459</td>\n",
       "      <td>W 20 St &amp; 11 Ave</td>\n",
       "      <td>40.746745</td>\n",
       "      <td>-74.007756</td>\n",
       "      <td>19816</td>\n",
       "      <td>Customer</td>\n",
       "      <td>\\N</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>178</td>\n",
       "      <td>2013-07-01 00:01:04</td>\n",
       "      <td>2013-07-01 00:04:02</td>\n",
       "      <td>293</td>\n",
       "      <td>Lafayette St &amp; E 8 St</td>\n",
       "      <td>40.730287</td>\n",
       "      <td>-73.990765</td>\n",
       "      <td>237</td>\n",
       "      <td>E 11 St &amp; 2 Ave</td>\n",
       "      <td>40.730473</td>\n",
       "      <td>-73.986724</td>\n",
       "      <td>14548</td>\n",
       "      <td>Subscriber</td>\n",
       "      <td>1980</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1580</td>\n",
       "      <td>2013-07-01 00:01:06</td>\n",
       "      <td>2013-07-01 00:27:26</td>\n",
       "      <td>531</td>\n",
       "      <td>Forsyth St &amp; Broome St</td>\n",
       "      <td>40.718939</td>\n",
       "      <td>-73.992663</td>\n",
       "      <td>499</td>\n",
       "      <td>Broadway &amp; W 60 St</td>\n",
       "      <td>40.769155</td>\n",
       "      <td>-73.981918</td>\n",
       "      <td>16063</td>\n",
       "      <td>Customer</td>\n",
       "      <td>\\N</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>757</td>\n",
       "      <td>2013-07-01 00:01:10</td>\n",
       "      <td>2013-07-01 00:13:47</td>\n",
       "      <td>382</td>\n",
       "      <td>University Pl &amp; E 14 St</td>\n",
       "      <td>40.734927</td>\n",
       "      <td>-73.992005</td>\n",
       "      <td>410</td>\n",
       "      <td>Suffolk St &amp; Stanton St</td>\n",
       "      <td>40.720664</td>\n",
       "      <td>-73.985180</td>\n",
       "      <td>19213</td>\n",
       "      <td>Subscriber</td>\n",
       "      <td>1986</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tripduration            starttime             stoptime  start station id  \\\n",
       "0           634  2013-07-01 00:00:00  2013-07-01 00:10:34               164   \n",
       "1          1547  2013-07-01 00:00:02  2013-07-01 00:25:49               388   \n",
       "2           178  2013-07-01 00:01:04  2013-07-01 00:04:02               293   \n",
       "3          1580  2013-07-01 00:01:06  2013-07-01 00:27:26               531   \n",
       "4           757  2013-07-01 00:01:10  2013-07-01 00:13:47               382   \n",
       "\n",
       "        start station name  start station latitude  start station longitude  \\\n",
       "0          E 47 St & 2 Ave               40.753231               -73.970325   \n",
       "1         W 26 St & 10 Ave               40.749718               -74.002950   \n",
       "2    Lafayette St & E 8 St               40.730287               -73.990765   \n",
       "3   Forsyth St & Broome St               40.718939               -73.992663   \n",
       "4  University Pl & E 14 St               40.734927               -73.992005   \n",
       "\n",
       "   end station id         end station name  end station latitude  \\\n",
       "0             504          1 Ave & E 15 St             40.732219   \n",
       "1             459         W 20 St & 11 Ave             40.746745   \n",
       "2             237          E 11 St & 2 Ave             40.730473   \n",
       "3             499       Broadway & W 60 St             40.769155   \n",
       "4             410  Suffolk St & Stanton St             40.720664   \n",
       "\n",
       "   end station longitude  bikeid    usertype birth year  gender  \n",
       "0             -73.981656   16950    Customer         \\N       0  \n",
       "1             -74.007756   19816    Customer         \\N       0  \n",
       "2             -73.986724   14548  Subscriber       1980       2  \n",
       "3             -73.981918   16063    Customer         \\N       0  \n",
       "4             -73.985180   19213  Subscriber       1986       1  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tripsdf.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>station_name</th>\n",
       "      <th>start station latitude</th>\n",
       "      <th>start station longitude</th>\n",
       "      <th>latlng</th>\n",
       "      <th>neighborhood</th>\n",
       "      <th>postal_code</th>\n",
       "      <th>state</th>\n",
       "      <th>sublocality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Bank St &amp; Hudson St</td>\n",
       "      <td>40.736529</td>\n",
       "      <td>-74.006180</td>\n",
       "      <td>40.73652889,-74.00618026</td>\n",
       "      <td>West Village</td>\n",
       "      <td>10014</td>\n",
       "      <td>NY</td>\n",
       "      <td>Manhattan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>E 59 St &amp; Sutton Pl</td>\n",
       "      <td>40.758491</td>\n",
       "      <td>-73.959206</td>\n",
       "      <td>40.75849116,-73.95920622</td>\n",
       "      <td>Sutton Place</td>\n",
       "      <td>10022</td>\n",
       "      <td>NY</td>\n",
       "      <td>Manhattan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>E 37 St &amp; Lexington Ave</td>\n",
       "      <td>40.748238</td>\n",
       "      <td>-73.978311</td>\n",
       "      <td>40.748238,-73.978311</td>\n",
       "      <td>Murray Hill</td>\n",
       "      <td>10016</td>\n",
       "      <td>NY</td>\n",
       "      <td>Manhattan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9 Ave &amp; W 14 St</td>\n",
       "      <td>40.740583</td>\n",
       "      <td>-74.005509</td>\n",
       "      <td>40.7405826,-74.00550867</td>\n",
       "      <td>Meatpacking District</td>\n",
       "      <td>10014</td>\n",
       "      <td>NY</td>\n",
       "      <td>Manhattan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Broadway &amp; W 53 St</td>\n",
       "      <td>40.763441</td>\n",
       "      <td>-73.982681</td>\n",
       "      <td>40.76344058,-73.98268129</td>\n",
       "      <td>Midtown West</td>\n",
       "      <td>10019</td>\n",
       "      <td>NY</td>\n",
       "      <td>Manhattan</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              station_name  start station latitude  start station longitude  \\\n",
       "0      Bank St & Hudson St               40.736529               -74.006180   \n",
       "1      E 59 St & Sutton Pl               40.758491               -73.959206   \n",
       "2  E 37 St & Lexington Ave               40.748238               -73.978311   \n",
       "3          9 Ave & W 14 St               40.740583               -74.005509   \n",
       "4       Broadway & W 53 St               40.763441               -73.982681   \n",
       "\n",
       "                     latlng          neighborhood  postal_code state  \\\n",
       "0  40.73652889,-74.00618026          West Village        10014    NY   \n",
       "1  40.75849116,-73.95920622          Sutton Place        10022    NY   \n",
       "2      40.748238,-73.978311           Murray Hill        10016    NY   \n",
       "3   40.7405826,-74.00550867  Meatpacking District        10014    NY   \n",
       "4  40.76344058,-73.98268129          Midtown West        10019    NY   \n",
       "\n",
       "  sublocality  \n",
       "0   Manhattan  \n",
       "1   Manhattan  \n",
       "2   Manhattan  \n",
       "3   Manhattan  \n",
       "4   Manhattan  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stationsdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tripduration                               634\n",
       "starttime                  2013-07-01 00:00:00\n",
       "stoptime                   2013-07-01 00:10:34\n",
       "start station id                           164\n",
       "start station name             E 47 St & 2 Ave\n",
       "start station latitude                 40.7532\n",
       "start station longitude               -73.9703\n",
       "end station id                             504\n",
       "end station name               1 Ave & E 15 St\n",
       "end station latitude                   40.7322\n",
       "end station longitude                 -73.9817\n",
       "bikeid                                   16950\n",
       "usertype                              Customer\n",
       "birth year                                  \\N\n",
       "gender                                       0\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tripsdf.head().iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "mdf = tripsdf[['start station name', 'end station name']\n",
    "            ].merge(stationsdf[['station_name', 'neighborhood']], \n",
    "                    left_on='start station name',\n",
    "                    right_on='station_name'\n",
    "                   ).rename(columns={'neighborhood': 'start_neighborhood'}\n",
    "                           ).merge(stationsdf[['station_name', 'neighborhood']],\n",
    "                                  left_on='end station name',\n",
    "                                   right_on='station_name'\n",
    "                                  ).rename(columns={'neighborhood': 'end_neighborhood'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((843416, 15), (843416, 6))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tripsdf.shape, mdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "start station name    E 47 St & 2 Ave\n",
       "end station name      1 Ave & E 15 St\n",
       "station_name_x        E 47 St & 2 Ave\n",
       "start_neighborhood       Midtown East\n",
       "station_name_y        1 Ave & E 15 St\n",
       "end_neighborhood      Stuyvesant Town\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mdf.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "statsdf = mdf[['start_neighborhood', \n",
    "               'end_neighborhood']].groupby(by=['start_neighborhood', \n",
    "               'end_neighborhood']).size().reset_index().rename(columns={0: 'count'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "843416"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "statsdf['count'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>end_neighborhood</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Chelsea</td>\n",
       "      <td>92721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Midtown East</td>\n",
       "      <td>41296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Greenwich Village</td>\n",
       "      <td>40157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>Tribeca</td>\n",
       "      <td>39614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Hell's Kitchen</td>\n",
       "      <td>37192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Financial District</td>\n",
       "      <td>36062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>West Village</td>\n",
       "      <td>35993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Midtown West</td>\n",
       "      <td>30617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Lower East Side</td>\n",
       "      <td>26753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Midtown</td>\n",
       "      <td>26216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Kips Bay</td>\n",
       "      <td>24865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Battery Park City</td>\n",
       "      <td>24230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Theater District</td>\n",
       "      <td>21648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>Ukrainian Village</td>\n",
       "      <td>21165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Alphabet City</td>\n",
       "      <td>18885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Murray Hill</td>\n",
       "      <td>18124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Fort Greene</td>\n",
       "      <td>16678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Flatiron District</td>\n",
       "      <td>16067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Garment District</td>\n",
       "      <td>15557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Lower Manhattan</td>\n",
       "      <td>15120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Bowery</td>\n",
       "      <td>14947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Gramercy Park</td>\n",
       "      <td>13935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>East Village</td>\n",
       "      <td>13860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Downtown Brooklyn</td>\n",
       "      <td>13525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>SoHo</td>\n",
       "      <td>13417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Chinatown</td>\n",
       "      <td>12613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>Williamsburg</td>\n",
       "      <td>12197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Stuyvesant Town</td>\n",
       "      <td>11610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>Union Square</td>\n",
       "      <td>11456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Nolita</td>\n",
       "      <td>11394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Civic Center</td>\n",
       "      <td>11178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Hudson Square</td>\n",
       "      <td>9644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Lincoln Square</td>\n",
       "      <td>9320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Brooklyn Heights</td>\n",
       "      <td>8630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Meatpacking District</td>\n",
       "      <td>7881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Clinton Hill</td>\n",
       "      <td>7494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Rose Hill</td>\n",
       "      <td>6366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>NoMad</td>\n",
       "      <td>6060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Central Park</td>\n",
       "      <td>5811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Dumbo</td>\n",
       "      <td>5606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Two Bridges</td>\n",
       "      <td>5120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bedford-Stuyvesant</td>\n",
       "      <td>4510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Korea Town</td>\n",
       "      <td>3576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>NoHo</td>\n",
       "      <td>3546</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Boerum Hill</td>\n",
       "      <td>3372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Peter Cooper Village</td>\n",
       "      <td>2892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Little Italy</td>\n",
       "      <td>2874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Brooklyn Navy Yard</td>\n",
       "      <td>2575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Fulton Ferry District</td>\n",
       "      <td>2157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Park Slope</td>\n",
       "      <td>2040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Columbia Street Waterfront District</td>\n",
       "      <td>1904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>Tudor City</td>\n",
       "      <td>1222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Sutton Place</td>\n",
       "      <td>891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>Vinegar Hill</td>\n",
       "      <td>833</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       end_neighborhood  count\n",
       "8                               Chelsea  92721\n",
       "33                         Midtown East  41296\n",
       "22                    Greenwich Village  40157\n",
       "46                              Tribeca  39614\n",
       "23                       Hell's Kitchen  37192\n",
       "16                   Financial District  36062\n",
       "52                         West Village  35993\n",
       "34                         Midtown West  30617\n",
       "29                      Lower East Side  26753\n",
       "32                              Midtown  26216\n",
       "25                             Kips Bay  24865\n",
       "1                     Battery Park City  24230\n",
       "45                     Theater District  21648\n",
       "49                    Ukrainian Village  21165\n",
       "0                         Alphabet City  18885\n",
       "35                          Murray Hill  18124\n",
       "18                          Fort Greene  16678\n",
       "17                    Flatiron District  16067\n",
       "20                     Garment District  15557\n",
       "30                      Lower Manhattan  15120\n",
       "4                                Bowery  14947\n",
       "21                        Gramercy Park  13935\n",
       "15                         East Village  13860\n",
       "13                    Downtown Brooklyn  13525\n",
       "42                                 SoHo  13417\n",
       "9                             Chinatown  12613\n",
       "53                         Williamsburg  12197\n",
       "43                      Stuyvesant Town  11610\n",
       "50                         Union Square  11456\n",
       "38                               Nolita  11394\n",
       "10                         Civic Center  11178\n",
       "24                        Hudson Square   9644\n",
       "27                       Lincoln Square   9320\n",
       "5                      Brooklyn Heights   8630\n",
       "31                 Meatpacking District   7881\n",
       "11                         Clinton Hill   7494\n",
       "41                            Rose Hill   6366\n",
       "37                                NoMad   6060\n",
       "7                          Central Park   5811\n",
       "14                                Dumbo   5606\n",
       "48                          Two Bridges   5120\n",
       "2                    Bedford-Stuyvesant   4510\n",
       "26                           Korea Town   3576\n",
       "36                                 NoHo   3546\n",
       "3                           Boerum Hill   3372\n",
       "40                 Peter Cooper Village   2892\n",
       "28                         Little Italy   2874\n",
       "6                    Brooklyn Navy Yard   2575\n",
       "19                Fulton Ferry District   2157\n",
       "39                           Park Slope   2040\n",
       "12  Columbia Street Waterfront District   1904\n",
       "47                           Tudor City   1222\n",
       "44                         Sutton Place    891\n",
       "51                         Vinegar Hill    833"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# What is the most popular destination overall?\n",
    "mdf[['end_neighborhood']].groupby(by='end_neighborhood').size().reset_index(\n",
    "            ).rename(columns={0: 'count'}).sort_values(by='count', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((54,), (54,))"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mdf['start_neighborhood'].unique().shape, mdf['end_neighborhood'].unique().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.special import softmax\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x[0] [1.  0.5 0.2 3. ]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.10587707, 0.06421769, 0.04757363, 0.78233161])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([[1, 0.5, 0.2, 3],\n",
    "               [1,  -1,   7, 3],\n",
    "               [2,  12,  13, 3]])\n",
    "softmax(x, axis=1)[0].sum()\n",
    "print('x[0]', x[0])\n",
    "softmax(x[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def make_probs(xdf):\n",
    "    xdf['prob'] = softmax(xdf['count'])\n",
    "    xdf['prob2'] = xdf['prob'].map(lambda x:round(x, ndigits=4))\n",
    "    #return xdf[['end_neighborhood', 'prob']].T\n",
    "    #blahdf = statsdf[statsdf['start_neighborhood'] == 'Alphabet City'].copy()\n",
    "    #blahdf['prob'] = softmax(blahdf['count'])\n",
    "    #statsdf.groupby(by='start_neighborhood').apply()\n",
    "    return pd.DataFrame(dict(\n",
    "        list(xdf[['end_neighborhood', 'prob2']].to_records(index=False))),\n",
    "                       index=[0])\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2020-06-06T034021Z'"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import datetime; import pytz\n",
    "def utc_ts():\n",
    "    return datetime.datetime.utcnow(\n",
    "        ).replace(tzinfo=pytz.UTC).strftime('%Y-%m-%dT%H%M%SZ')\n",
    "utc_ts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "hmmdf = statsdf.groupby(by='start_neighborhood').apply(make_probs)\n",
    "work_dir = os.getenv('WORK_DIR')\n",
    "statsdf.to_csv(f'{work_dir}/2020-05-31--citilearn/{utc_ts()}-statsdf.csv')\n",
    "hmmdf.to_csv(f'{work_dir}/2020-05-31--citilearn/{utc_ts()}-hmmdf.csv')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Alphabet City</th>\n",
       "      <th>Battery Park City</th>\n",
       "      <th>Bedford-Stuyvesant</th>\n",
       "      <th>Boerum Hill</th>\n",
       "      <th>Bowery</th>\n",
       "      <th>Brooklyn Heights</th>\n",
       "      <th>Brooklyn Navy Yard</th>\n",
       "      <th>Central Park</th>\n",
       "      <th>Chelsea</th>\n",
       "      <th>Chinatown</th>\n",
       "      <th>...</th>\n",
       "      <th>Sutton Place</th>\n",
       "      <th>Theater District</th>\n",
       "      <th>Tribeca</th>\n",
       "      <th>Tudor City</th>\n",
       "      <th>Two Bridges</th>\n",
       "      <th>Ukrainian Village</th>\n",
       "      <th>Union Square</th>\n",
       "      <th>Vinegar Hill</th>\n",
       "      <th>West Village</th>\n",
       "      <th>Williamsburg</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>start_neighborhood</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Alphabet City</th>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Battery Park City</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bedford-Stuyvesant</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Boerum Hill</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bowery</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Brooklyn Heights</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Brooklyn Navy Yard</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Central Park</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Chelsea</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Chinatown</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Civic Center</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Clinton Hill</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Columbia Street Waterfront District</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Downtown Brooklyn</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dumbo</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.9933</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>East Village</th>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Financial District</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Flatiron District</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fort Greene</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fulton Ferry District</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Garment District</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Gramercy Park</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Greenwich Village</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hell's Kitchen</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hudson Square</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kips Bay</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Korea Town</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lincoln Square</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Little Italy</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lower East Side</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lower Manhattan</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Meatpacking District</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Midtown</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Midtown East</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Midtown West</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Murray Hill</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NoHo</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NoMad</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Nolita</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Park Slope</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Peter Cooper Village</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rose Hill</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SoHo</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stuyvesant Town</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sutton Place</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Theater District</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Tribeca</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Tudor City</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0025</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Two Bridges</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ukrainian Village</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Union Square</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Vinegar Hill</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>West Village</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Williamsburg</th>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>54 rows × 54 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Alphabet City  Battery Park City  \\\n",
       "start_neighborhood                                                        \n",
       "Alphabet City                       0            1.0                0.0   \n",
       "Battery Park City                   0            0.0                1.0   \n",
       "Bedford-Stuyvesant                  0            0.0                0.0   \n",
       "Boerum Hill                         0            0.0                0.0   \n",
       "Bowery                              0            0.0                0.0   \n",
       "Brooklyn Heights                    0            0.0                0.0   \n",
       "Brooklyn Navy Yard                  0            0.0                0.0   \n",
       "Central Park                        0            0.0                0.0   \n",
       "Chelsea                             0            0.0                0.0   \n",
       "Chinatown                           0            0.0                0.0   \n",
       "Civic Center                        0            0.0                0.0   \n",
       "Clinton Hill                        0            0.0                0.0   \n",
       "Columbia Street Waterfront District 0            0.0                0.0   \n",
       "Downtown Brooklyn                   0            0.0                0.0   \n",
       "Dumbo                               0            0.0                0.0   \n",
       "East Village                        0            1.0                0.0   \n",
       "Financial District                  0            0.0                0.0   \n",
       "Flatiron District                   0            0.0                0.0   \n",
       "Fort Greene                         0            0.0                0.0   \n",
       "Fulton Ferry District               0            0.0                0.0   \n",
       "Garment District                    0            0.0                0.0   \n",
       "Gramercy Park                       0            0.0                0.0   \n",
       "Greenwich Village                   0            0.0                0.0   \n",
       "Hell's Kitchen                      0            0.0                0.0   \n",
       "Hudson Square                       0            0.0                0.0   \n",
       "Kips Bay                            0            0.0                0.0   \n",
       "Korea Town                          0            0.0                0.0   \n",
       "Lincoln Square                      0            0.0                0.0   \n",
       "Little Italy                        0            0.0                0.0   \n",
       "Lower East Side                     0            0.0                0.0   \n",
       "Lower Manhattan                     0            0.0                0.0   \n",
       "Meatpacking District                0            0.0                0.0   \n",
       "Midtown                             0            0.0                0.0   \n",
       "Midtown East                        0            0.0                0.0   \n",
       "Midtown West                        0            0.0                0.0   \n",
       "Murray Hill                         0            0.0                0.0   \n",
       "NoHo                                0            0.0                0.0   \n",
       "NoMad                               0            0.0                0.0   \n",
       "Nolita                              0            0.0                0.0   \n",
       "Park Slope                          0            0.0                0.0   \n",
       "Peter Cooper Village                0            0.0                0.0   \n",
       "Rose Hill                           0            0.0                0.0   \n",
       "SoHo                                0            0.0                0.0   \n",
       "Stuyvesant Town                     0            0.0                0.0   \n",
       "Sutton Place                        0            0.0                0.0   \n",
       "Theater District                    0            0.0                0.0   \n",
       "Tribeca                             0            0.0                0.0   \n",
       "Tudor City                          0            0.0                0.0   \n",
       "Two Bridges                         0            0.0                0.0   \n",
       "Ukrainian Village                   0            0.0                0.0   \n",
       "Union Square                        0            0.0                0.0   \n",
       "Vinegar Hill                        0            0.0                0.0   \n",
       "West Village                        0            0.0                0.0   \n",
       "Williamsburg                        0            0.0                0.0   \n",
       "\n",
       "                                       Bedford-Stuyvesant  Boerum Hill  \\\n",
       "start_neighborhood                                                       \n",
       "Alphabet City                       0                 0.0          0.0   \n",
       "Battery Park City                   0                 0.0          0.0   \n",
       "Bedford-Stuyvesant                  0                 1.0          0.0   \n",
       "Boerum Hill                         0                 0.0          0.0   \n",
       "Bowery                              0                 0.0          0.0   \n",
       "Brooklyn Heights                    0                 0.0          0.0   \n",
       "Brooklyn Navy Yard                  0                 0.0          0.0   \n",
       "Central Park                        0                 0.0          NaN   \n",
       "Chelsea                             0                 0.0          0.0   \n",
       "Chinatown                           0                 0.0          0.0   \n",
       "Civic Center                        0                 0.0          0.0   \n",
       "Clinton Hill                        0                 0.0          0.0   \n",
       "Columbia Street Waterfront District 0                 0.0          0.0   \n",
       "Downtown Brooklyn                   0                 0.0          0.0   \n",
       "Dumbo                               0                 0.0          0.0   \n",
       "East Village                        0                 0.0          0.0   \n",
       "Financial District                  0                 0.0          0.0   \n",
       "Flatiron District                   0                 0.0          0.0   \n",
       "Fort Greene                         0                 0.0          0.0   \n",
       "Fulton Ferry District               0                 0.0          0.0   \n",
       "Garment District                    0                 0.0          0.0   \n",
       "Gramercy Park                       0                 0.0          0.0   \n",
       "Greenwich Village                   0                 0.0          0.0   \n",
       "Hell's Kitchen                      0                 0.0          0.0   \n",
       "Hudson Square                       0                 0.0          0.0   \n",
       "Kips Bay                            0                 0.0          0.0   \n",
       "Korea Town                          0                 0.0          0.0   \n",
       "Lincoln Square                      0                 0.0          0.0   \n",
       "Little Italy                        0                 0.0          0.0   \n",
       "Lower East Side                     0                 0.0          0.0   \n",
       "Lower Manhattan                     0                 0.0          0.0   \n",
       "Meatpacking District                0                 0.0          0.0   \n",
       "Midtown                             0                 0.0          0.0   \n",
       "Midtown East                        0                 0.0          0.0   \n",
       "Midtown West                        0                 0.0          0.0   \n",
       "Murray Hill                         0                 0.0          0.0   \n",
       "NoHo                                0                 0.0          0.0   \n",
       "NoMad                               0                 0.0          0.0   \n",
       "Nolita                              0                 0.0          0.0   \n",
       "Park Slope                          0                 0.0          0.0   \n",
       "Peter Cooper Village                0                 0.0          0.0   \n",
       "Rose Hill                           0                 0.0          0.0   \n",
       "SoHo                                0                 0.0          0.0   \n",
       "Stuyvesant Town                     0                 0.0          0.0   \n",
       "Sutton Place                        0                 0.0          0.0   \n",
       "Theater District                    0                 0.0          0.0   \n",
       "Tribeca                             0                 0.0          0.0   \n",
       "Tudor City                          0                 0.0          NaN   \n",
       "Two Bridges                         0                 0.0          0.0   \n",
       "Ukrainian Village                   0                 0.0          0.0   \n",
       "Union Square                        0                 0.0          0.0   \n",
       "Vinegar Hill                        0                 0.0          0.0   \n",
       "West Village                        0                 0.0          0.0   \n",
       "Williamsburg                        0                 0.0          0.0   \n",
       "\n",
       "                                       Bowery  Brooklyn Heights  \\\n",
       "start_neighborhood                                                \n",
       "Alphabet City                       0     0.0            0.0000   \n",
       "Battery Park City                   0     0.0            0.0000   \n",
       "Bedford-Stuyvesant                  0     0.0            0.0000   \n",
       "Boerum Hill                         0     0.0            0.0000   \n",
       "Bowery                              0     0.0            0.0000   \n",
       "Brooklyn Heights                    0     0.0            1.0000   \n",
       "Brooklyn Navy Yard                  0     0.0            0.0000   \n",
       "Central Park                        0     0.0            0.0000   \n",
       "Chelsea                             0     0.0            0.0000   \n",
       "Chinatown                           0     0.0            0.0000   \n",
       "Civic Center                        0     0.0            0.0000   \n",
       "Clinton Hill                        0     0.0            0.0000   \n",
       "Columbia Street Waterfront District 0     0.0            0.0000   \n",
       "Downtown Brooklyn                   0     0.0            0.0000   \n",
       "Dumbo                               0     0.0            0.9933   \n",
       "East Village                        0     0.0            0.0000   \n",
       "Financial District                  0     0.0            0.0000   \n",
       "Flatiron District                   0     0.0            0.0000   \n",
       "Fort Greene                         0     0.0            0.0000   \n",
       "Fulton Ferry District               0     0.0            0.0000   \n",
       "Garment District                    0     0.0            0.0000   \n",
       "Gramercy Park                       0     0.0            0.0000   \n",
       "Greenwich Village                   0     0.0            0.0000   \n",
       "Hell's Kitchen                      0     0.0            0.0000   \n",
       "Hudson Square                       0     0.0            0.0000   \n",
       "Kips Bay                            0     0.0            0.0000   \n",
       "Korea Town                          0     0.0            0.0000   \n",
       "Lincoln Square                      0     0.0            0.0000   \n",
       "Little Italy                        0     0.0            0.0000   \n",
       "Lower East Side                     0     0.0            0.0000   \n",
       "Lower Manhattan                     0     0.0            0.0000   \n",
       "Meatpacking District                0     0.0            0.0000   \n",
       "Midtown                             0     0.0            0.0000   \n",
       "Midtown East                        0     0.0            0.0000   \n",
       "Midtown West                        0     0.0            0.0000   \n",
       "Murray Hill                         0     0.0            0.0000   \n",
       "NoHo                                0     0.0            0.0000   \n",
       "NoMad                               0     0.0            0.0000   \n",
       "Nolita                              0     0.0            0.0000   \n",
       "Park Slope                          0     0.0            1.0000   \n",
       "Peter Cooper Village                0     0.0            0.0000   \n",
       "Rose Hill                           0     0.0            0.0000   \n",
       "SoHo                                0     0.0            0.0000   \n",
       "Stuyvesant Town                     0     0.0            0.0000   \n",
       "Sutton Place                        0     0.0            0.0000   \n",
       "Theater District                    0     0.0            0.0000   \n",
       "Tribeca                             0     0.0            0.0000   \n",
       "Tudor City                          0     0.0            0.0000   \n",
       "Two Bridges                         0     0.0            0.0000   \n",
       "Ukrainian Village                   0     0.0            0.0000   \n",
       "Union Square                        0     0.0            0.0000   \n",
       "Vinegar Hill                        0     0.0            0.0000   \n",
       "West Village                        0     0.0            0.0000   \n",
       "Williamsburg                        0     0.0            0.0000   \n",
       "\n",
       "                                       Brooklyn Navy Yard  Central Park  \\\n",
       "start_neighborhood                                                        \n",
       "Alphabet City                       0                 0.0           0.0   \n",
       "Battery Park City                   0                 0.0           0.0   \n",
       "Bedford-Stuyvesant                  0                 0.0           0.0   \n",
       "Boerum Hill                         0                 0.0           NaN   \n",
       "Bowery                              0                 0.0           0.0   \n",
       "Brooklyn Heights                    0                 0.0           0.0   \n",
       "Brooklyn Navy Yard                  0                 0.0           NaN   \n",
       "Central Park                        0                 0.0           1.0   \n",
       "Chelsea                             0                 0.0           0.0   \n",
       "Chinatown                           0                 0.0           0.0   \n",
       "Civic Center                        0                 0.0           0.0   \n",
       "Clinton Hill                        0                 0.0           NaN   \n",
       "Columbia Street Waterfront District 0                 0.0           0.0   \n",
       "Downtown Brooklyn                   0                 0.0           0.0   \n",
       "Dumbo                               0                 0.0           0.0   \n",
       "East Village                        0                 0.0           0.0   \n",
       "Financial District                  0                 0.0           0.0   \n",
       "Flatiron District                   0                 0.0           0.0   \n",
       "Fort Greene                         0                 0.0           0.0   \n",
       "Fulton Ferry District               0                 0.0           0.0   \n",
       "Garment District                    0                 0.0           0.0   \n",
       "Gramercy Park                       0                 0.0           0.0   \n",
       "Greenwich Village                   0                 0.0           0.0   \n",
       "Hell's Kitchen                      0                 0.0           0.0   \n",
       "Hudson Square                       0                 0.0           0.0   \n",
       "Kips Bay                            0                 0.0           0.0   \n",
       "Korea Town                          0                 0.0           0.0   \n",
       "Lincoln Square                      0                 0.0           0.0   \n",
       "Little Italy                        0                 0.0           0.0   \n",
       "Lower East Side                     0                 0.0           0.0   \n",
       "Lower Manhattan                     0                 0.0           0.0   \n",
       "Meatpacking District                0                 0.0           0.0   \n",
       "Midtown                             0                 0.0           0.0   \n",
       "Midtown East                        0                 0.0           0.0   \n",
       "Midtown West                        0                 0.0           0.0   \n",
       "Murray Hill                         0                 0.0           0.0   \n",
       "NoHo                                0                 0.0           0.0   \n",
       "NoMad                               0                 0.0           0.0   \n",
       "Nolita                              0                 0.0           0.0   \n",
       "Park Slope                          0                 0.0           0.0   \n",
       "Peter Cooper Village                0                 0.0           0.0   \n",
       "Rose Hill                           0                 NaN           0.0   \n",
       "SoHo                                0                 0.0           0.0   \n",
       "Stuyvesant Town                     0                 0.0           0.0   \n",
       "Sutton Place                        0                 0.0           0.0   \n",
       "Theater District                    0                 0.0           0.0   \n",
       "Tribeca                             0                 0.0           0.0   \n",
       "Tudor City                          0                 0.0           0.0   \n",
       "Two Bridges                         0                 0.0           0.0   \n",
       "Ukrainian Village                   0                 0.0           0.0   \n",
       "Union Square                        0                 0.0           0.0   \n",
       "Vinegar Hill                        0                 0.0           NaN   \n",
       "West Village                        0                 0.0           0.0   \n",
       "Williamsburg                        0                 0.0           0.0   \n",
       "\n",
       "                                       Chelsea  Chinatown  ...  Sutton Place  \\\n",
       "start_neighborhood                                         ...                 \n",
       "Alphabet City                       0   0.0000        0.0  ...           0.0   \n",
       "Battery Park City                   0   0.0000        0.0  ...           0.0   \n",
       "Bedford-Stuyvesant                  0   0.0000        0.0  ...           0.0   \n",
       "Boerum Hill                         0   0.0000        0.0  ...           0.0   \n",
       "Bowery                              0   0.0000        0.0  ...           0.0   \n",
       "Brooklyn Heights                    0   0.0000        0.0  ...           0.0   \n",
       "Brooklyn Navy Yard                  0   0.0000        0.0  ...           0.0   \n",
       "Central Park                        0   0.0000        0.0  ...           0.0   \n",
       "Chelsea                             0   1.0000        0.0  ...           0.0   \n",
       "Chinatown                           0   0.0000        0.0  ...           0.0   \n",
       "Civic Center                        0   0.0000        0.0  ...           0.0   \n",
       "Clinton Hill                        0   0.0000        0.0  ...           0.0   \n",
       "Columbia Street Waterfront District 0   0.0000        0.0  ...           0.0   \n",
       "Downtown Brooklyn                   0   0.0000        0.0  ...           0.0   \n",
       "Dumbo                               0   0.0000        0.0  ...           0.0   \n",
       "East Village                        0   0.0000        0.0  ...           0.0   \n",
       "Financial District                  0   0.0000        0.0  ...           0.0   \n",
       "Flatiron District                   0   1.0000        0.0  ...           0.0   \n",
       "Fort Greene                         0   0.0000        0.0  ...           0.0   \n",
       "Fulton Ferry District               0   0.0000        0.0  ...           0.0   \n",
       "Garment District                    0   1.0000        0.0  ...           0.0   \n",
       "Gramercy Park                       0   1.0000        0.0  ...           0.0   \n",
       "Greenwich Village                   0   0.0000        0.0  ...           0.0   \n",
       "Hell's Kitchen                      0   1.0000        0.0  ...           0.0   \n",
       "Hudson Square                       0   0.0000        0.0  ...           0.0   \n",
       "Kips Bay                            0   1.0000        0.0  ...           0.0   \n",
       "Korea Town                          0   1.0000        0.0  ...           0.0   \n",
       "Lincoln Square                      0   0.0001        0.0  ...           0.0   \n",
       "Little Italy                        0   0.0000        0.0  ...           0.0   \n",
       "Lower East Side                     0   0.0000        0.0  ...           0.0   \n",
       "Lower Manhattan                     0   1.0000        0.0  ...           0.0   \n",
       "Meatpacking District                0   1.0000        0.0  ...           0.0   \n",
       "Midtown                             0   1.0000        0.0  ...           0.0   \n",
       "Midtown East                        0   0.0000        0.0  ...           0.0   \n",
       "Midtown West                        0   0.0000        0.0  ...           0.0   \n",
       "Murray Hill                         0   1.0000        0.0  ...           0.0   \n",
       "NoHo                                0   0.0000        0.0  ...           0.0   \n",
       "NoMad                               0   1.0000        0.0  ...           0.0   \n",
       "Nolita                              0   0.0000        0.0  ...           0.0   \n",
       "Park Slope                          0   0.0000        0.0  ...           0.0   \n",
       "Peter Cooper Village                0   1.0000        0.0  ...           0.0   \n",
       "Rose Hill                           0   1.0000        0.0  ...           0.0   \n",
       "SoHo                                0   0.0000        0.0  ...           0.0   \n",
       "Stuyvesant Town                     0   1.0000        0.0  ...           0.0   \n",
       "Sutton Place                        0   0.0000        0.0  ...           0.0   \n",
       "Theater District                    0   1.0000        0.0  ...           0.0   \n",
       "Tribeca                             0   0.0000        0.0  ...           0.0   \n",
       "Tudor City                          0   0.0025        0.0  ...           0.0   \n",
       "Two Bridges                         0   0.0000        0.0  ...           0.0   \n",
       "Ukrainian Village                   0   0.0000        0.0  ...           0.0   \n",
       "Union Square                        0   1.0000        0.0  ...           0.0   \n",
       "Vinegar Hill                        0   0.0000        0.0  ...           0.0   \n",
       "West Village                        0   1.0000        0.0  ...           0.0   \n",
       "Williamsburg                        0   0.0000        0.0  ...           0.0   \n",
       "\n",
       "                                       Theater District  Tribeca  Tudor City  \\\n",
       "start_neighborhood                                                             \n",
       "Alphabet City                       0               0.0    0.000         0.0   \n",
       "Battery Park City                   0               0.0    0.000         0.0   \n",
       "Bedford-Stuyvesant                  0               0.0    0.000         0.0   \n",
       "Boerum Hill                         0               0.0    0.000         NaN   \n",
       "Bowery                              0               0.0    0.000         0.0   \n",
       "Brooklyn Heights                    0               0.0    0.000         0.0   \n",
       "Brooklyn Navy Yard                  0               0.0    0.000         NaN   \n",
       "Central Park                        0               0.0    0.000         0.0   \n",
       "Chelsea                             0               0.0    0.000         0.0   \n",
       "Chinatown                           0               0.0    0.000         0.0   \n",
       "Civic Center                        0               0.0    0.982         0.0   \n",
       "Clinton Hill                        0               0.0    0.000         NaN   \n",
       "Columbia Street Waterfront District 0               0.0    0.000         NaN   \n",
       "Downtown Brooklyn                   0               0.0    0.000         0.0   \n",
       "Dumbo                               0               0.0    0.000         0.0   \n",
       "East Village                        0               0.0    0.000         0.0   \n",
       "Financial District                  0               0.0    0.000         0.0   \n",
       "Flatiron District                   0               0.0    0.000         0.0   \n",
       "Fort Greene                         0               0.0    0.000         0.0   \n",
       "Fulton Ferry District               0               0.0    0.000         NaN   \n",
       "Garment District                    0               0.0    0.000         0.0   \n",
       "Gramercy Park                       0               0.0    0.000         0.0   \n",
       "Greenwich Village                   0               0.0    0.000         0.0   \n",
       "Hell's Kitchen                      0               0.0    0.000         0.0   \n",
       "Hudson Square                       0               0.0    1.000         0.0   \n",
       "Kips Bay                            0               0.0    0.000         0.0   \n",
       "Korea Town                          0               0.0    0.000         0.0   \n",
       "Lincoln Square                      0               0.0    0.000         0.0   \n",
       "Little Italy                        0               0.0    0.000         0.0   \n",
       "Lower East Side                     0               0.0    0.000         0.0   \n",
       "Lower Manhattan                     0               0.0    0.000         0.0   \n",
       "Meatpacking District                0               0.0    0.000         0.0   \n",
       "Midtown                             0               0.0    0.000         0.0   \n",
       "Midtown East                        0               0.0    0.000         0.0   \n",
       "Midtown West                        0               0.0    0.000         0.0   \n",
       "Murray Hill                         0               0.0    0.000         0.0   \n",
       "NoHo                                0               0.0    0.000         0.0   \n",
       "NoMad                               0               0.0    0.000         0.0   \n",
       "Nolita                              0               0.0    0.000         0.0   \n",
       "Park Slope                          0               0.0    0.000         NaN   \n",
       "Peter Cooper Village                0               0.0    0.000         0.0   \n",
       "Rose Hill                           0               0.0    0.000         0.0   \n",
       "SoHo                                0               0.0    0.000         0.0   \n",
       "Stuyvesant Town                     0               0.0    0.000         0.0   \n",
       "Sutton Place                        0               0.0    0.000         0.0   \n",
       "Theater District                    0               0.0    0.000         0.0   \n",
       "Tribeca                             0               0.0    1.000         0.0   \n",
       "Tudor City                          0               0.0    0.000         0.0   \n",
       "Two Bridges                         0               0.0    0.000         0.0   \n",
       "Ukrainian Village                   0               0.0    0.000         0.0   \n",
       "Union Square                        0               0.0    0.000         0.0   \n",
       "Vinegar Hill                        0               0.0    0.000         NaN   \n",
       "West Village                        0               0.0    0.000         0.0   \n",
       "Williamsburg                        0               0.0    0.000         0.0   \n",
       "\n",
       "                                       Two Bridges  Ukrainian Village  \\\n",
       "start_neighborhood                                                      \n",
       "Alphabet City                       0          0.0                0.0   \n",
       "Battery Park City                   0          0.0                0.0   \n",
       "Bedford-Stuyvesant                  0          0.0                0.0   \n",
       "Boerum Hill                         0          0.0                0.0   \n",
       "Bowery                              0          0.0                0.0   \n",
       "Brooklyn Heights                    0          0.0                0.0   \n",
       "Brooklyn Navy Yard                  0          0.0                0.0   \n",
       "Central Park                        0          0.0                0.0   \n",
       "Chelsea                             0          0.0                0.0   \n",
       "Chinatown                           0          0.0                0.0   \n",
       "Civic Center                        0          0.0                0.0   \n",
       "Clinton Hill                        0          0.0                0.0   \n",
       "Columbia Street Waterfront District 0          0.0                0.0   \n",
       "Downtown Brooklyn                   0          0.0                0.0   \n",
       "Dumbo                               0          0.0                0.0   \n",
       "East Village                        0          0.0                0.0   \n",
       "Financial District                  0          0.0                0.0   \n",
       "Flatiron District                   0          0.0                0.0   \n",
       "Fort Greene                         0          0.0                0.0   \n",
       "Fulton Ferry District               0          0.0                0.0   \n",
       "Garment District                    0          0.0                0.0   \n",
       "Gramercy Park                       0          0.0                0.0   \n",
       "Greenwich Village                   0          0.0                0.0   \n",
       "Hell's Kitchen                      0          0.0                0.0   \n",
       "Hudson Square                       0          0.0                0.0   \n",
       "Kips Bay                            0          0.0                0.0   \n",
       "Korea Town                          0          0.0                0.0   \n",
       "Lincoln Square                      0          0.0                0.0   \n",
       "Little Italy                        0          0.0                0.0   \n",
       "Lower East Side                     0          0.0                0.0   \n",
       "Lower Manhattan                     0          0.0                0.0   \n",
       "Meatpacking District                0          0.0                0.0   \n",
       "Midtown                             0          0.0                0.0   \n",
       "Midtown East                        0          0.0                0.0   \n",
       "Midtown West                        0          0.0                0.0   \n",
       "Murray Hill                         0          0.0                0.0   \n",
       "NoHo                                0          0.0                0.0   \n",
       "NoMad                               0          0.0                0.0   \n",
       "Nolita                              0          0.0                0.0   \n",
       "Park Slope                          0          0.0                0.0   \n",
       "Peter Cooper Village                0          0.0                0.0   \n",
       "Rose Hill                           0          0.0                0.0   \n",
       "SoHo                                0          0.0                0.0   \n",
       "Stuyvesant Town                     0          0.0                0.0   \n",
       "Sutton Place                        0          0.0                0.0   \n",
       "Theater District                    0          0.0                0.0   \n",
       "Tribeca                             0          0.0                0.0   \n",
       "Tudor City                          0          0.0                0.0   \n",
       "Two Bridges                         0          0.0                0.0   \n",
       "Ukrainian Village                   0          0.0                0.0   \n",
       "Union Square                        0          0.0                0.0   \n",
       "Vinegar Hill                        0          0.0                0.0   \n",
       "West Village                        0          0.0                0.0   \n",
       "Williamsburg                        0          0.0                0.0   \n",
       "\n",
       "                                       Union Square  Vinegar Hill  \\\n",
       "start_neighborhood                                                  \n",
       "Alphabet City                       0           0.0           0.0   \n",
       "Battery Park City                   0           0.0           NaN   \n",
       "Bedford-Stuyvesant                  0           0.0           0.0   \n",
       "Boerum Hill                         0           0.0           0.0   \n",
       "Bowery                              0           0.0           0.0   \n",
       "Brooklyn Heights                    0           0.0           0.0   \n",
       "Brooklyn Navy Yard                  0           0.0           0.0   \n",
       "Central Park                        0           0.0           NaN   \n",
       "Chelsea                             0           0.0           0.0   \n",
       "Chinatown                           0           0.0           0.0   \n",
       "Civic Center                        0           0.0           0.0   \n",
       "Clinton Hill                        0           0.0           0.0   \n",
       "Columbia Street Waterfront District 0           0.0           0.0   \n",
       "Downtown Brooklyn                   0           0.0           0.0   \n",
       "Dumbo                               0           0.0           0.0   \n",
       "East Village                        0           0.0           0.0   \n",
       "Financial District                  0           0.0           0.0   \n",
       "Flatiron District                   0           0.0           0.0   \n",
       "Fort Greene                         0           0.0           0.0   \n",
       "Fulton Ferry District               0           0.0           0.0   \n",
       "Garment District                    0           0.0           NaN   \n",
       "Gramercy Park                       0           0.0           0.0   \n",
       "Greenwich Village                   0           0.0           0.0   \n",
       "Hell's Kitchen                      0           0.0           NaN   \n",
       "Hudson Square                       0           0.0           0.0   \n",
       "Kips Bay                            0           0.0           0.0   \n",
       "Korea Town                          0           0.0           0.0   \n",
       "Lincoln Square                      0           0.0           NaN   \n",
       "Little Italy                        0           0.0           0.0   \n",
       "Lower East Side                     0           0.0           0.0   \n",
       "Lower Manhattan                     0           0.0           0.0   \n",
       "Meatpacking District                0           0.0           NaN   \n",
       "Midtown                             0           0.0           0.0   \n",
       "Midtown East                        0           0.0           0.0   \n",
       "Midtown West                        0           0.0           0.0   \n",
       "Murray Hill                         0           0.0           0.0   \n",
       "NoHo                                0           0.0           0.0   \n",
       "NoMad                               0           0.0           0.0   \n",
       "Nolita                              0           0.0           0.0   \n",
       "Park Slope                          0           0.0           0.0   \n",
       "Peter Cooper Village                0           0.0           0.0   \n",
       "Rose Hill                           0           0.0           NaN   \n",
       "SoHo                                0           0.0           0.0   \n",
       "Stuyvesant Town                     0           0.0           NaN   \n",
       "Sutton Place                        0           0.0           0.0   \n",
       "Theater District                    0           0.0           NaN   \n",
       "Tribeca                             0           0.0           0.0   \n",
       "Tudor City                          0           0.0           0.0   \n",
       "Two Bridges                         0           0.0           0.0   \n",
       "Ukrainian Village                   0           0.0           0.0   \n",
       "Union Square                        0           0.0           NaN   \n",
       "Vinegar Hill                        0           0.0           0.0   \n",
       "West Village                        0           0.0           0.0   \n",
       "Williamsburg                        0           0.0           0.0   \n",
       "\n",
       "                                       West Village  Williamsburg  \n",
       "start_neighborhood                                                 \n",
       "Alphabet City                       0           0.0           0.0  \n",
       "Battery Park City                   0           0.0           0.0  \n",
       "Bedford-Stuyvesant                  0           0.0           0.0  \n",
       "Boerum Hill                         0           0.0           0.0  \n",
       "Bowery                              0           0.0           0.0  \n",
       "Brooklyn Heights                    0           0.0           0.0  \n",
       "Brooklyn Navy Yard                  0           0.0           0.0  \n",
       "Central Park                        0           0.0           0.0  \n",
       "Chelsea                             0           0.0           0.0  \n",
       "Chinatown                           0           0.0           0.0  \n",
       "Civic Center                        0           0.0           0.0  \n",
       "Clinton Hill                        0           0.0           0.0  \n",
       "Columbia Street Waterfront District 0           0.0           0.0  \n",
       "Downtown Brooklyn                   0           0.0           0.0  \n",
       "Dumbo                               0           0.0           0.0  \n",
       "East Village                        0           0.0           0.0  \n",
       "Financial District                  0           0.0           0.0  \n",
       "Flatiron District                   0           0.0           0.0  \n",
       "Fort Greene                         0           0.0           0.0  \n",
       "Fulton Ferry District               0           0.0           0.0  \n",
       "Garment District                    0           0.0           0.0  \n",
       "Gramercy Park                       0           0.0           0.0  \n",
       "Greenwich Village                   0           0.0           0.0  \n",
       "Hell's Kitchen                      0           0.0           0.0  \n",
       "Hudson Square                       0           0.0           0.0  \n",
       "Kips Bay                            0           0.0           0.0  \n",
       "Korea Town                          0           0.0           0.0  \n",
       "Lincoln Square                      0           0.0           0.0  \n",
       "Little Italy                        0           0.0           0.0  \n",
       "Lower East Side                     0           0.0           0.0  \n",
       "Lower Manhattan                     0           0.0           0.0  \n",
       "Meatpacking District                0           0.0           0.0  \n",
       "Midtown                             0           0.0           0.0  \n",
       "Midtown East                        0           0.0           0.0  \n",
       "Midtown West                        0           0.0           0.0  \n",
       "Murray Hill                         0           0.0           0.0  \n",
       "NoHo                                0           0.0           0.0  \n",
       "NoMad                               0           0.0           0.0  \n",
       "Nolita                              0           0.0           0.0  \n",
       "Park Slope                          0           0.0           0.0  \n",
       "Peter Cooper Village                0           0.0           0.0  \n",
       "Rose Hill                           0           0.0           0.0  \n",
       "SoHo                                0           0.0           0.0  \n",
       "Stuyvesant Town                     0           0.0           0.0  \n",
       "Sutton Place                        0           0.0           0.0  \n",
       "Theater District                    0           0.0           0.0  \n",
       "Tribeca                             0           0.0           0.0  \n",
       "Tudor City                          0           0.0           0.0  \n",
       "Two Bridges                         0           0.0           0.0  \n",
       "Ukrainian Village                   0           0.0           0.0  \n",
       "Union Square                        0           0.0           0.0  \n",
       "Vinegar Hill                        0           0.0           0.0  \n",
       "West Village                        0           0.0           0.0  \n",
       "Williamsburg                        0           0.0           1.0  \n",
       "\n",
       "[54 rows x 54 columns]"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hmmdf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### So a super basic model that just predicts using the top prior probabity\n",
    "- Let's use that to choose some metrics and try to evaluate a model\n",
    "- So with this 54 neighborhood setup, we can say, a prediction on an input, \n",
    "is a vector of the probability of the 54 output neighborhoods. And we can make the evaluation\n",
    "metric the multilabel logloss.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.model_selection import train_test_split # (*arrays, **options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y \n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.21616187468057912"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import log_loss\n",
    "import numpy as np\n",
    "log_loss([\"spam\", \"ham\", \"ham\", \"spam\"],\n",
    "         [[.1, .9], [.9, .1], [.8, .2], [.35, .65]])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.1094237467877998e-15"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Expecting near 0 for a perfect prediction\n",
    "perfect_pred = np.array([[0, 0, 1],\n",
    "                   [1, 0, 0],\n",
    "                   [1, 0, 0],\n",
    "                   [0, 0, 1],\n",
    "                   [0, 1, 0]]) # [ham, jam, spam]\n",
    "log_loss([\"spam\", \"ham\", \"ham\", \"spam\", \"jam\"],\n",
    "         perfect_pred)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.1094237467877998e-15"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ok cool and using numberic labels okay too?\n",
    "perfect_pred = np.array([[0, 0, 1],\n",
    "                   [1, 0, 0],\n",
    "                   [1, 0, 0],\n",
    "                   [0, 0, 1],\n",
    "                   [0, 1, 0]]) # [ham, jam, spam]\n",
    "log_loss([3, 1, 1, 3, 2],\n",
    "         perfect_pred)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.07347496827220674"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# and some noisy predictions? \n",
    "noisy_pred = np.array([[.05, .05, .9],\n",
    "                   [.95, 0.05, 0],\n",
    "                   [.9, 0.1, 0],\n",
    "                   [0.05, .05, .9],\n",
    "                   [0, 1, 0]]) # [ham, jam, spam]\n",
    "log_loss([3, 1, 1, 3, 2],\n",
    "         noisy_pred)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.07347496827220787"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# and some noisy predictions? \n",
    "noisy_pred = np.array([[.05, .05, .9, 0],\n",
    "                   [.95, 0.05, 0, 0],\n",
    "                   [.9, 0.1, 0, 0],\n",
    "                   [0.05, .05, .9, 0],\n",
    "                   [0, 1, 0, 0]]) # [ham, jam, spam]\n",
    "log_loss([3, 1, 1, 3, 2],\n",
    "         noisy_pred,\n",
    "        labels=[1,2,3,4])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train this super dumb baseline model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['hi'], dtype='<U2')"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array([\"hi\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Manual cross validation\n",
    "- Doing this manually just because I had already written the train/fit using pandas\n",
    "- And I want to just quickly do this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((150, 4), (150,))"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import datasets\n",
    "from sklearn import svm\n",
    "\n",
    "X, y = datasets.load_iris(return_X_y=True)\n",
    "X.shape, y.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.96666667, 1.        , 0.96666667, 0.96666667, 1.        ])"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "clf = svm.SVC(kernel='linear', C=1)\n",
    "scores = cross_val_score(clf, X, y, cv=5)\n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> /usr/local/miniconda3/envs/pandars3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py(394)cross_val_score()\n",
      "-> scorer = check_scoring(estimator, scoring=scoring)\n",
      "(Pdb) l\n",
      "389  \t    :func:`sklearn.metrics.make_scorer`:\n",
      "390  \t        Make a scorer from a performance metric or loss function.\n",
      "391  \t\n",
      "392  \t    \"\"\"\n",
      "393  \t    # To ensure multimetric format is not supported\n",
      "394  ->\t    scorer = check_scoring(estimator, scoring=scoring)\n",
      "395  \t\n",
      "396  \t    cv_results = cross_validate(estimator=estimator, X=X, y=y, groups=groups,\n",
      "397  \t                                scoring={'score': scorer}, cv=cv,\n",
      "398  \t                                return_train_score=False,\n",
      "399  \t                                n_jobs=n_jobs, verbose=verbose,\n",
      "(Pdb) q\n"
     ]
    }
   ],
   "source": [
    "pdb.runcall(cross_val_score, clf, X, y, cv=5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FooFlassifiler():\n",
    "    def __init__(self):\n",
    "        return\n",
    "    def fit(self, X, y):\n",
    "        pass\n",
    "    def score(self, X, y):\n",
    "        return .9\n",
    "    def get_params(self, deep):\n",
    "        return {}\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = FooFlassifiler()\n",
    "scores = cross_val_score(clf, X, y, cv=5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.9, 0.9, 0.9, 0.9, 0.9])"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = mdf[['start_neighborhood']].values, mdf['end_neighborhood'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(  start_neighborhood\n",
       " 0       Midtown East\n",
       " 1       Midtown East\n",
       " 2       Midtown East\n",
       " 3       Midtown East\n",
       " 4       Midtown East,\n",
       " ['Stuyvesant Town',\n",
       "  'Stuyvesant Town',\n",
       "  'Stuyvesant Town',\n",
       "  'Stuyvesant Town',\n",
       "  'Stuyvesant Town'])"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(X[:5], columns=['start_neighborhood']), y[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['Midtown East', 'Stuyvesant Town'],\n",
       "       ['Midtown East', 'Stuyvesant Town'],\n",
       "       ['Midtown East', 'Stuyvesant Town'],\n",
       "       ['Midtown East', 'Stuyvesant Town'],\n",
       "       ['Midtown East', 'Stuyvesant Town']], dtype=object)"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.vstack((X[:5, 0], y[:5])).T\n",
    "#X[:5,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Stuyvesant Town', 'Stuyvesant Town', 'Stuyvesant Town',\n",
       "       'Stuyvesant Town', 'Stuyvesant Town'], dtype='<U15')"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(y[:5]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>start_neighborhood</th>\n",
       "      <th>end_neighborhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Midtown East</td>\n",
       "      <td>Stuyvesant Town</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Midtown East</td>\n",
       "      <td>Stuyvesant Town</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Midtown East</td>\n",
       "      <td>Stuyvesant Town</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Midtown East</td>\n",
       "      <td>Stuyvesant Town</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Midtown East</td>\n",
       "      <td>Stuyvesant Town</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  start_neighborhood end_neighborhood\n",
       "0       Midtown East  Stuyvesant Town\n",
       "1       Midtown East  Stuyvesant Town\n",
       "2       Midtown East  Stuyvesant Town\n",
       "3       Midtown East  Stuyvesant Town\n",
       "4       Midtown East  Stuyvesant Town"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(np.vstack((X[:5, 0], y[:5])).T, columns=['start_neighborhood',\n",
    "                                                   'end_neighborhood'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "split_data = model.iloc[:5].to_dict(orient='split')\n",
    "lookup = {a: np.array(split_data['data'])[i] for (i, a) in enumerate([x[0] for x in split_data['index']])}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "54"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(split_data['columns'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dumb_fit(X, y):\n",
    "    mdf = X_y_to_mydf(X, y)\n",
    "    statsdf = mdf[['start_neighborhood', \n",
    "               'end_neighborhood']].groupby(by=['start_neighborhood', \n",
    "               'end_neighborhood']).size().reset_index().rename(columns={0: 'count'})\n",
    "    hmmdf = statsdf.groupby(by='start_neighborhood').apply(make_probs).fillna(0)\n",
    "    \n",
    "    split_data = hmmdf.to_dict(orient='split')\n",
    "    lookup_dict = {a: np.array(split_data['data'])[i] \n",
    "              for (i, a) in enumerate([x[0] for x in split_data['index']])}\n",
    "    labels = split_data['columns']\n",
    "    return hmmdf, lookup_dict, labels\n",
    "    \n",
    "def X_y_to_mydf(X, y):\n",
    "    return pd.DataFrame(np.vstack((X[:, 0], y)).T, columns=['start_neighborhood',\n",
    "                                                   'end_neighborhood'])\n",
    "\n",
    "class SimpleFlassifiler():\n",
    "    def __init__(self):\n",
    "        return\n",
    "    def fit(self, X, y):\n",
    "        (self.lookup_df, self.lookup_dict, \n",
    "             self.labels) = dumb_fit(X, y)\n",
    "        \n",
    "    def score(self, X, y_true):\n",
    "        y_preds = predict(self.lookup_dict, X)\n",
    "        return log_loss(y_true, y_preds, labels=self.labels)\n",
    "\n",
    "    def get_params(self, deep):\n",
    "        return {}\n",
    "    \n",
    "def predict(lookup_dict, X):\n",
    "    # return np.concatenate([lookup_dict[x[0]] for x in X])\n",
    "    array_size = len(list(lookup_dict.values())[0])\n",
    "    return np.concatenate([np.reshape(lookup_dict[x[0]], (1, array_size)) \n",
    "           for x in X])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "54"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(list(lookup.values())[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29.131042558171746"
      ]
     },
     "execution_count": 238,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make data from mdf\n",
    "# (where mdf here just contains the ['start_neighborhood',\n",
    "#                                    'end_neighborhood'] )\n",
    "X, y = mdf[['start_neighborhood']].values, np.array(mdf['end_neighborhood'].tolist())\n",
    "clf = SimpleFlassifiler()\n",
    "clf.fit(X, y)\n",
    "#model = dumb_fit(X, y)\n",
    "#X.shape, y.shape\n",
    "clf.score(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "#clf.score(X, y)\n",
    "#(X[0][0])\n",
    "#'Midtown East' in \n",
    "#print(list(clf.lookup_dict.keys()))\n",
    "# y_preds = predict(clf.lookup_dict, X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29.131042558171746"
      ]
     },
     "execution_count": 235,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "#y_preds.shape, X.shape\n",
    "array_size = len(clf.lookup_dict)\n",
    "predarr = np.concatenate([np.reshape(clf.lookup_dict[x[0]], (1, array_size)) \n",
    "           for x in X])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([['Midtown East'],\n",
       "        ['Midtown East'],\n",
       "        ['Midtown East'],\n",
       "        ['Midtown East'],\n",
       "        ['Midtown East']], dtype=object),\n",
       " array(['Stuyvesant Town', 'Stuyvesant Town', 'Stuyvesant Town',\n",
       "        'Stuyvesant Town', 'Stuyvesant Town'], dtype='<U35'))"
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[:5], y[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ipdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "# clf = svm.SVC(kernel='linear', C=1)\n",
    "clf = SimpleFlassifiler()\n",
    "with ipdb.launch_ipdb_on_exception():\n",
    "    scores = cross_val_score(clf, X, y, cv=5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([29.03426394, 25.61716199, 29.19083979, 28.312853  , 22.04601817])"
      ]
     },
     "execution_count": 258,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>usertype</th>\n",
       "      <th>gender</th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Customer</td>\n",
       "      <td>0</td>\n",
       "      <td>174889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Customer</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Customer</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Subscriber</td>\n",
       "      <td>0</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Subscriber</td>\n",
       "      <td>1</td>\n",
       "      <td>511454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Subscriber</td>\n",
       "      <td>2</td>\n",
       "      <td>157005</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     usertype  gender       0\n",
       "0    Customer       0  174889\n",
       "1    Customer       1      25\n",
       "2    Customer       2       1\n",
       "3  Subscriber       0      42\n",
       "4  Subscriber       1  511454\n",
       "5  Subscriber       2  157005"
      ]
     },
     "execution_count": 263,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tripsdf.fillna('NaN').groupby(by=['usertype', 'gender']).size().reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020-06-13\n",
    "\n",
    "#### Redo some of the above ^^ to also just get a raw accuracy metric here\n",
    "- This is a Stopped notebook, so numbering is starting over here \n",
    "- Going to just copy/paste relevant snippets from above, which is painful, but the pain is brief!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "import fresh.v0_dumb as fv\n",
    "import fresh.utils as fu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'fresh.v0_dumb' from '/opt/program/fresh/v0_dumb.py'>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reload(fv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "datadir = '/opt/data'\n",
    "localdir = '/opt/program'\n",
    "tripsdf = pd.read_csv(f'{datadir}/2013-07 - Citi Bike trip data.csv')\n",
    "stationsdf = pd.read_csv(f'{localdir}/datas/stations/stations-2018-12-04-c.csv',\n",
    "                        index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X, y = mdf[['start_neighborhood']].values, np.array(mdf['end_neighborhood'].tolist())\n",
    "X, y, neighborhoods = fu.prepare_data(tripsdf, stationsdf)\n",
    "\n",
    "# Slight modification here b/c this data is built from the future notebook, \n",
    "#     2020-06-10-again , which uses one additional column\n",
    "X = X[:, :1] \n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "clf = fv.SimpleFlassifiler()\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_pred\n",
    "y_pred_class = np.argmax(y_pred, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(\"Hell's Kitchen\", 8),\n",
       " ('Greenwich Village', 8),\n",
       " ('Chinatown', 29),\n",
       " (\"Hell's Kitchen\", 33),\n",
       " ('Downtown Brooklyn', 12)]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(y_test[:5], y_pred_class[:5]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Alphabet City',\n",
       " 'Battery Park City',\n",
       " 'Bedford-Stuyvesant',\n",
       " 'Boerum Hill',\n",
       " 'Bowery']"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.labels[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[23, 22, 9, 23, 13]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[clf.labels.index(a) for a in y_test[:5]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_class = [clf.labels.index(a) for a in y_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(23, 8), (22, 8), (9, 29), (23, 33), (13, 12)]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(y_test_class[:5], y_pred_class[:5]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.15323873391066806"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_test_class, y_pred_class)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Note!\n",
    "This is good to have as a simple baseline"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}