ML_PumpItUp_BestScore.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": " ML_PumpItUp_BestScore.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "KwgxOudB2vgL",
        "CNuTAGjZ3F7a",
        "Pay9Tixy-GIh",
        "qh1UbzU6ltOE"
      ],
      "authorship_tag": "ABX9TyNeL0v133FhxNlHg7z3A5Zb",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Sachini-Dissanayaka/DrivenData_PumpItUp/blob/master/ML_PumpItUp_BestScore.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bsBPYksE000v"
      },
      "source": [
        "#**Getting Started**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p1oOf7B75C6C"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zGEp_ofO5KB1"
      },
      "source": [
        "%cd /content/drive/MyDrive/PumpIt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aZz3qarrMrhv"
      },
      "source": [
        "!pip install catboost\n",
        "!pip install category_encoders"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GkJ7D59GMyE9"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "from sklearn import preprocessing\n",
        "import datetime as dt\n",
        "from numpy import mean\n",
        "from numpy import std\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn import metrics\n",
        "from sklearn.metrics import classification_report, confusion_matrix\n",
        "from sklearn.impute import KNNImputer\n",
        "\n",
        "# encoders\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.preprocessing import OrdinalEncoder\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "import category_encoders as ce\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn import metrics\n",
        "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n",
        "from sklearn.model_selection import RepeatedStratifiedKFold\n",
        "from sklearn.model_selection import cross_val_score\n",
        "\n",
        "# classifiers\n",
        "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, StackingClassifier, VotingClassifier, GradientBoostingClassifier\n",
        "from xgboost.sklearn import XGBClassifier\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "import lightgbm as lgb\n",
        "from sklearn.linear_model import SGDClassifier\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from catboost import CatBoostClassifier\n",
        "from sklearn import svm\n",
        "\n",
        "# hyperparameter tuning\n",
        "from sklearn.model_selection import RandomizedSearchCV, GridSearchCV\n",
        "\n",
        "from sklearn.decomposition import PCA\n",
        "from matplotlib.pyplot import figure"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FI-DVKIe1Dyd"
      },
      "source": [
        "#**EDA**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KwgxOudB2vgL"
      },
      "source": [
        "#**Data Collection**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WnPyVCb4NYK1"
      },
      "source": [
        "features_df = pd.read_csv(\n",
        "    \"TrainingSetValues.csv\", \n",
        "    index_col=\"id\",\n",
        "    low_memory=False\n",
        ")\n",
        "labels_df = pd.read_csv(\n",
        "    \"TrainingSetLabels.csv\", \n",
        "    index_col=\"id\"\n",
        ")\n",
        "test_df = pd.read_csv(\n",
        "    \"TestSetValues.csv\", \n",
        "    index_col=\"id\",\n",
        "    low_memory=False\n",
        ")"
      ],
      "execution_count": 104,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uEePm45y5XSl",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 437
        },
        "outputId": "033e9b7d-c3fe-459e-fd28-d93ae867f183"
      },
      "source": [
        "features_df.head()"
      ],
      "execution_count": 105,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>amount_tsh</th>\n",
              "      <th>date_recorded</th>\n",
              "      <th>funder</th>\n",
              "      <th>gps_height</th>\n",
              "      <th>installer</th>\n",
              "      <th>longitude</th>\n",
              "      <th>latitude</th>\n",
              "      <th>wpt_name</th>\n",
              "      <th>num_private</th>\n",
              "      <th>basin</th>\n",
              "      <th>subvillage</th>\n",
              "      <th>region</th>\n",
              "      <th>region_code</th>\n",
              "      <th>district_code</th>\n",
              "      <th>lga</th>\n",
              "      <th>ward</th>\n",
              "      <th>population</th>\n",
              "      <th>public_meeting</th>\n",
              "      <th>recorded_by</th>\n",
              "      <th>scheme_management</th>\n",
              "      <th>scheme_name</th>\n",
              "      <th>permit</th>\n",
              "      <th>construction_year</th>\n",
              "      <th>extraction_type</th>\n",
              "      <th>extraction_type_group</th>\n",
              "      <th>extraction_type_class</th>\n",
              "      <th>management</th>\n",
              "      <th>management_group</th>\n",
              "      <th>payment</th>\n",
              "      <th>payment_type</th>\n",
              "      <th>water_quality</th>\n",
              "      <th>quality_group</th>\n",
              "      <th>quantity</th>\n",
              "      <th>quantity_group</th>\n",
              "      <th>source</th>\n",
              "      <th>source_type</th>\n",
              "      <th>source_class</th>\n",
              "      <th>waterpoint_type</th>\n",
              "      <th>waterpoint_type_group</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>69572</th>\n",
              "      <td>6000.0</td>\n",
              "      <td>2011-03-14</td>\n",
              "      <td>Roman</td>\n",
              "      <td>1390</td>\n",
              "      <td>Roman</td>\n",
              "      <td>34.938093</td>\n",
              "      <td>-9.856322</td>\n",
              "      <td>none</td>\n",
              "      <td>0</td>\n",
              "      <td>Lake Nyasa</td>\n",
              "      <td>Mnyusi B</td>\n",
              "      <td>Iringa</td>\n",
              "      <td>11</td>\n",
              "      <td>5</td>\n",
              "      <td>Ludewa</td>\n",
              "      <td>Mundindi</td>\n",
              "      <td>109</td>\n",
              "      <td>True</td>\n",
              "      <td>GeoData Consultants Ltd</td>\n",
              "      <td>VWC</td>\n",
              "      <td>Roman</td>\n",
              "      <td>False</td>\n",
              "      <td>1999</td>\n",
              "      <td>gravity</td>\n",
              "      <td>gravity</td>\n",
              "      <td>gravity</td>\n",
              "      <td>vwc</td>\n",
              "      <td>user-group</td>\n",
              "      <td>pay annually</td>\n",
              "      <td>annually</td>\n",
              "      <td>soft</td>\n",
              "      <td>good</td>\n",
              "      <td>enough</td>\n",
              "      <td>enough</td>\n",
              "      <td>spring</td>\n",
              "      <td>spring</td>\n",
              "      <td>groundwater</td>\n",
              "      <td>communal standpipe</td>\n",
              "      <td>communal standpipe</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8776</th>\n",
              "      <td>0.0</td>\n",
              "      <td>2013-03-06</td>\n",
              "      <td>Grumeti</td>\n",
              "      <td>1399</td>\n",
              "      <td>GRUMETI</td>\n",
              "      <td>34.698766</td>\n",
              "      <td>-2.147466</td>\n",
              "      <td>Zahanati</td>\n",
              "      <td>0</td>\n",
              "      <td>Lake Victoria</td>\n",
              "      <td>Nyamara</td>\n",
              "      <td>Mara</td>\n",
              "      <td>20</td>\n",
              "      <td>2</td>\n",
              "      <td>Serengeti</td>\n",
              "      <td>Natta</td>\n",
              "      <td>280</td>\n",
              "      <td>NaN</td>\n",
              "      <td>GeoData Consultants Ltd</td>\n",
              "      <td>Other</td>\n",
              "      <td>NaN</td>\n",
              "      <td>True</td>\n",
              "      <td>2010</td>\n",
              "      <td>gravity</td>\n",
              "      <td>gravity</td>\n",
              "      <td>gravity</td>\n",
              "      <td>wug</td>\n",
              "      <td>user-group</td>\n",
              "      <td>never pay</td>\n",
              "      <td>never pay</td>\n",
              "      <td>soft</td>\n",
              "      <td>good</td>\n",
              "      <td>insufficient</td>\n",
              "      <td>insufficient</td>\n",
              "      <td>rainwater harvesting</td>\n",
              "      <td>rainwater harvesting</td>\n",
              "      <td>surface</td>\n",
              "      <td>communal standpipe</td>\n",
              "      <td>communal standpipe</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34310</th>\n",
              "      <td>25.0</td>\n",
              "      <td>2013-02-25</td>\n",
              "      <td>Lottery Club</td>\n",
              "      <td>686</td>\n",
              "      <td>World vision</td>\n",
              "      <td>37.460664</td>\n",
              "      <td>-3.821329</td>\n",
              "      <td>Kwa Mahundi</td>\n",
              "      <td>0</td>\n",
              "      <td>Pangani</td>\n",
              "      <td>Majengo</td>\n",
              "      <td>Manyara</td>\n",
              "      <td>21</td>\n",
              "      <td>4</td>\n",
              "      <td>Simanjiro</td>\n",
              "      <td>Ngorika</td>\n",
              "      <td>250</td>\n",
              "      <td>True</td>\n",
              "      <td>GeoData Consultants Ltd</td>\n",
              "      <td>VWC</td>\n",
              "      <td>Nyumba ya mungu pipe scheme</td>\n",
              "      <td>True</td>\n",
              "      <td>2009</td>\n",
              "      <td>gravity</td>\n",
              "      <td>gravity</td>\n",
              "      <td>gravity</td>\n",
              "      <td>vwc</td>\n",
              "      <td>user-group</td>\n",
              "      <td>pay per bucket</td>\n",
              "      <td>per bucket</td>\n",
              "      <td>soft</td>\n",
              "      <td>good</td>\n",
              "      <td>enough</td>\n",
              "      <td>enough</td>\n",
              "      <td>dam</td>\n",
              "      <td>dam</td>\n",
              "      <td>surface</td>\n",
              "      <td>communal standpipe multiple</td>\n",
              "      <td>communal standpipe</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>67743</th>\n",
              "      <td>0.0</td>\n",
              "      <td>2013-01-28</td>\n",
              "      <td>Unicef</td>\n",
              "      <td>263</td>\n",
              "      <td>UNICEF</td>\n",
              "      <td>38.486161</td>\n",
              "      <td>-11.155298</td>\n",
              "      <td>Zahanati Ya Nanyumbu</td>\n",
              "      <td>0</td>\n",
              "      <td>Ruvuma / Southern Coast</td>\n",
              "      <td>Mahakamani</td>\n",
              "      <td>Mtwara</td>\n",
              "      <td>90</td>\n",
              "      <td>63</td>\n",
              "      <td>Nanyumbu</td>\n",
              "      <td>Nanyumbu</td>\n",
              "      <td>58</td>\n",
              "      <td>True</td>\n",
              "      <td>GeoData Consultants Ltd</td>\n",
              "      <td>VWC</td>\n",
              "      <td>NaN</td>\n",
              "      <td>True</td>\n",
              "      <td>1986</td>\n",
              "      <td>submersible</td>\n",
              "      <td>submersible</td>\n",
              "      <td>submersible</td>\n",
              "      <td>vwc</td>\n",
              "      <td>user-group</td>\n",
              "      <td>never pay</td>\n",
              "      <td>never pay</td>\n",
              "      <td>soft</td>\n",
              "      <td>good</td>\n",
              "      <td>dry</td>\n",
              "      <td>dry</td>\n",
              "      <td>machine dbh</td>\n",
              "      <td>borehole</td>\n",
              "      <td>groundwater</td>\n",
              "      <td>communal standpipe multiple</td>\n",
              "      <td>communal standpipe</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19728</th>\n",
              "      <td>0.0</td>\n",
              "      <td>2011-07-13</td>\n",
              "      <td>Action In A</td>\n",
              "      <td>0</td>\n",
              "      <td>Artisan</td>\n",
              "      <td>31.130847</td>\n",
              "      <td>-1.825359</td>\n",
              "      <td>Shuleni</td>\n",
              "      <td>0</td>\n",
              "      <td>Lake Victoria</td>\n",
              "      <td>Kyanyamisa</td>\n",
              "      <td>Kagera</td>\n",
              "      <td>18</td>\n",
              "      <td>1</td>\n",
              "      <td>Karagwe</td>\n",
              "      <td>Nyakasimbi</td>\n",
              "      <td>0</td>\n",
              "      <td>True</td>\n",
              "      <td>GeoData Consultants Ltd</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>True</td>\n",
              "      <td>0</td>\n",
              "      <td>gravity</td>\n",
              "      <td>gravity</td>\n",
              "      <td>gravity</td>\n",
              "      <td>other</td>\n",
              "      <td>other</td>\n",
              "      <td>never pay</td>\n",
              "      <td>never pay</td>\n",
              "      <td>soft</td>\n",
              "      <td>good</td>\n",
              "      <td>seasonal</td>\n",
              "      <td>seasonal</td>\n",
              "      <td>rainwater harvesting</td>\n",
              "      <td>rainwater harvesting</td>\n",
              "      <td>surface</td>\n",
              "      <td>communal standpipe</td>\n",
              "      <td>communal standpipe</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       amount_tsh  ... waterpoint_type_group\n",
              "id                 ...                      \n",
              "69572      6000.0  ...    communal standpipe\n",
              "8776          0.0  ...    communal standpipe\n",
              "34310        25.0  ...    communal standpipe\n",
              "67743         0.0  ...    communal standpipe\n",
              "19728         0.0  ...    communal standpipe\n",
              "\n",
              "[5 rows x 39 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 105
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dEQquKvz5kqb"
      },
      "source": [
        "X = features_df.copy()\n",
        "y = labels_df.copy()\n",
        "X_test = test_df.copy()"
      ],
      "execution_count": 106,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qcLsurGLN5BV"
      },
      "source": [
        "X.info()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SFP5LCmQkC0G"
      },
      "source": [
        "# Combine train data with test data\n",
        "df=X.append(X_test)"
      ],
      "execution_count": 108,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZLFjEggQ1o-c"
      },
      "source": [
        "df.describe()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m94Z7Msukr7S"
      },
      "source": [
        "y.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CNuTAGjZ3F7a"
      },
      "source": [
        "#**Data Cleaning**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "do_HTtbtOhWN"
      },
      "source": [
        "X_null = X.isnull().sum()\n",
        "test_null = X_test.isnull().sum()"
      ],
      "execution_count": 113,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NXp-aE9mOJZG"
      },
      "source": [
        "list_features = X_test.columns\n",
        "print('Number of features : ',len(list_features))\n",
        "print('')\n",
        "i = 0\n",
        "for f in list_features:\n",
        "    print('feature:', f, '|| Type:', type(X[f][0]), '|| Example:', X[f][0], '|| number of unique values', len(X[f].unique()), '|| missing values training : ', (X_null.values[i]/X.shape[0])*100, '|| missing values test : ', (test_null.values[i]/X_test.shape[0])*100 )\n",
        "    i +=1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m_uSefPqUBik"
      },
      "source": [
        "#finding Zero Percentage in each column\n",
        "all = X.shape[0]\n",
        "X_zero = X.astype(bool).sum(axis=0)\n",
        "list_X = X_test.columns\n",
        "print('Number of features : ',len(list_X))\n",
        "print('')\n",
        "i = 0\n",
        "for f in list_X:\n",
        "  if (((all - X_zero.values[i])/all)*100 > 0):\n",
        "    print('feature:', f, '|| Type:', type(X[f][0]),'|| Example:', X[f][0], '|| zero percentage', ((all - X_zero.values[i])/all)*100 )\n",
        "  i += 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e6UOgdcfl6yt"
      },
      "source": [
        "#remove columns with zeros and more missing values along with more unique values\n",
        "df.drop(columns=['num_private', 'scheme_name'], axis=1, inplace=True)"
      ],
      "execution_count": 116,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y36STGON7q4q"
      },
      "source": [
        "#drop unwanted columns\n",
        "df.drop(columns=['amount_tsh','recorded_by','wpt_name'], axis=1, inplace=True)"
      ],
      "execution_count": 117,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wyCsUWSpojaW"
      },
      "source": [
        "**Handling Missing Values**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pi-CvTJSojHt"
      },
      "source": [
        "# Number of missing values in each column\n",
        "missing_val_count_by_column = (df.isnull().sum())\n",
        "print(missing_val_count_by_column[missing_val_count_by_column > 0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9jR7c1tIpa9U"
      },
      "source": [
        "*   num_private - droped - only one unique value\n",
        "*   wpt_name - droped - number of unique values 37400\n",
        "*   num_private, scheme_name - droped\n",
        "*   Population - need to search more 36% zero count - relaced zeros with nan and mean\n",
        "*   funder - mode\n",
        "*   installer - mode\n",
        "*   subvilage - mode\n",
        "*   public_meeting - mode\n",
        "*   scheme_management - mode\n",
        "*   permit - mode\n",
        "*   construction year - replace zeros with nan and use mode\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lrEcWDbI2e-G"
      },
      "source": [
        "mis_columns = ['construction_year','population']\n",
        "for col in mis_columns:\n",
        "  df[col].replace(0, np.nan, inplace=True)"
      ],
      "execution_count": 119,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sPUO4ftHPFTo"
      },
      "source": [
        "# Fill missing values with mode\n",
        "def fill_mode(all_column,X):\n",
        "  for X_column in all_column:\n",
        "    X[X_column] = X[X_column].fillna(X[X_column].mode()[0])\n",
        "  return X"
      ],
      "execution_count": 121,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jy5nK_iWPGUD"
      },
      "source": [
        "df = fill_mode([\"funder\",\"installer\",\"subvillage\",\"public_meeting\",\"scheme_management\",\"permit\",\"construction_year\"],df)"
      ],
      "execution_count": 122,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AaeiRyflPsQp"
      },
      "source": [
        "# Fill missing values with mean\n",
        "def fill_mean(all_column,X):\n",
        "  for X_column in all_column:\n",
        "    X[X_column] = X[X_column].fillna(round(X[X_column].mean()))\n",
        "  return X"
      ],
      "execution_count": 123,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7VWtZ6NJP5aQ"
      },
      "source": [
        "df = fill_mean([\"population\"],df)"
      ],
      "execution_count": 124,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ArtzCUol9C7i",
        "outputId": "7a8fc44a-f9de-4721-9913-c34af90c53ef"
      },
      "source": [
        "# Number of missing values in each column\n",
        "missing_val_count_by_column = (df.isnull().sum())\n",
        "print(missing_val_count_by_column[missing_val_count_by_column > 0])"
      ],
      "execution_count": 125,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Series([], dtype: int64)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YP0oglRp9Nyh"
      },
      "source": [
        "*   *No missing values*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xzRJ9F0kRTJi"
      },
      "source": [
        "#**Data Preprocessing**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P3I9JxUtSwL-"
      },
      "source": [
        "#date time\n",
        "df.date_recorded = pd.to_datetime(df.date_recorded)"
      ],
      "execution_count": 173,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8OM34uMGYp8l"
      },
      "source": [
        "df['year'] = df['date_recorded'].dt.year\n",
        "df['month'] = df['date_recorded'].dt.month\n",
        "df['day_of_months'] = df['date_recorded'].dt.day"
      ],
      "execution_count": 174,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GzFCsBgrZt6W"
      },
      "source": [
        "df.drop(columns=['date_recorded'], axis=1, inplace=True)"
      ],
      "execution_count": 175,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8R7KSEt5Sun3"
      },
      "source": [
        "#Payment\n",
        "df[\"Payment_all\"] = df[\"payment\"] + df[\"payment_type\"]\n",
        "\n",
        "#quality\n",
        "df[\"Quality_all\"] = df[\"water_quality\"] + df[\"quality_group\"]\n",
        "\n",
        "#Quantitiy\n",
        "df[\"Quantity_all\"] = df[\"quantity\"] + df[\"quantity_group\"]\n",
        "\n",
        "#waterpoint_type\n",
        "df[\"Waterpoint_all\"] = df[\"waterpoint_type\"] + df[\"waterpoint_type_group\"]"
      ],
      "execution_count": 176,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6zfKCdoQfP91"
      },
      "source": [
        "#drop unwanted columns\n",
        "df.drop(columns=['payment','payment_type','water_quality','quality_group','quantity','quantity_group','waterpoint_type','waterpoint_type_group'], axis=1, inplace=True)"
      ],
      "execution_count": 177,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aFOSjq35Jp9F"
      },
      "source": [
        "**Encoding**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g3VQQJSaRLc3"
      },
      "source": [
        "#one hot encoding\n",
        "def one_hot_encode(df,object_cols):\n",
        "  OH_encoder = OneHotEncoder(handle_unknown='ignore', sparse=False)\n",
        "  OH_cols_train = pd.DataFrame(OH_encoder.fit_transform(df[object_cols]))\n",
        "\n",
        "  # One-hot encoding removed index; put it back\n",
        "  OH_cols_train.index = df.index\n",
        "\n",
        "  # Remove categorical columns (will replace with one-hot encoding)\n",
        "  num_X_train = df.drop(object_cols, axis=1)\n",
        "\n",
        "  # Add one-hot encoded columns to numerical features\n",
        "  df = pd.concat([num_X_train, OH_cols_train], axis=1)\n",
        "  return df"
      ],
      "execution_count": 178,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZQRhUXf1RXMf"
      },
      "source": [
        "columns=['source','source_type','source_class','public_meeting','permit','extraction_type','extraction_type_group','extraction_type_class','management','management_group',\"basin\", \"scheme_management\",\"Payment_all\",\"Quality_all\",\"Quantity_all\",\"Waterpoint_all\"]\n",
        "df = one_hot_encode(df, columns)\n",
        "# one_hot_encode(test, columns)"
      ],
      "execution_count": 179,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2OMcT-piRhq3"
      },
      "source": [
        "#binary encoder\n",
        "#use if count of unique values is more than 15\n",
        " \n",
        "enc = ce.BinaryEncoder(cols=['funder','installer', 'subvillage', 'region','lga','ward']).fit(df)\n",
        "df = enc.transform(df)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qh1UbzU6ltOE"
      },
      "source": [
        "#**Separate training and test set**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9MfMGuJHlyDx"
      },
      "source": [
        "X = df[:len(features_df)]\n",
        "X_test = df[len(features_df):]"
      ],
      "execution_count": 188,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N9mhq0j3l1Fw"
      },
      "source": [
        "X.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FRQuVXuZl-bZ"
      },
      "source": [
        "X_test.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_eqCEVu-mCFl",
        "outputId": "e73019a6-7d02-4845-8242-bcd037754407"
      },
      "source": [
        "X.shape"
      ],
      "execution_count": 191,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(59400, 212)"
            ]
          },
          "metadata": {},
          "execution_count": 191
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "65B61fnomEoc",
        "outputId": "408908e8-24a1-4f90-da29-dcac2577c67f"
      },
      "source": [
        "X_test.shape"
      ],
      "execution_count": 192,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(14850, 212)"
            ]
          },
          "metadata": {},
          "execution_count": 192
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AUWvzAg9mIF5"
      },
      "source": [
        "#**Model Training**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ysK4gyApmH0h"
      },
      "source": [
        "def score_dataset(X, y, model):\n",
        "    cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=0)\n",
        "    # Metric is Accuracy\n",
        "    score = cross_val_score(\n",
        "        model, X, y, scoring='accuracy', cv=cv, n_jobs=-1, error_score='raise')\n",
        "    print('Accuracy: %.3f (%.3f)' % (mean(score), std(score)))"
      ],
      "execution_count": 193,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eoZ5j382mNxZ"
      },
      "source": [
        "modelRF = RandomForestClassifier(n_estimators=400, random_state=0,max_depth=20)\n",
        "modelETC = ExtraTreesClassifier(n_estimators=1000,min_samples_split=10)\n",
        "modelXGB = XGBClassifier()\n",
        "modelCAT = CatBoostClassifier(max_ctr_complexity=5, task_type='CPU', iterations=1000,\n",
        "                   eval_metric='AUC', od_type='Iter', od_wait=500, loss_function = 'MultiClass', random_seed=1, depth=10, learning_rate = 0.03, l2_leaf_reg = 1)\n",
        "modelMLP = MLPClassifier()\n",
        "modelSVM = SVC(kernel='linear', C=1,gamma='auto')"
      ],
      "execution_count": 194,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jITiGCjZtNMp"
      },
      "source": [
        "# score model\n",
        "scoreSVM = score_dataset(X,y,modelSVM)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4UY0l4IKcjwM",
        "outputId": "beda4c41-64f3-4b2f-e41f-130d82f75f57"
      },
      "source": [
        "# score model\n",
        "scoreRFTest = score_dataset(X,y,modelRF)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy: 0.816 (0.004)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EOkGe031mesr",
        "outputId": "96aa5f9c-6279-426c-def5-46ebe90d486c"
      },
      "source": [
        "# score model\n",
        "scoreXGB = score_dataset(X,y,modelXGB)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy: 0.745 (0.004)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_tiQbbGITdc-"
      },
      "source": [
        "# score model\n",
        "scoreETC = score_dataset(X,y,modelETC)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fslCsVp9tPen"
      },
      "source": [
        "#**Hyperparameter Tunning**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yd4CQgeNtTei"
      },
      "source": [
        "def grid_search(model, params, X_tune, y_tune):\n",
        "  clf =  GridSearchCV(model, params, cv=5, scoring='accuracy')\n",
        "  clf.fit(X_tune, y_tune)\n",
        "  return clf.best_score_, clf.best_params_"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WRbSrfC-tc_q"
      },
      "source": [
        "model_params = {\n",
        "    'random_forest': {\n",
        "        'model': RandomForestClassifier(),\n",
        "        'params' : {\n",
        "            'n_estimators': [50,100,200,400],\n",
        "            'max_depth': [5, 10, 20, 100]\n",
        "        }\n",
        "    },\n",
        "    'xg_boost': {\n",
        "        'model': XGBClassifier(),\n",
        "        'params' : {\n",
        "            'eta': [0.01, 0.1, 0.2, 0.4],\n",
        "            'max_depth': [5, 10, 15, 20, 22, 100]\n",
        "        }\n",
        "    },\n",
        "    'cat_boost': {\n",
        "        'model': CatBoostClassifier(loss_function = 'MultiClass', iterations=1000),\n",
        "        'params' : {\n",
        "            'learning_rate': [0.03, 0.1],\n",
        "            'depth':[4, 6, 10],\n",
        "            'l2_leaf_reg': [1, 3, 5, 7, 9]\n",
        "        }\n",
        "    }\n",
        "}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OMLXdFNgtgA_"
      },
      "source": [
        "X_tune=X\n",
        "y_tune = y.status_group"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "vGOhwsh7tkZW",
        "outputId": "81f8f584-4d65-452d-ed5d-2c6ffa81b435"
      },
      "source": [
        "best_score, best_params = grid_search(model_params['random_forest']['model'], model_params['random_forest']['params'], X_tune, y_tune)\n",
        "print(best_score, best_params)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.8132491582491583 {'max_depth': 20, 'n_estimators': 400}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HS2Oa_9Ztmj9"
      },
      "source": [
        "best_score, best_params = grid_search(model_params['xg_boost']['model'], model_params['xg_boost']['params'], X_tune, y_tune)\n",
        "print(best_score, best_params)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Thgp8gQtnXt"
      },
      "source": [
        "best_score, best_params = grid_search(model_params['cat_boost']['model'], model_params['cat_boost']['params'], X_tune, y_tune)\n",
        "print(best_score, best_params)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ql9fCGpvrKeA"
      },
      "source": [
        "#**Ensembling**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SGrFsMUkqFXr"
      },
      "source": [
        "estimators_stacked = [\n",
        "              #('xbg', modelXGB),\n",
        "              # ('mlp', modelMLP),\n",
        "              #('cat',modelCAT),\n",
        "              ('xt', modelETC),\n",
        "              ('rn', modelRF)\n",
        "             ]\n",
        "\n",
        "# Stacking Classifier\n",
        "stacked_estimator =  StackingClassifier(estimators=estimators_stacked,\n",
        "                                    final_estimator=LogisticRegression()  \n",
        "                                  )\n",
        "# Voting Classifier\n",
        "voting_estimator = VotingClassifier(estimators=estimators_stacked, \n",
        "                        voting='hard')  # hard if labels needed, soft if probabilites needed"
      ],
      "execution_count": 195,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BTw4UMFMo8SK"
      },
      "source": [
        "# score model\n",
        "scoreEns = score_dataset(X,y,stacked_estimator)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BXlAclbqrVqP"
      },
      "source": [
        "#**Train on entire dataset**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C3-xaUyerP4a",
        "outputId": "99447a5e-7f9f-4b0b-ab9d-408bc28cb328"
      },
      "source": [
        "model = modelRF\n",
        "X_train = X\n",
        "y_train = y\n",
        "\n",
        "model.fit(X_train, y_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:5: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
            "  \"\"\"\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
              "                       criterion='gini', max_depth=20, max_features='auto',\n",
              "                       max_leaf_nodes=None, max_samples=None,\n",
              "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "                       min_samples_leaf=1, min_samples_split=2,\n",
              "                       min_weight_fraction_leaf=0.0, n_estimators=400,\n",
              "                       n_jobs=None, oob_score=False, random_state=0, verbose=0,\n",
              "                       warm_start=False)"
            ]
          },
          "metadata": {},
          "execution_count": 97
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QRMeR6QShd4X",
        "outputId": "e782833b-8c9b-4833-f460-817305c3334d"
      },
      "source": [
        "#ETC,RF\n",
        "modelEns = stacked_estimator\n",
        "X_train = X\n",
        "y_train = y\n",
        "\n",
        "modelEns.fit(X_train, y_train)"
      ],
      "execution_count": 97,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/preprocessing/_label.py:235: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/preprocessing/_label.py:268: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "StackingClassifier(cv=None,\n",
              "                   estimators=[('xt',\n",
              "                                ExtraTreesClassifier(bootstrap=False,\n",
              "                                                     ccp_alpha=0.0,\n",
              "                                                     class_weight=None,\n",
              "                                                     criterion='gini',\n",
              "                                                     max_depth=None,\n",
              "                                                     max_features='auto',\n",
              "                                                     max_leaf_nodes=None,\n",
              "                                                     max_samples=None,\n",
              "                                                     min_impurity_decrease=0.0,\n",
              "                                                     min_impurity_split=None,\n",
              "                                                     min_samples_leaf=1,\n",
              "                                                     min_samples_split=10,\n",
              "                                                     min_weight_fraction_leaf=0.0,\n",
              "                                                     n_estimators=1000,\n",
              "                                                     n_jobs=Non...\n",
              "                                                       random_state=0,\n",
              "                                                       verbose=0,\n",
              "                                                       warm_start=False))],\n",
              "                   final_estimator=LogisticRegression(C=1.0, class_weight=None,\n",
              "                                                      dual=False,\n",
              "                                                      fit_intercept=True,\n",
              "                                                      intercept_scaling=1,\n",
              "                                                      l1_ratio=None,\n",
              "                                                      max_iter=100,\n",
              "                                                      multi_class='auto',\n",
              "                                                      n_jobs=None, penalty='l2',\n",
              "                                                      random_state=None,\n",
              "                                                      solver='lbfgs',\n",
              "                                                      tol=0.0001, verbose=0,\n",
              "                                                      warm_start=False),\n",
              "                   n_jobs=None, passthrough=False, stack_method='auto',\n",
              "                   verbose=0)"
            ]
          },
          "metadata": {},
          "execution_count": 97
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fqEUEm7ezFet"
      },
      "source": [
        "#ETC,RF,cat\n",
        "modelEns = stacked_estimator\n",
        "X_train = X\n",
        "y_train = y\n",
        "\n",
        "modelEns.fit(X_train, y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mvsIiXParx2R"
      },
      "source": [
        "#**Predict**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kUdjdGoYrux_"
      },
      "source": [
        "predict=model.predict(X_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GPshukiYwdXN"
      },
      "source": [
        "predict=modelEns.predict(X_test)"
      ],
      "execution_count": 197,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2kBH1mphr1_o"
      },
      "source": [
        "#**Submission**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uigVwuznr1gH"
      },
      "source": [
        "submission_df = pd.read_csv(\"SubmissionFormat.csv\", \n",
        "                            index_col=\"id\")"
      ],
      "execution_count": 198,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nh4xUYFOtKwX"
      },
      "source": [
        "# Make sure we have the rows in the same order\n",
        "np.testing.assert_array_equal(test_df.index.values, \n",
        "                              submission_df.index.values)\n",
        "\n",
        "# Save predictions to submission data frame\n",
        "submission_df[\"status_group\"] = predict"
      ],
      "execution_count": 199,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UvBq5yR6tS-m"
      },
      "source": [
        "submission_df.to_csv('my_submission.csv', index=True)"
      ],
      "execution_count": 200,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "vEmAq2zytTxF",
        "outputId": "c2d347d5-6a5f-43ee-df82-4fbc192ea432"
      },
      "source": [
        "from google.colab import files\n",
        "files.download('my_submission.csv')"
      ],
      "execution_count": 201,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_e84b70f5-4ae9-4ec1-99ce-5fd6d2c3db45\", \"ENS3.csv\", 277585)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        }
      ]
    }
  ]
}