Churn_Prediction_Project.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Churn Prediction Project.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "fb91d98cebab4bbe90d5a8123d60a9ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_eb355b827ff1440296adce6e383fcccc",
              "IPY_MODEL_7eed98b55f3c47e7a211ba33a966fe5f",
              "IPY_MODEL_212e5c3bd5f44ce8bf1ab8bee51e16ff"
            ],
            "layout": "IPY_MODEL_d6f9a289b07d447d8d329b70de7e1784"
          }
        },
        "eb355b827ff1440296adce6e383fcccc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dfcbe3fa676f4c9ba40aaa52cdf618da",
            "placeholder": "​",
            "style": "IPY_MODEL_27f0cc1c712c4c64a5f8519befe85f05",
            "value": "100%"
          }
        },
        "7eed98b55f3c47e7a211ba33a966fe5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_14b90626139143259f0d47b34b165b55",
            "max": 7,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0aa548606dc9469bbfeabfafdbfdced2",
            "value": 7
          }
        },
        "212e5c3bd5f44ce8bf1ab8bee51e16ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b0abb94bb47e43038028fe925de666b7",
            "placeholder": "​",
            "style": "IPY_MODEL_4127591b58c34d4f8e069fd643998664",
            "value": " 7/7 [00:29&lt;00:00,  3.49s/it]"
          }
        },
        "d6f9a289b07d447d8d329b70de7e1784": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dfcbe3fa676f4c9ba40aaa52cdf618da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "27f0cc1c712c4c64a5f8519befe85f05": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "14b90626139143259f0d47b34b165b55": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0aa548606dc9469bbfeabfafdbfdced2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b0abb94bb47e43038028fe925de666b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4127591b58c34d4f8e069fd643998664": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "**3.2 Data Preparation**"
      ],
      "metadata": {
        "id": "6QsTB47hALc5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "import matplotlib.pyplot as plt"
      ],
      "metadata": {
        "id": "CXvR6cwn_fjT"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = 'https://raw.githubusercontent.com/alexeygrigorev/mlbookcamp-code/master/chapter-03-churn-prediction/WA_Fn-UseC_-Telco-Customer-Churn.csv'\n"
      ],
      "metadata": {
        "id": "JNPnQXo4AJXh"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!wget $data -O data-week-3.csv \n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SR2qJkvzAVbE",
        "outputId": "819ef277-a8b5-4b1f-8345-103419ba2e27"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-03-31 06:21:07--  https://raw.githubusercontent.com/alexeygrigorev/mlbookcamp-code/master/chapter-03-churn-prediction/WA_Fn-UseC_-Telco-Customer-Churn.csv\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.109.133, 185.199.108.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 977501 (955K) [text/plain]\n",
            "Saving to: ‘data-week-3.csv’\n",
            "\n",
            "\rdata-week-3.csv       0%[                    ]       0  --.-KB/s               \rdata-week-3.csv     100%[===================>] 954.59K  --.-KB/s    in 0.009s  \n",
            "\n",
            "2022-03-31 06:21:07 (106 MB/s) - ‘data-week-3.csv’ saved [977501/977501]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('data-week-3.csv')\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 386
        },
        "id": "UkySDbqqAYmn",
        "outputId": "5983052a-cb5f-4fef-dce6-4f4ce9afc15e"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   customerID  gender  SeniorCitizen Partner Dependents  tenure PhoneService  \\\n",
              "0  7590-VHVEG  Female              0     Yes         No       1           No   \n",
              "1  5575-GNVDE    Male              0      No         No      34          Yes   \n",
              "2  3668-QPYBK    Male              0      No         No       2          Yes   \n",
              "3  7795-CFOCW    Male              0      No         No      45           No   \n",
              "4  9237-HQITU  Female              0      No         No       2          Yes   \n",
              "\n",
              "      MultipleLines InternetService OnlineSecurity  ... DeviceProtection  \\\n",
              "0  No phone service             DSL             No  ...               No   \n",
              "1                No             DSL            Yes  ...              Yes   \n",
              "2                No             DSL            Yes  ...               No   \n",
              "3  No phone service             DSL            Yes  ...              Yes   \n",
              "4                No     Fiber optic             No  ...               No   \n",
              "\n",
              "  TechSupport StreamingTV StreamingMovies        Contract PaperlessBilling  \\\n",
              "0          No          No              No  Month-to-month              Yes   \n",
              "1          No          No              No        One year               No   \n",
              "2          No          No              No  Month-to-month              Yes   \n",
              "3         Yes          No              No        One year               No   \n",
              "4          No          No              No  Month-to-month              Yes   \n",
              "\n",
              "               PaymentMethod MonthlyCharges  TotalCharges Churn  \n",
              "0           Electronic check          29.85         29.85    No  \n",
              "1               Mailed check          56.95        1889.5    No  \n",
              "2               Mailed check          53.85        108.15   Yes  \n",
              "3  Bank transfer (automatic)          42.30       1840.75    No  \n",
              "4           Electronic check          70.70        151.65   Yes  \n",
              "\n",
              "[5 rows x 21 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6d18d414-e60b-49cd-a24c-b3ee4a5dc70b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>customerID</th>\n",
              "      <th>gender</th>\n",
              "      <th>SeniorCitizen</th>\n",
              "      <th>Partner</th>\n",
              "      <th>Dependents</th>\n",
              "      <th>tenure</th>\n",
              "      <th>PhoneService</th>\n",
              "      <th>MultipleLines</th>\n",
              "      <th>InternetService</th>\n",
              "      <th>OnlineSecurity</th>\n",
              "      <th>...</th>\n",
              "      <th>DeviceProtection</th>\n",
              "      <th>TechSupport</th>\n",
              "      <th>StreamingTV</th>\n",
              "      <th>StreamingMovies</th>\n",
              "      <th>Contract</th>\n",
              "      <th>PaperlessBilling</th>\n",
              "      <th>PaymentMethod</th>\n",
              "      <th>MonthlyCharges</th>\n",
              "      <th>TotalCharges</th>\n",
              "      <th>Churn</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>7590-VHVEG</td>\n",
              "      <td>Female</td>\n",
              "      <td>0</td>\n",
              "      <td>Yes</td>\n",
              "      <td>No</td>\n",
              "      <td>1</td>\n",
              "      <td>No</td>\n",
              "      <td>No phone service</td>\n",
              "      <td>DSL</td>\n",
              "      <td>No</td>\n",
              "      <td>...</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>Month-to-month</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Electronic check</td>\n",
              "      <td>29.85</td>\n",
              "      <td>29.85</td>\n",
              "      <td>No</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>5575-GNVDE</td>\n",
              "      <td>Male</td>\n",
              "      <td>0</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>34</td>\n",
              "      <td>Yes</td>\n",
              "      <td>No</td>\n",
              "      <td>DSL</td>\n",
              "      <td>Yes</td>\n",
              "      <td>...</td>\n",
              "      <td>Yes</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>One year</td>\n",
              "      <td>No</td>\n",
              "      <td>Mailed check</td>\n",
              "      <td>56.95</td>\n",
              "      <td>1889.5</td>\n",
              "      <td>No</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3668-QPYBK</td>\n",
              "      <td>Male</td>\n",
              "      <td>0</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>2</td>\n",
              "      <td>Yes</td>\n",
              "      <td>No</td>\n",
              "      <td>DSL</td>\n",
              "      <td>Yes</td>\n",
              "      <td>...</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>Month-to-month</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Mailed check</td>\n",
              "      <td>53.85</td>\n",
              "      <td>108.15</td>\n",
              "      <td>Yes</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>7795-CFOCW</td>\n",
              "      <td>Male</td>\n",
              "      <td>0</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>45</td>\n",
              "      <td>No</td>\n",
              "      <td>No phone service</td>\n",
              "      <td>DSL</td>\n",
              "      <td>Yes</td>\n",
              "      <td>...</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Yes</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>One year</td>\n",
              "      <td>No</td>\n",
              "      <td>Bank transfer (automatic)</td>\n",
              "      <td>42.30</td>\n",
              "      <td>1840.75</td>\n",
              "      <td>No</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>9237-HQITU</td>\n",
              "      <td>Female</td>\n",
              "      <td>0</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>2</td>\n",
              "      <td>Yes</td>\n",
              "      <td>No</td>\n",
              "      <td>Fiber optic</td>\n",
              "      <td>No</td>\n",
              "      <td>...</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>No</td>\n",
              "      <td>Month-to-month</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Electronic check</td>\n",
              "      <td>70.70</td>\n",
              "      <td>151.65</td>\n",
              "      <td>Yes</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 21 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6d18d414-e60b-49cd-a24c-b3ee4a5dc70b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6d18d414-e60b-49cd-a24c-b3ee4a5dc70b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6d18d414-e60b-49cd-a24c-b3ee4a5dc70b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.columns = df.columns.str.lower().str.replace(' ', '_')\n",
        "\n",
        "categorical_columns = list(df.dtypes[df.dtypes == 'object'].index)\n",
        "\n",
        "for c in categorical_columns:\n",
        "    df[c] = df[c].str.lower().str.replace(' ', '_')"
      ],
      "metadata": {
        "id": "ZO9XZMeJAdBp"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "df.head().T\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 708
        },
        "id": "B_1e1f0yAmiC",
        "outputId": "29316bad-36df-441a-bc57-176d10d749d2"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                 0             1               2  \\\n",
              "customerid              7590-vhveg    5575-gnvde      3668-qpybk   \n",
              "gender                      female          male            male   \n",
              "seniorcitizen                    0             0               0   \n",
              "partner                        yes            no              no   \n",
              "dependents                      no            no              no   \n",
              "tenure                           1            34               2   \n",
              "phoneservice                    no           yes             yes   \n",
              "multiplelines     no_phone_service            no              no   \n",
              "internetservice                dsl           dsl             dsl   \n",
              "onlinesecurity                  no           yes             yes   \n",
              "onlinebackup                   yes            no             yes   \n",
              "deviceprotection                no           yes              no   \n",
              "techsupport                     no            no              no   \n",
              "streamingtv                     no            no              no   \n",
              "streamingmovies                 no            no              no   \n",
              "contract            month-to-month      one_year  month-to-month   \n",
              "paperlessbilling               yes            no             yes   \n",
              "paymentmethod     electronic_check  mailed_check    mailed_check   \n",
              "monthlycharges               29.85         56.95           53.85   \n",
              "totalcharges                 29.85        1889.5          108.15   \n",
              "churn                           no            no             yes   \n",
              "\n",
              "                                          3                 4  \n",
              "customerid                       7795-cfocw        9237-hqitu  \n",
              "gender                                 male            female  \n",
              "seniorcitizen                             0                 0  \n",
              "partner                                  no                no  \n",
              "dependents                               no                no  \n",
              "tenure                                   45                 2  \n",
              "phoneservice                             no               yes  \n",
              "multiplelines              no_phone_service                no  \n",
              "internetservice                         dsl       fiber_optic  \n",
              "onlinesecurity                          yes                no  \n",
              "onlinebackup                             no                no  \n",
              "deviceprotection                        yes                no  \n",
              "techsupport                             yes                no  \n",
              "streamingtv                              no                no  \n",
              "streamingmovies                          no                no  \n",
              "contract                           one_year    month-to-month  \n",
              "paperlessbilling                         no               yes  \n",
              "paymentmethod     bank_transfer_(automatic)  electronic_check  \n",
              "monthlycharges                         42.3              70.7  \n",
              "totalcharges                        1840.75            151.65  \n",
              "churn                                    no               yes  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b9659a35-2b12-4b14-b026-bd4973c20cff\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>customerid</th>\n",
              "      <td>7590-vhveg</td>\n",
              "      <td>5575-gnvde</td>\n",
              "      <td>3668-qpybk</td>\n",
              "      <td>7795-cfocw</td>\n",
              "      <td>9237-hqitu</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>gender</th>\n",
              "      <td>female</td>\n",
              "      <td>male</td>\n",
              "      <td>male</td>\n",
              "      <td>male</td>\n",
              "      <td>female</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>seniorcitizen</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>partner</th>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>dependents</th>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>tenure</th>\n",
              "      <td>1</td>\n",
              "      <td>34</td>\n",
              "      <td>2</td>\n",
              "      <td>45</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>phoneservice</th>\n",
              "      <td>no</td>\n",
              "      <td>yes</td>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "      <td>yes</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>multiplelines</th>\n",
              "      <td>no_phone_service</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>no_phone_service</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>internetservice</th>\n",
              "      <td>dsl</td>\n",
              "      <td>dsl</td>\n",
              "      <td>dsl</td>\n",
              "      <td>dsl</td>\n",
              "      <td>fiber_optic</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>onlinesecurity</th>\n",
              "      <td>no</td>\n",
              "      <td>yes</td>\n",
              "      <td>yes</td>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>onlinebackup</th>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>deviceprotection</th>\n",
              "      <td>no</td>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>techsupport</th>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>streamingtv</th>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>streamingmovies</th>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>contract</th>\n",
              "      <td>month-to-month</td>\n",
              "      <td>one_year</td>\n",
              "      <td>month-to-month</td>\n",
              "      <td>one_year</td>\n",
              "      <td>month-to-month</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>paperlessbilling</th>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "      <td>yes</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>paymentmethod</th>\n",
              "      <td>electronic_check</td>\n",
              "      <td>mailed_check</td>\n",
              "      <td>mailed_check</td>\n",
              "      <td>bank_transfer_(automatic)</td>\n",
              "      <td>electronic_check</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>monthlycharges</th>\n",
              "      <td>29.85</td>\n",
              "      <td>56.95</td>\n",
              "      <td>53.85</td>\n",
              "      <td>42.3</td>\n",
              "      <td>70.7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>totalcharges</th>\n",
              "      <td>29.85</td>\n",
              "      <td>1889.5</td>\n",
              "      <td>108.15</td>\n",
              "      <td>1840.75</td>\n",
              "      <td>151.65</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>churn</th>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "      <td>yes</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b9659a35-2b12-4b14-b026-bd4973c20cff')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b9659a35-2b12-4b14-b026-bd4973c20cff button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b9659a35-2b12-4b14-b026-bd4973c20cff');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tc = pd.to_numeric(df.totalcharges, errors='coerce')\n"
      ],
      "metadata": {
        "id": "G9WpFJ6IApnZ"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.totalcharges = pd.to_numeric(df.totalcharges, errors='coerce')\n"
      ],
      "metadata": {
        "id": "qXM29GVfAykA"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.totalcharges = df.totalcharges.fillna(0)\n"
      ],
      "metadata": {
        "id": "XqRVW4H2A3gw"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.churn.head()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tm9ScmcsA4Qb",
        "outputId": "5e35f4a9-92fd-44d0-c8d5-a4763edda05d"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0     no\n",
              "1     no\n",
              "2    yes\n",
              "3     no\n",
              "4    yes\n",
              "Name: churn, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.churn = (df.churn == 'yes').astype(int)\n"
      ],
      "metadata": {
        "id": "k_9zzETnA-Vq"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**3.3 Setting up the validation framework**\n",
        "\n",
        "*Perform the train/validation/test split with Scikit-Learn\n"
      ],
      "metadata": {
        "id": "4tiyveT2BZtK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n"
      ],
      "metadata": {
        "id": "M3awwsKSBAhy"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train, df_test = train_test_split(df, test_size=0.2, random_state=1)\n",
        "df_train, df_val = train_test_split(df_full_train, test_size=0.25, random_state=1)\n"
      ],
      "metadata": {
        "id": "EsksQkcJBsdr"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(df_train), len(df_val), len(df_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_9giMcT-BvTL",
        "outputId": "b71549a3-a8b8-4367-9053-16119f75571b"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(4225, 1409, 1409)"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_train = df_train.reset_index(drop=True)\n",
        "df_val = df_val.reset_index(drop=True)\n",
        "df_test = df_test.reset_index(drop=True)"
      ],
      "metadata": {
        "id": "BIAjM8SDBxMd"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y_train = df_train.churn.values\n",
        "y_val = df_val.churn.values\n",
        "y_test = df_test.churn.values\n",
        "\n",
        "del df_train['churn']\n",
        "del df_val['churn']\n",
        "del df_test['churn']"
      ],
      "metadata": {
        "id": "sdQrgrWNB8XD"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**3.4 EDA**\n",
        "- Check missing values\n",
        "- Look at the target variable (churn)\n",
        "- Look at numerical and categorical variables"
      ],
      "metadata": {
        "id": "IhsR3jNBCJo4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train = df_full_train.reset_index(drop=True)\n"
      ],
      "metadata": {
        "id": "29oA2GVnB-a8"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train.isnull().sum()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9tG9bkW9CTfO",
        "outputId": "7c647495-ce84-45fa-f44d-5ff18244ed5f"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "customerid          0\n",
              "gender              0\n",
              "seniorcitizen       0\n",
              "partner             0\n",
              "dependents          0\n",
              "tenure              0\n",
              "phoneservice        0\n",
              "multiplelines       0\n",
              "internetservice     0\n",
              "onlinesecurity      0\n",
              "onlinebackup        0\n",
              "deviceprotection    0\n",
              "techsupport         0\n",
              "streamingtv         0\n",
              "streamingmovies     0\n",
              "contract            0\n",
              "paperlessbilling    0\n",
              "paymentmethod       0\n",
              "monthlycharges      0\n",
              "totalcharges        0\n",
              "churn               0\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train.churn.value_counts(normalize=True)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ase7qEUgCVff",
        "outputId": "d5f4f84e-137f-435c-a9f9-fd07cbd443e6"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    0.730032\n",
              "1    0.269968\n",
              "Name: churn, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train.churn.mean()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w0eqYrm3Cjoy",
        "outputId": "ad8ae1b6-22da-4c2c-c458-6afc1845b401"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.26996805111821087"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "numerical = ['tenure', 'monthlycharges', 'totalcharges']\n"
      ],
      "metadata": {
        "id": "8I9Gmj5TCntF"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "categorical = [\n",
        "    'gender',\n",
        "    'seniorcitizen',\n",
        "    'partner',\n",
        "    'dependents',\n",
        "    'phoneservice',\n",
        "    'multiplelines',\n",
        "    'internetservice',\n",
        "    'onlinesecurity',\n",
        "    'onlinebackup',\n",
        "    'deviceprotection',\n",
        "    'techsupport',\n",
        "    'streamingtv',\n",
        "    'streamingmovies',\n",
        "    'contract',\n",
        "    'paperlessbilling',\n",
        "    'paymentmethod',\n",
        "]"
      ],
      "metadata": {
        "id": "pSIvwt42C4oe"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train[categorical].nunique()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p0dTLj2BC8Pe",
        "outputId": "36791a8e-48f3-42d5-b2c1-c80fe5b51d71"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "gender              2\n",
              "seniorcitizen       2\n",
              "partner             2\n",
              "dependents          2\n",
              "phoneservice        2\n",
              "multiplelines       3\n",
              "internetservice     3\n",
              "onlinesecurity      3\n",
              "onlinebackup        3\n",
              "deviceprotection    3\n",
              "techsupport         3\n",
              "streamingtv         3\n",
              "streamingmovies     3\n",
              "contract            3\n",
              "paperlessbilling    2\n",
              "paymentmethod       4\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**3.5 Feature importance: Churn rate and risk ratio**\n",
        "\n",
        "Feature importance analysis (part of EDA) - identifying which features affect our target variable\n",
        "\n",
        "- Churn rate\n",
        "- Risk ratio\n",
        "- Mutual information - later\n"
      ],
      "metadata": {
        "id": "Ct9PilhrDv4b"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Churn rate**"
      ],
      "metadata": {
        "id": "M781mHJKEELy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 299
        },
        "id": "Ig97_AH4DVqj",
        "outputId": "728e0a37-10ab-48ff-aab0-794136e9bd8d"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   customerid  gender  seniorcitizen partner dependents  tenure phoneservice  \\\n",
              "0  5442-pptjy    male              0     yes        yes      12          yes   \n",
              "1  6261-rcvns  female              0      no         no      42          yes   \n",
              "2  2176-osjuv    male              0     yes         no      71          yes   \n",
              "3  6161-erdgd    male              0     yes        yes      71          yes   \n",
              "4  2364-ufrom    male              0      no         no      30          yes   \n",
              "\n",
              "  multiplelines internetservice       onlinesecurity  ...  \\\n",
              "0            no              no  no_internet_service  ...   \n",
              "1            no             dsl                  yes  ...   \n",
              "2           yes             dsl                  yes  ...   \n",
              "3           yes             dsl                  yes  ...   \n",
              "4            no             dsl                  yes  ...   \n",
              "\n",
              "      deviceprotection          techsupport          streamingtv  \\\n",
              "0  no_internet_service  no_internet_service  no_internet_service   \n",
              "1                  yes                  yes                   no   \n",
              "2                   no                  yes                   no   \n",
              "3                  yes                  yes                  yes   \n",
              "4                   no                  yes                  yes   \n",
              "\n",
              "       streamingmovies  contract paperlessbilling              paymentmethod  \\\n",
              "0  no_internet_service  two_year               no               mailed_check   \n",
              "1                  yes  one_year               no    credit_card_(automatic)   \n",
              "2                   no  two_year               no  bank_transfer_(automatic)   \n",
              "3                  yes  one_year               no           electronic_check   \n",
              "4                   no  one_year               no           electronic_check   \n",
              "\n",
              "  monthlycharges  totalcharges  churn  \n",
              "0          19.70        258.35      0  \n",
              "1          73.90       3160.55      1  \n",
              "2          65.15       4681.75      0  \n",
              "3          85.45       6300.85      0  \n",
              "4          70.40       2044.75      0  \n",
              "\n",
              "[5 rows x 21 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a30a9d17-1945-4bfe-bece-c39e36c480ee\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>customerid</th>\n",
              "      <th>gender</th>\n",
              "      <th>seniorcitizen</th>\n",
              "      <th>partner</th>\n",
              "      <th>dependents</th>\n",
              "      <th>tenure</th>\n",
              "      <th>phoneservice</th>\n",
              "      <th>multiplelines</th>\n",
              "      <th>internetservice</th>\n",
              "      <th>onlinesecurity</th>\n",
              "      <th>...</th>\n",
              "      <th>deviceprotection</th>\n",
              "      <th>techsupport</th>\n",
              "      <th>streamingtv</th>\n",
              "      <th>streamingmovies</th>\n",
              "      <th>contract</th>\n",
              "      <th>paperlessbilling</th>\n",
              "      <th>paymentmethod</th>\n",
              "      <th>monthlycharges</th>\n",
              "      <th>totalcharges</th>\n",
              "      <th>churn</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>5442-pptjy</td>\n",
              "      <td>male</td>\n",
              "      <td>0</td>\n",
              "      <td>yes</td>\n",
              "      <td>yes</td>\n",
              "      <td>12</td>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>no_internet_service</td>\n",
              "      <td>...</td>\n",
              "      <td>no_internet_service</td>\n",
              "      <td>no_internet_service</td>\n",
              "      <td>no_internet_service</td>\n",
              "      <td>no_internet_service</td>\n",
              "      <td>two_year</td>\n",
              "      <td>no</td>\n",
              "      <td>mailed_check</td>\n",
              "      <td>19.70</td>\n",
              "      <td>258.35</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>6261-rcvns</td>\n",
              "      <td>female</td>\n",
              "      <td>0</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>42</td>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "      <td>dsl</td>\n",
              "      <td>yes</td>\n",
              "      <td>...</td>\n",
              "      <td>yes</td>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "      <td>yes</td>\n",
              "      <td>one_year</td>\n",
              "      <td>no</td>\n",
              "      <td>credit_card_(automatic)</td>\n",
              "      <td>73.90</td>\n",
              "      <td>3160.55</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2176-osjuv</td>\n",
              "      <td>male</td>\n",
              "      <td>0</td>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "      <td>71</td>\n",
              "      <td>yes</td>\n",
              "      <td>yes</td>\n",
              "      <td>dsl</td>\n",
              "      <td>yes</td>\n",
              "      <td>...</td>\n",
              "      <td>no</td>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>two_year</td>\n",
              "      <td>no</td>\n",
              "      <td>bank_transfer_(automatic)</td>\n",
              "      <td>65.15</td>\n",
              "      <td>4681.75</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>6161-erdgd</td>\n",
              "      <td>male</td>\n",
              "      <td>0</td>\n",
              "      <td>yes</td>\n",
              "      <td>yes</td>\n",
              "      <td>71</td>\n",
              "      <td>yes</td>\n",
              "      <td>yes</td>\n",
              "      <td>dsl</td>\n",
              "      <td>yes</td>\n",
              "      <td>...</td>\n",
              "      <td>yes</td>\n",
              "      <td>yes</td>\n",
              "      <td>yes</td>\n",
              "      <td>yes</td>\n",
              "      <td>one_year</td>\n",
              "      <td>no</td>\n",
              "      <td>electronic_check</td>\n",
              "      <td>85.45</td>\n",
              "      <td>6300.85</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2364-ufrom</td>\n",
              "      <td>male</td>\n",
              "      <td>0</td>\n",
              "      <td>no</td>\n",
              "      <td>no</td>\n",
              "      <td>30</td>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "      <td>dsl</td>\n",
              "      <td>yes</td>\n",
              "      <td>...</td>\n",
              "      <td>no</td>\n",
              "      <td>yes</td>\n",
              "      <td>yes</td>\n",
              "      <td>no</td>\n",
              "      <td>one_year</td>\n",
              "      <td>no</td>\n",
              "      <td>electronic_check</td>\n",
              "      <td>70.40</td>\n",
              "      <td>2044.75</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 21 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a30a9d17-1945-4bfe-bece-c39e36c480ee')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a30a9d17-1945-4bfe-bece-c39e36c480ee button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a30a9d17-1945-4bfe-bece-c39e36c480ee');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "churn_female = df_full_train[df_full_train.gender == 'female'].churn.mean()\n",
        "churn_female"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gbO1ew74EGtL",
        "outputId": "f29bc14c-bea6-4359-d451-54e799e40f96"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.27682403433476394"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "churn_male = df_full_train[df_full_train.gender == 'male'].churn.mean()\n",
        "churn_male"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5YwGU8CDEN3s",
        "outputId": "5785191c-dcc1-4c88-d487-5ab62318ad3a"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.2632135306553911"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "global_churn = df_full_train.churn.mean()\n",
        "global_churn"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uLj2S1_TESdi",
        "outputId": "9223fa74-00c0-49fe-f35d-811a3e7c5341"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.26996805111821087"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "global_churn - churn_female\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gT-_iwGuEXqH",
        "outputId": "ddb221b8-8906-4c01-f487-5cc3e58adfe6"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-0.006855983216553063"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "global_churn - churn_male\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3h-P0g2bEaN-",
        "outputId": "25594b4c-6164-47d3-f936-fa74732e6f20"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.006754520462819769"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train.partner.value_counts()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HBDhtwOPEdEz",
        "outputId": "a7c0f521-d7ca-4063-b1de-4264d651f6f6"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "no     2932\n",
              "yes    2702\n",
              "Name: partner, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "churn_partner = df_full_train[df_full_train.partner == 'yes'].churn.mean()\n",
        "churn_partner"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eynkCSZDEga3",
        "outputId": "c642db93-8564-47ac-be54-08d2ae6f5b07"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.20503330866025166"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "global_churn - churn_partner\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8b8O4jYQEjhZ",
        "outputId": "32df84cd-505c-4d66-d90b-52f8b4bf11ed"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.06493474245795922"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "churn_no_partner = df_full_train[df_full_train.partner == 'no'].churn.mean()\n",
        "churn_no_partner"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5rDNLs_UElWC",
        "outputId": "c7a66d04-fe5b-4417-91aa-785c99a8d6dd"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.3298090040927694"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "global_churn - churn_no_partner\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q_FiJp1AEndK",
        "outputId": "e2731eb8-551f-4254-f9d3-d0ab5e21ee72"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-0.05984095297455855"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Risk ratio**\n"
      ],
      "metadata": {
        "id": "5RD4YCXsEvsE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "churn_no_partner / global_churn\n"
      ],
      "metadata": {
        "id": "kZmTK9WFEp6K",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b936aecb-0014-417d-fc21-2cff0be8e2ee"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.2216593879412643"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "churn_partner / global_churn\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PeDRhlpkcxa5",
        "outputId": "86d09136-5344-4d99-dc45-85c7fc962a8a"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7594724924338315"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "SELECT\n",
        "  - gender,\n",
        "  - AVG(churn),\n",
        "  - AVG(churn) - global_churn AS diff,\n",
        "  - AVG(churn) / global_churn AS risk\n",
        "\n",
        "FROM\n",
        "  - data\n",
        "\n",
        "GROUP BY\n",
        "  - gender;"
      ],
      "metadata": {
        "id": "qxhL3ChYdqT-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import display\n"
      ],
      "metadata": {
        "id": "HUKsQ0jSdhUz"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for c in categorical:\n",
        "    print(c)\n",
        "    df_group = df_full_train.groupby(c).churn.agg(['mean', 'count'])\n",
        "    df_group['diff'] = df_group['mean'] - global_churn\n",
        "    df_group['risk'] = df_group['mean'] / global_churn\n",
        "    display(df_group)\n",
        "    print()\n",
        "    print()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "jF72JxR0eSKX",
        "outputId": "91641e4d-3e04-4e1c-f65a-6c3e7eac0861"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "gender\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "            mean  count      diff      risk\n",
              "gender                                     \n",
              "female  0.276824   2796  0.006856  1.025396\n",
              "male    0.263214   2838 -0.006755  0.974980"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4bccdacc-8e99-4692-8641-6c2c05f77405\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>count</th>\n",
              "      <th>diff</th>\n",
              "      <th>risk</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>gender</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>female</th>\n",
              "      <td>0.276824</td>\n",
              "      <td>2796</td>\n",
              "      <td>0.006856</td>\n",
              "      <td>1.025396</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>male</th>\n",
              "      <td>0.263214</td>\n",
              "      <td>2838</td>\n",
              "      <td>-0.006755</td>\n",
              "      <td>0.974980</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4bccdacc-8e99-4692-8641-6c2c05f77405')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4bccdacc-8e99-4692-8641-6c2c05f77405 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4bccdacc-8e99-4692-8641-6c2c05f77405');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "seniorcitizen\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   mean  count      diff      risk\n",
              "seniorcitizen                                     \n",
              "0              0.242270   4722 -0.027698  0.897403\n",
              "1              0.413377    912  0.143409  1.531208"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-25d53273-61c7-4596-9765-e54a68658774\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>count</th>\n",
              "      <th>diff</th>\n",
              "      <th>risk</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>seniorcitizen</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.242270</td>\n",
              "      <td>4722</td>\n",
              "      <td>-0.027698</td>\n",
              "      <td>0.897403</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.413377</td>\n",
              "      <td>912</td>\n",
              "      <td>0.143409</td>\n",
              "      <td>1.531208</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-25d53273-61c7-4596-9765-e54a68658774')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-25d53273-61c7-4596-9765-e54a68658774 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-25d53273-61c7-4596-9765-e54a68658774');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "partner\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "             mean  count      diff      risk\n",
              "partner                                     \n",
              "no       0.329809   2932  0.059841  1.221659\n",
              "yes      0.205033   2702 -0.064935  0.759472"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-49429d6d-4160-43b1-8906-c7eefefc273d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>count</th>\n",
              "      <th>diff</th>\n",
              "      <th>risk</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>partner</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>no</th>\n",
              "      <td>0.329809</td>\n",
              "      <td>2932</td>\n",
              "      <td>0.059841</td>\n",
              "      <td>1.221659</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>yes</th>\n",
              "      <td>0.205033</td>\n",
              "      <td>2702</td>\n",
              "      <td>-0.064935</td>\n",
              "      <td>0.759472</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-49429d6d-4160-43b1-8906-c7eefefc273d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-49429d6d-4160-43b1-8906-c7eefefc273d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-49429d6d-4160-43b1-8906-c7eefefc273d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "dependents\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                mean  count      diff      risk\n",
              "dependents                                     \n",
              "no          0.313760   3968  0.043792  1.162212\n",
              "yes         0.165666   1666 -0.104302  0.613651"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ddae4cea-518b-4c6c-8d74-ec2f5b00bf48\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>count</th>\n",
              "      <th>diff</th>\n",
              "      <th>risk</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>dependents</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>no</th>\n",
              "      <td>0.313760</td>\n",
              "      <td>3968</td>\n",
              "      <td>0.043792</td>\n",
              "      <td>1.162212</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>yes</th>\n",
              "      <td>0.165666</td>\n",
              "      <td>1666</td>\n",
              "      <td>-0.104302</td>\n",
              "      <td>0.613651</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ddae4cea-518b-4c6c-8d74-ec2f5b00bf48')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ddae4cea-518b-4c6c-8d74-ec2f5b00bf48 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ddae4cea-518b-4c6c-8d74-ec2f5b00bf48');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "phoneservice\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                  mean  count      diff      risk\n",
              "phoneservice                                     \n",
              "no            0.241316    547 -0.028652  0.893870\n",
              "yes           0.273049   5087  0.003081  1.011412"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-61c245b6-c909-4a07-8117-2e22583c9d69\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>count</th>\n",
              "      <th>diff</th>\n",
              "      <th>risk</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>phoneservice</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>no</th>\n",
              "      <td>0.241316</td>\n",
              "      <td>547</td>\n",
              "      <td>-0.028652</td>\n",
              "      <td>0.893870</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>yes</th>\n",
              "      <td>0.273049</td>\n",
              "      <td>5087</td>\n",
              "      <td>0.003081</td>\n",
              "      <td>1.011412</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-61c245b6-c909-4a07-8117-2e22583c9d69')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-61c245b6-c909-4a07-8117-2e22583c9d69 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-61c245b6-c909-4a07-8117-2e22583c9d69');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "multiplelines\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                      mean  count      diff      risk\n",
              "multiplelines                                        \n",
              "no                0.257407   2700 -0.012561  0.953474\n",
              "no_phone_service  0.241316    547 -0.028652  0.893870\n",
              "yes               0.290742   2387  0.020773  1.076948"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8908514f-62df-4297-b2c2-a52882eb0519\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>count</th>\n",
              "      <th>diff</th>\n",
              "      <th>risk</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>multiplelines</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>no</th>\n",
              "      <td>0.257407</td>\n",
              "      <td>2700</td>\n",
              "      <td>-0.012561</td>\n",
              "      <td>0.953474</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>no_phone_service</th>\n",
              "      <td>0.241316</td>\n",
              "      <td>547</td>\n",
              "      <td>-0.028652</td>\n",
              "      <td>0.893870</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>yes</th>\n",
              "      <td>0.290742</td>\n",
              "      <td>2387</td>\n",
              "      <td>0.020773</td>\n",
              "      <td>1.076948</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8908514f-62df-4297-b2c2-a52882eb0519')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8908514f-62df-4297-b2c2-a52882eb0519 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8908514f-62df-4297-b2c2-a52882eb0519');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "internetservice\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                     mean  count      diff      risk\n",
              "internetservice                                     \n",
              "dsl              0.192347   1934 -0.077621  0.712482\n",
              "fiber_optic      0.425171   2479  0.155203  1.574895\n",
              "no               0.077805   1221 -0.192163  0.288201"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-03cc591a-282c-4695-bf6b-bc44b27e7e04\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>count</th>\n",
              "      <th>diff</th>\n",
              "      <th>risk</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>internetservice</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>dsl</th>\n",
              "      <td>0.192347</td>\n",
              "      <td>1934</td>\n",
              "      <td>-0.077621</td>\n",
              "      <td>0.712482</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>fiber_optic</th>\n",
              "      <td>0.425171</td>\n",
              "      <td>2479</td>\n",
              "      <td>0.155203</td>\n",
              "      <td>1.574895</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>no</th>\n",
              "      <td>0.077805</td>\n",
              "      <td>1221</td>\n",
              "      <td>-0.192163</td>\n",
              "      <td>0.288201</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-03cc591a-282c-4695-bf6b-bc44b27e7e04')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-03cc591a-282c-4695-bf6b-bc44b27e7e04 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-03cc591a-282c-4695-bf6b-bc44b27e7e04');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "onlinesecurity\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                         mean  count      diff      risk\n",
              "onlinesecurity                                          \n",
              "no                   0.420921   2801  0.150953  1.559152\n",
              "no_internet_service  0.077805   1221 -0.192163  0.288201\n",
              "yes                  0.153226   1612 -0.116742  0.567570"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f8281239-7988-4756-ba42-62bdcc9ad0a7\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>count</th>\n",
              "      <th>diff</th>\n",
              "      <th>risk</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>onlinesecurity</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>no</th>\n",
              "      <td>0.420921</td>\n",
              "      <td>2801</td>\n",
              "      <td>0.150953</td>\n",
              "      <td>1.559152</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>no_internet_service</th>\n",
              "      <td>0.077805</td>\n",
              "      <td>1221</td>\n",
              "      <td>-0.192163</td>\n",
              "      <td>0.288201</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>yes</th>\n",
              "      <td>0.153226</td>\n",
              "      <td>1612</td>\n",
              "      <td>-0.116742</td>\n",
              "      <td>0.567570</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f8281239-7988-4756-ba42-62bdcc9ad0a7')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f8281239-7988-4756-ba42-62bdcc9ad0a7 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f8281239-7988-4756-ba42-62bdcc9ad0a7');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "onlinebackup\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                         mean  count      diff      risk\n",
              "onlinebackup                                            \n",
              "no                   0.404323   2498  0.134355  1.497672\n",
              "no_internet_service  0.077805   1221 -0.192163  0.288201\n",
              "yes                  0.217232   1915 -0.052736  0.804660"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-23cde1f7-078a-48ed-a0ac-650573525f5b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>count</th>\n",
              "      <th>diff</th>\n",
              "      <th>risk</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>onlinebackup</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>no</th>\n",
              "      <td>0.404323</td>\n",
              "      <td>2498</td>\n",
              "      <td>0.134355</td>\n",
              "      <td>1.497672</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>no_internet_service</th>\n",
              "      <td>0.077805</td>\n",
              "      <td>1221</td>\n",
              "      <td>-0.192163</td>\n",
              "      <td>0.288201</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>yes</th>\n",
              "      <td>0.217232</td>\n",
              "      <td>1915</td>\n",
              "      <td>-0.052736</td>\n",
              "      <td>0.804660</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-23cde1f7-078a-48ed-a0ac-650573525f5b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-23cde1f7-078a-48ed-a0ac-650573525f5b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-23cde1f7-078a-48ed-a0ac-650573525f5b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "deviceprotection\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                         mean  count      diff      risk\n",
              "deviceprotection                                        \n",
              "no                   0.395875   2473  0.125907  1.466379\n",
              "no_internet_service  0.077805   1221 -0.192163  0.288201\n",
              "yes                  0.230412   1940 -0.039556  0.853480"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b6112ee7-793a-448a-8279-022610152581\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>count</th>\n",
              "      <th>diff</th>\n",
              "      <th>risk</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>deviceprotection</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>no</th>\n",
              "      <td>0.395875</td>\n",
              "      <td>2473</td>\n",
              "      <td>0.125907</td>\n",
              "      <td>1.466379</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>no_internet_service</th>\n",
              "      <td>0.077805</td>\n",
              "      <td>1221</td>\n",
              "      <td>-0.192163</td>\n",
              "      <td>0.288201</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>yes</th>\n",
              "      <td>0.230412</td>\n",
              "      <td>1940</td>\n",
              "      <td>-0.039556</td>\n",
              "      <td>0.853480</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b6112ee7-793a-448a-8279-022610152581')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b6112ee7-793a-448a-8279-022610152581 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b6112ee7-793a-448a-8279-022610152581');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "techsupport\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                         mean  count      diff      risk\n",
              "techsupport                                             \n",
              "no                   0.418914   2781  0.148946  1.551717\n",
              "no_internet_service  0.077805   1221 -0.192163  0.288201\n",
              "yes                  0.159926   1632 -0.110042  0.592390"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a9cd7033-50ee-426f-8026-47f80ccc40a2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>count</th>\n",
              "      <th>diff</th>\n",
              "      <th>risk</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>techsupport</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>no</th>\n",
              "      <td>0.418914</td>\n",
              "      <td>2781</td>\n",
              "      <td>0.148946</td>\n",
              "      <td>1.551717</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>no_internet_service</th>\n",
              "      <td>0.077805</td>\n",
              "      <td>1221</td>\n",
              "      <td>-0.192163</td>\n",
              "      <td>0.288201</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>yes</th>\n",
              "      <td>0.159926</td>\n",
              "      <td>1632</td>\n",
              "      <td>-0.110042</td>\n",
              "      <td>0.592390</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a9cd7033-50ee-426f-8026-47f80ccc40a2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a9cd7033-50ee-426f-8026-47f80ccc40a2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a9cd7033-50ee-426f-8026-47f80ccc40a2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "streamingtv\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                         mean  count      diff      risk\n",
              "streamingtv                                             \n",
              "no                   0.342832   2246  0.072864  1.269897\n",
              "no_internet_service  0.077805   1221 -0.192163  0.288201\n",
              "yes                  0.302723   2167  0.032755  1.121328"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6fc7c190-adad-4b12-b329-bdbc44eadf48\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>count</th>\n",
              "      <th>diff</th>\n",
              "      <th>risk</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>streamingtv</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>no</th>\n",
              "      <td>0.342832</td>\n",
              "      <td>2246</td>\n",
              "      <td>0.072864</td>\n",
              "      <td>1.269897</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>no_internet_service</th>\n",
              "      <td>0.077805</td>\n",
              "      <td>1221</td>\n",
              "      <td>-0.192163</td>\n",
              "      <td>0.288201</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>yes</th>\n",
              "      <td>0.302723</td>\n",
              "      <td>2167</td>\n",
              "      <td>0.032755</td>\n",
              "      <td>1.121328</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6fc7c190-adad-4b12-b329-bdbc44eadf48')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6fc7c190-adad-4b12-b329-bdbc44eadf48 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6fc7c190-adad-4b12-b329-bdbc44eadf48');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "streamingmovies\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                         mean  count      diff      risk\n",
              "streamingmovies                                         \n",
              "no                   0.338906   2213  0.068938  1.255358\n",
              "no_internet_service  0.077805   1221 -0.192163  0.288201\n",
              "yes                  0.307273   2200  0.037305  1.138182"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ab51af19-cfab-459c-9bb3-ad8f233f059c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>count</th>\n",
              "      <th>diff</th>\n",
              "      <th>risk</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>streamingmovies</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>no</th>\n",
              "      <td>0.338906</td>\n",
              "      <td>2213</td>\n",
              "      <td>0.068938</td>\n",
              "      <td>1.255358</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>no_internet_service</th>\n",
              "      <td>0.077805</td>\n",
              "      <td>1221</td>\n",
              "      <td>-0.192163</td>\n",
              "      <td>0.288201</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>yes</th>\n",
              "      <td>0.307273</td>\n",
              "      <td>2200</td>\n",
              "      <td>0.037305</td>\n",
              "      <td>1.138182</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ab51af19-cfab-459c-9bb3-ad8f233f059c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ab51af19-cfab-459c-9bb3-ad8f233f059c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ab51af19-cfab-459c-9bb3-ad8f233f059c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "contract\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                    mean  count      diff      risk\n",
              "contract                                           \n",
              "month-to-month  0.431701   3104  0.161733  1.599082\n",
              "one_year        0.120573   1186 -0.149395  0.446621\n",
              "two_year        0.028274   1344 -0.241694  0.104730"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e372ccb8-c5e1-4b41-9b52-a975e305bf10\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>count</th>\n",
              "      <th>diff</th>\n",
              "      <th>risk</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>contract</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>month-to-month</th>\n",
              "      <td>0.431701</td>\n",
              "      <td>3104</td>\n",
              "      <td>0.161733</td>\n",
              "      <td>1.599082</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>one_year</th>\n",
              "      <td>0.120573</td>\n",
              "      <td>1186</td>\n",
              "      <td>-0.149395</td>\n",
              "      <td>0.446621</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>two_year</th>\n",
              "      <td>0.028274</td>\n",
              "      <td>1344</td>\n",
              "      <td>-0.241694</td>\n",
              "      <td>0.104730</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e372ccb8-c5e1-4b41-9b52-a975e305bf10')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e372ccb8-c5e1-4b41-9b52-a975e305bf10 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e372ccb8-c5e1-4b41-9b52-a975e305bf10');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "paperlessbilling\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                      mean  count      diff      risk\n",
              "paperlessbilling                                     \n",
              "no                0.172071   2313 -0.097897  0.637375\n",
              "yes               0.338151   3321  0.068183  1.252560"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4cc2741d-505a-4bfb-9c31-3f5a90587f34\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>count</th>\n",
              "      <th>diff</th>\n",
              "      <th>risk</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>paperlessbilling</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>no</th>\n",
              "      <td>0.172071</td>\n",
              "      <td>2313</td>\n",
              "      <td>-0.097897</td>\n",
              "      <td>0.637375</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>yes</th>\n",
              "      <td>0.338151</td>\n",
              "      <td>3321</td>\n",
              "      <td>0.068183</td>\n",
              "      <td>1.252560</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4cc2741d-505a-4bfb-9c31-3f5a90587f34')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4cc2741d-505a-4bfb-9c31-3f5a90587f34 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4cc2741d-505a-4bfb-9c31-3f5a90587f34');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "paymentmethod\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                               mean  count      diff      risk\n",
              "paymentmethod                                                 \n",
              "bank_transfer_(automatic)  0.168171   1219 -0.101797  0.622928\n",
              "credit_card_(automatic)    0.164339   1217 -0.105630  0.608733\n",
              "electronic_check           0.455890   1893  0.185922  1.688682\n",
              "mailed_check               0.193870   1305 -0.076098  0.718121"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1b5c8779-42dd-4b0c-9f6a-471da7f3bc3a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mean</th>\n",
              "      <th>count</th>\n",
              "      <th>diff</th>\n",
              "      <th>risk</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>paymentmethod</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>bank_transfer_(automatic)</th>\n",
              "      <td>0.168171</td>\n",
              "      <td>1219</td>\n",
              "      <td>-0.101797</td>\n",
              "      <td>0.622928</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>credit_card_(automatic)</th>\n",
              "      <td>0.164339</td>\n",
              "      <td>1217</td>\n",
              "      <td>-0.105630</td>\n",
              "      <td>0.608733</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>electronic_check</th>\n",
              "      <td>0.455890</td>\n",
              "      <td>1893</td>\n",
              "      <td>0.185922</td>\n",
              "      <td>1.688682</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mailed_check</th>\n",
              "      <td>0.193870</td>\n",
              "      <td>1305</td>\n",
              "      <td>-0.076098</td>\n",
              "      <td>0.718121</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1b5c8779-42dd-4b0c-9f6a-471da7f3bc3a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1b5c8779-42dd-4b0c-9f6a-471da7f3bc3a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1b5c8779-42dd-4b0c-9f6a-471da7f3bc3a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**3.6 Feature importance: Mutual information**\n",
        "\n",
        "\n",
        "> Mutual information - concept from information theory, it tells us how much we can learn about one variable if we know the value of another\n",
        "\n",
        "https://en.wikipedia.org/wiki/Mutual_information\n"
      ],
      "metadata": {
        "id": "Dr0ZhALxfG6H"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import mutual_info_score\n"
      ],
      "metadata": {
        "id": "rdXoCzmGeSyR"
      },
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mutual_info_score(df_full_train.churn, df_full_train.contract)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gLw1OcGhflDz",
        "outputId": "a29bd653-a5a9-4189-be58-3217431626e5"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.0983203874041556"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mutual_info_score(df_full_train.gender, df_full_train.churn)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TePRouBBfnlG",
        "outputId": "73083920-bd6f-49ed-b5f9-bf7c9b630417"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.0001174846211139946"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mutual_info_score(df_full_train.contract, df_full_train.churn)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eezvNO0Zfo4-",
        "outputId": "9c6b4c00-9c4f-417d-d123-d50b901106b1"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.0983203874041556"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mutual_info_score(df_full_train.partner, df_full_train.churn)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KWjtV78efvwU",
        "outputId": "ef385387-32e1-490c-f46e-d3ec1633d170"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.009967689095399745"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def mutual_info_churn_score(series):\n",
        "    return mutual_info_score(series, df_full_train.churn)"
      ],
      "metadata": {
        "id": "jmyXJ1c9fwWz"
      },
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mi = df_full_train[categorical].apply(mutual_info_churn_score)\n",
        "mi.sort_values(ascending=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KvnIjJntfzwB",
        "outputId": "6cc31cce-d976-408f-c485-39158ea1d970"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "contract            0.098320\n",
              "onlinesecurity      0.063085\n",
              "techsupport         0.061032\n",
              "internetservice     0.055868\n",
              "onlinebackup        0.046923\n",
              "deviceprotection    0.043453\n",
              "paymentmethod       0.043210\n",
              "streamingtv         0.031853\n",
              "streamingmovies     0.031581\n",
              "paperlessbilling    0.017589\n",
              "dependents          0.012346\n",
              "partner             0.009968\n",
              "seniorcitizen       0.009410\n",
              "multiplelines       0.000857\n",
              "phoneservice        0.000229\n",
              "gender              0.000117\n",
              "dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**3.7 Feature importance: Correlation**\n",
        "\n",
        "\n",
        "How about numerical columns?\n",
        "\n",
        "\n",
        ">  Correlation coefficient - https://en.wikipedia.org/wiki/Pearson_correlation_coefficient\n",
        "\n"
      ],
      "metadata": {
        "id": "fDbZxTRkhQM9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train.tenure.max()"
      ],
      "metadata": {
        "id": "iDDHbgMTf1sA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "223c03c7-304f-486d-9b77-ea976236e093"
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "72"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train[numerical].corrwith(df_full_train.churn).abs()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZeP9RfxGiJ5L",
        "outputId": "988cfe77-8933-4c09-dab9-30c1edeb886b"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tenure            0.351885\n",
              "monthlycharges    0.196805\n",
              "totalcharges      0.196353\n",
              "dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train[df_full_train.tenure <= 2].churn.mean()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aDCvEcGWiKlI",
        "outputId": "e01af940-7604-4eb1-8392-b5bc00b4d387"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5953420669577875"
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train[(df_full_train.tenure > 2) & (df_full_train.tenure <= 12)].churn.mean()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gWEkZMy_iZI2",
        "outputId": "2fd41eb5-25b5-4ddf-b2f3-7befb93d3acd"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.3994413407821229"
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train[df_full_train.tenure > 12].churn.mean()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6N4eITp4inn8",
        "outputId": "ba70e437-6153-4fd1-b358-207d8beccf76"
      },
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.17634908339788277"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train[df_full_train.monthlycharges <= 20].churn.mean()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bftwg4RMi3W-",
        "outputId": "138a5efa-eede-4bd3-dd18-00706c1dc170"
      },
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.08795411089866156"
            ]
          },
          "metadata": {},
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train[(df_full_train.monthlycharges > 20) & (df_full_train.monthlycharges <= 50)].churn.mean()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p9oO4EJVi97e",
        "outputId": "39a6d2a5-0bec-44c0-af46-97ca3b5107e5"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.18340943683409436"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train[df_full_train.monthlycharges > 50].churn.mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X27gqqy1js7U",
        "outputId": "2ef00aac-784d-4f04-e0a8-51405d0f70ed"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.32499341585462205"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**3.8 One-hot encoding**\n",
        "- Use Scikit-Learn to encode categorical features"
      ],
      "metadata": {
        "id": "hCJ5GiiXKHuj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction import DictVectorizer\n"
      ],
      "metadata": {
        "id": "n_EcHAC6Iq-u"
      },
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dv = DictVectorizer(sparse=False)\n",
        "\n",
        "train_dict = df_train[categorical + numerical].to_dict(orient='records')\n",
        "X_train = dv.fit_transform(train_dict)\n",
        "\n",
        "val_dict = df_val[categorical + numerical].to_dict(orient='records')\n",
        "X_val = dv.transform(val_dict)"
      ],
      "metadata": {
        "id": "V_3J5gUQKXx7"
      },
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**3.9 Logistic regression**\n",
        "\n",
        "\n",
        "\n",
        "- Binary classification\n",
        "- Linear vs logistic regression"
      ],
      "metadata": {
        "id": "a2EZg3ScL5Wq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def sigmoid(z):\n",
        "    return 1 / (1 + np.exp(-z))"
      ],
      "metadata": {
        "id": "E2JOcJmVMBfe"
      },
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "z = np.linspace(-7, 7, 51)\n"
      ],
      "metadata": {
        "id": "fg6D6vv3Mc3U"
      },
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sigmoid(z)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v6Bn79xcNcSc",
        "outputId": "abb852ae-4201-43dd-85ce-62076b41009b"
      },
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([9.11051194e-04, 1.20508423e-03, 1.59386223e-03, 2.10780106e-03,\n",
              "       2.78699622e-03, 3.68423990e-03, 4.86893124e-03, 6.43210847e-03,\n",
              "       8.49286285e-03, 1.12064063e-02, 1.47740317e-02, 1.94550846e-02,\n",
              "       2.55807883e-02, 3.35692233e-02, 4.39398154e-02, 5.73241759e-02,\n",
              "       7.44679452e-02, 9.62155417e-02, 1.23467048e-01, 1.57095469e-01,\n",
              "       1.97816111e-01, 2.46011284e-01, 3.01534784e-01, 3.63547460e-01,\n",
              "       4.30453776e-01, 5.00000000e-01, 5.69546224e-01, 6.36452540e-01,\n",
              "       6.98465216e-01, 7.53988716e-01, 8.02183889e-01, 8.42904531e-01,\n",
              "       8.76532952e-01, 9.03784458e-01, 9.25532055e-01, 9.42675824e-01,\n",
              "       9.56060185e-01, 9.66430777e-01, 9.74419212e-01, 9.80544915e-01,\n",
              "       9.85225968e-01, 9.88793594e-01, 9.91507137e-01, 9.93567892e-01,\n",
              "       9.95131069e-01, 9.96315760e-01, 9.97213004e-01, 9.97892199e-01,\n",
              "       9.98406138e-01, 9.98794916e-01, 9.99088949e-01])"
            ]
          },
          "metadata": {},
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sigmoid(10000)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HZfAYZJNNgzI",
        "outputId": "3f4d56e5-6b88-4386-e022-3e0bcd1ecf02"
      },
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.0"
            ]
          },
          "metadata": {},
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plt.plot(z, sigmoid(z))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "djWBk-5oNnLq",
        "outputId": "9a0070e1-2385-48ce-c1e6-374e497171ae"
      },
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f0c13f6d1d0>]"
            ]
          },
          "metadata": {},
          "execution_count": 60
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAfhklEQVR4nO3deXiV9Z338ff3nKwkJAESEQiQRECJiIqAVBG1qHUbndGZCm0dt5Z2WvvY0Zl5dJyxTnu1Y5enT23HacW6VGtVxmrLjLjUpSIiCi5ssiVhC1vCFkjIes53/ki0EYEc5ST3WT6v6zrXue/73OR8xOTDL79zL+buiIhI8gsFHUBEROJDhS4ikiJU6CIiKUKFLiKSIlToIiIpIiOoNy4uLvaysrKg3l5EJCm9/fbbO9295FCvBVboZWVlLFmyJKi3FxFJSma28XCvacpFRCRFqNBFRFKECl1EJEWo0EVEUoQKXUQkRfRY6Gb2gJnVmdmKw7xuZvYzM6sys2VmNiH+MUVEpCexjNAfAi48wusXAaO7HrOAXxx9LBER+aR6PA7d3eebWdkRdrkceNg7r8O7yMyKzGyIu2+LU0YRkV7h7rRForR1RGnt6Hxu64jSHonSHvGu5yhtkSgdEScS7dwWiTod0T+vR71zPdq1LeIQiUaJOkS9c3v35eljB3Py8KK4//fE48SiYcDmbuu1Xds+VuhmNovOUTwjRoyIw1uLSDpxdw60RWhobv/YY39LB40tHTS1ddDY2kFT1+NAW4Tm9gjNbRFa2rstd5V3EI4pyEnYQo+Zu88GZgNMnDhRd9YQEaCzqHc1tbFtbwtb9jazdW8zdftb2dXYys7GVnY2tnUuN7X1WMLZGSHyszPI++CRFSY/O4Pi/GxyM8Odj6ww2ZkhsjPCZGeEyM4IkdXtOTMcIiMUIivDPlzODBsZ4RAZISMjbGSEjHCocz0U6lwPmREOdT5CRtfzn7eHDMys1/4e41HoW4Dh3dZLu7aJiHwoGnW2NjRTU99ETX0jNTubWL+zido9nQXeelBRZ4aNQXnZFPfPYlBeNmMG96e4fxYD+mVRmJtJUW4mhbmZFHzwnJNJXnaYjHD6HrwXj0KfC9xoZo8DpwMNmj8XSW8t7RFWbdvHii0NLKttYOXWfdTsbKSl/c+lnZ+dQUVJHpVDCzi/cjBDC3MYUpTLsKJchhTmMDAvq1dHs6mox0I3s8eAc4BiM6sFvg1kArj7L4F5wMVAFXAAuK63wopIYtqxr4WF1TtZVL2bpbV7WVfXSCTaOas6oF8m44YV8pnjRlJRkkdFcT7HleRR0j9bhR1nsRzlMrOH1x34RtwSiUjC23ugjUU1u3i9ahcLq3dSXd8EQFG/TE4uLeK8sYMZN6yQk0oLGVqYo+LuI4FdPldEksuuxlaeW7mdZ5ZtY1HNLqIO/bLCTC4fyIxJI/jMcYOoHFJAKKTyDooKXUQOa09TG8+v3M4zy7exsHoXkahTUZzHN84dxdljShhfWkRWRvp+CJloVOgi8jHvbNrDg69v4Nnl2+iIOiMH9eNrZ1dwyUlDGTukv6ZQEpQKXUQAaI9EeXbFdh5YsJ73Nu+lf3YGf/uZMq6YMIwThxaoxJOACl0kze1raeeRNzbyyBsb2b6vhbJB/fi3y07kytNKyc9WRSQT/d8SSVPtkSiPvbWJn764jt1NbUwdVcz3/moc5x5/jD7YTFIqdJE04+68tKqO7z+7ipr6JqZUDOT2iys5qbQw6GhylFToImlkxZYGvvfMKt6o2UVFSR73/e1Ezht7jObHU4QKXSQNtLRHuOvZ1fz6jQ0M6JfFdy4/kZmTR5CZxtc9SUUqdJEUt3JrA996/D3W1TVy7Rll3HzBGApyMoOOJb1AhS6SoqJR51cLavjR82sY0C+Lh6+fzLQxJUHHkl6kQhdJQdsamrllzlIWVu/icycO5t+vGM/AvKygY0kvU6GLpJiXVu3g5jlLaeuIctcVJ3HVpOH60DNNqNBFUsgjizby7T+soHJoAT+fOYHy4rygI0kfUqGLpAB358cvrOGeV6r57AnH8B9fOJV+WfrxTjf6Py6S5No6otz61DKeemcLMycP57uXj0vr27ClMxW6SBJrbO3g737zNq+t28nN54/hm58dpfnyNKZCF0lSdftauPbBxazZsZ8f/vV4Pj9xeM9/SFKaCl0kCe1qbGXGfYvY3tDC/ddM5Jzjjwk6kiQAFbpIkmls7eC6hxazZU8zj9xwOpPLBwYdSRKECl0kibR1RPnaI2+zcus+7v3SaSpz+Qh9FC6SJKJR5+Y577Ggaid3XXES51UODjqSJBgVukgScHf+7b9X8j/LtnHbRSfwN/oAVA5BhS6SBH7+chW/fmMjs6ZV8NWzjws6jiQoFbpIgvvtm5v4yR/X8tenlXLbRScEHUcSmApdJIG9u2kPd/xhBeceX8JdV5ykk4bkiFToIglq74E2bvztuxxbmMNPrzpVp/NLj3TYokgCikadW+YspW5/C09+7QwK++kOQ9Iz/ZMvkoDue62Gl1bXcfvFYzl5eFHQcSRJqNBFEsySDbv54fNruPikY7nmjLKg40gSUaGLJJDdTZ3z5qUDcrnryvH6EFQ+kZgK3cwuNLM1ZlZlZrce4vURZvaKmb1rZsvM7OL4RxVJbdGo8/dPvMfuA23c84UJFORo3lw+mR4L3czCwD3ARUAlMNPMKg/a7V+AOe5+KjAD+M94BxVJdb+cX82ra+u549JKxg0rDDqOJKFYRuiTgSp3r3H3NuBx4PKD9nGgoGu5ENgav4giqW/N9v385IW1XDJ+CF88fUTQcSRJxVLow4DN3dZru7Z1dyfwJTOrBeYB3zzUFzKzWWa2xMyW1NfXf4q4IqknEnVufWoZBbmZfPfycZo3l08tXh+KzgQecvdS4GLgETP72Nd299nuPtHdJ5aUlMTprUWS228WbeTdTXu549JKBuZlBR1Hklgshb4F6H5pt9Kubd3dAMwBcPc3gBygOB4BRVLZ1r3N/PC51UwbU8LlpwwNOo4kuVgKfTEw2szKzSyLzg895x60zyZgOoCZjaWz0DWnInIE7s6//n4FUYfv/aWmWuTo9Vjo7t4B3Ag8D6yi82iWlWb2HTO7rGu3W4CvmNlS4DHgWnf33gotkgqeWb6Nl1bXccsFYxg+sF/QcSQFxHQtF3efR+eHnd233dFt+X3gzPhGE0ldew+0cefclYwvLeS6M8uDjiMpQhfnEgnA9+etYs+Bdh6+/nTCIU21SHzo1H+RPraweidzltTylbMqqBxa0PMfEImRCl2kD7V1RLn96RWMHNSPb503Oug4kmI05SLSh36zaCPrdzbx4HWTyMkMBx1HUoxG6CJ9pKG5nZ+9vI6zRhdzzhidWCfxp0IX6SP/+UoVDc3t3HbRWB1zLr1ChS7SBzbvPsCDr2/gygml+iBUeo0KXaQP/PiFNYRCcMsFY4KOIilMhS7Sy5Zu3ssf3tvKl6dWMKQwN+g4ksJU6CK9yN353rxVFOdn8bVzjgs6jqQ4FbpIL/rj+zt4a/1uvnXeGPKzdZSw9C4VukgvaY9Eueu51RxXkseMScN7/gMiR0mFLtJLHn9rEzX1Tdx20VgywvpRk96n7zKRXtDU2sFPX1zHlIqBTB97TNBxJE1oUk+kFzyyaCO7mtq478ITdBKR9BmN0EXi7EBbB/fNr2HamBImjBgQdBxJIyp0kTj7Tdfo/Kbpo4KOImlGhS4SR81tEWbPr2HqqGJOGzkw6DiSZlToInH06Jsb2dnYxk261rkEQIUuEifNbRF++WoNZxw3iEllGp1L31Ohi8TJb9/axM7GVm6artG5BEOFLhIHLe0RfvlqNVMqBnJ6xaCg40iaUqGLxMHjb22ifn8rN03X5XElOCp0kaPU0h7hF69WM7lsIFMqNHcuwVGhixylOUs2s2NfKzedN1pnhUqgVOgiR6G1I8Iv/lTNxJEDOOM4zZ1LsFToIkfhD+9uZVtDC9+crtG5BE+FLvIpRaPO7NdqGDukgGmji4OOI6JCF/m0Xl5dR1VdI1+dVqHRuSQEFbrIp3Tv/GqGFeVyyfghQUcRAWIsdDO70MzWmFmVmd16mH0+b2bvm9lKM/ttfGOKJJa3N+5h8YY93DC1nEzdjUgSRI83uDCzMHAPcD5QCyw2s7nu/n63fUYDtwFnuvseM9MtWiSlzZ5fTWFuJlfpXqGSQGIZWkwGqty9xt3bgMeByw/a5yvAPe6+B8Dd6+IbUyRx1NQ38sL7O7h6ykjysnXTL0kcsRT6MGBzt/Xarm3djQHGmNnrZrbIzC481Bcys1lmtsTMltTX13+6xCIBu++19WSGQ1xzRlnQUUQ+Il6TfxnAaOAcYCZwn5kVHbyTu89294nuPrGkpCROby3Sd+r3t/K7d2q5ckIpJf2zg44j8hGxFPoWoPtEYWnXtu5qgbnu3u7u64G1dBa8SEr59cINtEeifOWs8qCjiHxMLIW+GBhtZuVmlgXMAOYetM/v6RydY2bFdE7B1MQxp0jgmlo7eGTRRj5XeSwVJflBxxH5mB4L3d07gBuB54FVwBx3X2lm3zGzy7p2ex7YZWbvA68A/+juu3ortEgQnli8mYbmdmadXRF0FJFDiukjenefB8w7aNsd3ZYduLnrIZJy2iNR7l+wnsllA5kwYkDQcUQOSWdEiMTg2RXb2bK3mVnTNDqXxKVCF+mBu3P/azVUFOfx2RN0zpwkLhW6SA/e3riHpbUNXHdmGaGQLsIliUuFLtKD+xespzA3kytPKw06isgRqdBFjmDz7gM8v3I7Xzh9BP2ydJq/JDYVusgRPLRwAyEzrvlMWdBRRHqkQhc5jP0t7TyxeDOXjB/CsYU5QccR6ZEKXeQwnli8mcbWDm6YqtP8JTmo0EUOIRJ1Hlq4gUllAxhf+rHrzIkkJBW6yCG8sHI7tXuaNTqXpKJCFzmE+xesZ/jAXM6vPDboKCIxU6GLHGTp5r0s2biH684oJ6wTiSSJqNBFDnL/gvX0z87g87pfqCQZFbpIN9samnlm+TaumjScfN0vVJKMCl2km4cWbsDddb9QSUoqdJEuTa0dPPbmJi4cdyzDB/YLOo7IJ6ZCF+nyu3dq2dfSwQ1Tdc1zSU4qdBEgGnUefH0Dpwwv4rSRuiORJCcVugjw8uo61u9s0olEktRU6CLArxbUMLQwh4vG6UQiSV4qdEl7K7c2sKhmN9ecUUZGWD8Skrz03Stp7/4F6+mXFWbG5BFBRxE5Kip0SWt1+1r476Vb+fzE4RTmZgYdR+SoqNAlrT2yaCMdUee6M8uCjiJy1FTokrZa2iP8ZtFGzhs7mJGD8oKOI3LUVOiStp56Zwt7DrTrUEVJGSp0SUvuzgOvr+fEoQWcXj4w6DgicaFCl7T06tp6quoauWFqOWa65rmkBhW6pKXZ82sYXJDNpeOHBh1FJG5U6JJ2ltc2sLB6FzdMLScrQz8Ckjr03Sxp59751fTPzmCmTiSSFBNToZvZhWa2xsyqzOzWI+x3pZm5mU2MX0SR+Nm06wDzlm/jC1NG0D9HJxJJaumx0M0sDNwDXARUAjPNrPIQ+/UHbgLejHdIkXj51YIawiHj+jN1qKKknlhG6JOBKnevcfc24HHg8kPs913gB0BLHPOJxM3upjbmLNnMX54yjMEFOUHHEYm7WAp9GLC523pt17YPmdkEYLi7P3OkL2Rms8xsiZktqa+v/8RhRY7Gw29soKU9yqxpuiORpKaj/lDUzELAT4BbetrX3We7+0R3n1hSUnK0by0Ss+a2CA+/sZHpJxzD6MH9g44j0itiKfQtwPBu66Vd2z7QHxgH/MnMNgBTgLn6YFQSyZNvb2Z3UxtfPfu4oKOI9JpYCn0xMNrMys0sC5gBzP3gRXdvcPdidy9z9zJgEXCZuy/plcQin1Ak6tz32npOHVHEpDLdL1RSV4+F7u4dwI3A88AqYI67rzSz75jZZb0dUORoPbdiO5t2H+Cr0yp0mr+ktIxYdnL3ecC8g7bdcZh9zzn6WCLx4e7cO7+a8uI8zq/U/UIltelMUUlpi2p2s6y2gS+fVU44pNG5pDYVuqS0n7+8juL8bK6cUBp0FJFep0KXlLV4w24WVu/ia2dXkJMZDjqOSK9ToUvKuvvFdRTnZ/HF00cGHUWkT6jQJSW9vXE3C6p2MmtaBblZGp1LelChS0q6+6UqBuVl8aUpGp1L+lChS8p5d9Me5q+t5yvTKuiXFdORuSIpQYUuKeful9YxMC+LqzU6lzSjQpeU8t7mvfxpTT1fPqucvGyNziW9qNAlpfzspXUU9cvkbz9TFnQUkT6nQpeUsax2Ly+vruPLU8vJ1+hc0pAKXVLGz15aR2FuJtecURZ0FJFAqNAlJazY0sCLq+q4YWq5bv4saUuFLinhR8+voSAng2vPLAs6ikhgVOiS9F5bV8+ra+u58bOjKNDoXNKYCl2SWiTqfO+ZVZQOyNWRLZL2VOiS1J56p5bV2/fzTxeeoCsqStpToUvSam6L8P9eWMvJw4v4i/FDgo4jEjgVuiSt+xfUsH1fC7dfPFb3ChVBhS5Jqn5/K7/4UzUXVA5mcvnAoOOIJAQVuiSlu19aS2tHlFsvOiHoKCIJQ4UuSaeqbj+PvbWZL54+goqS/KDjiCQMFboknbueXU2/zDD/Z/rooKOIJBQVuiSVN6p38eKqOr5+7igG5WcHHUckoajQJWm0dUS5c+5KhhXlcp1O8Rf5GF1jVJLG7PnVrNmxn/uvmaiTiEQOQSN0SQrV9Y387KUqLhk/hOljBwcdRyQhqdAl4UWjzm1PLScnM8S3/6Iy6DgiCUuFLgnviSWbeWv9bm6/ZCzH9M8JOo5IwlKhS0Kr29fC9+etYkrFQD4/cXjQcUQSWkyFbmYXmtkaM6sys1sP8frNZva+mS0zs5fMbGT8o0o6+vbclbR2RPn3K8brei0iPeix0M0sDNwDXARUAjPN7OCJzHeBie4+HngS+GG8g0r6eWHldp5dsZ2bpo+mvDgv6DgiCS+WEfpkoMrda9y9DXgcuLz7Du7+irsf6FpdBJTGN6akm30t7fzrH1ZwwrH9mTWtIug4IkkhlkIfBmzutl7bte1wbgCePdQLZjbLzJaY2ZL6+vrYU0ra+fd5q6nb38pdV44nM6yPekRiEdefFDP7EjAR+NGhXnf32e4+0d0nlpSUxPOtJYX899KtPPbWJmadVcEpw4uCjiOSNGI5U3QL0P3wgtKubR9hZucBtwNnu3trfOJJuqmpb+TW3y3jtJED+IfPHR90HJGkEssIfTEw2szKzSwLmAHM7b6DmZ0K3Atc5u518Y8p6aClPcLXH32HrIwQP595qqZaRD6hHn9i3L0DuBF4HlgFzHH3lWb2HTO7rGu3HwH5wH+Z2XtmNvcwX07ksO6cu5LV2/fzk6tOYWhRbtBxRJJOTBfncvd5wLyDtt3Rbfm8OOeSNPP0u7U8vngzXz/nOM49/pig44gkJf1OK4GrqtvPPz+1gsnlA7n5/DFBxxFJWip0CdSBtg6+/ug79MsK8/OZp5KheXORT03XQ5fAuDu3P72CdXWNPHL96Qwu0IW3RI6GhkMSmLueW83T727hlvPHMHV0cdBxRJKeCl0CMXt+Nfe+WsPVU0byjXNHBR1HJCWo0KXPPfl2Ld+ft5pLxg/hzstO1FUUReJEhS596sX3d/B/f7eMqaOK+cnnTyYcUpmLxIsKXfrM4g27+cZv3+HEoQX88urTyM7QjZ5F4kmFLn1i1bZ9XP/QYoYV5fLgtZPIz9YBViLxpkKXXresdi9X3/8m/bLCPHzDZAblZwcdSSQlqdClV72ypo4ZsxeRnRHm0S9PoXRAv6AjiaQs/d4rvWbO4s3c9vRyjh/cn4eum8QxOnFIpFep0CXu3J27X1rHT19cx1mji/nPL06gf05m0LFEUp4KXeKqIxLlX36/gscXb+aKCcP4gW4hJ9JnVOgSN7ub2vj7J97j1bX13HjuKG65YIxOGhLpQyp0iYtX19bzD/+1lL0H2vjeX43ji6ePDDqSSNpRoctRaWmP8IPnVvPg6xsYfUw+v75uMpVDC4KOJZKWVOjyqa3evo+bHnuPNTv2c+0ZZdx60QnkZOrsT5GgqNDlE+uIRHlo4QZ++NwaCnIzefC6SbptnEgCUKFLzNydV9bU8f15q6mqa+S8sYP5wZUn6cxPkQShQpeYrNzawPfnreL1ql2UF+dx79WncUHlYB3FIpJAVOhyRNsbWvjxC2v43Tu1FOVmcudfVPLFKSN1bLlIAlKhyyGt3bGfB1/fwNPv1hKNwlfOquAb546iMFdnfIokKhW6fCgadV5dW88Dr6/ntXU7yc4I8ZenDOPGz45i+EBdVEsk0anQhT1NbcxdupVfL9xAzc4mBhdk84+fO56Zk0cwMC8r6HgiEiMVeppqONDO8yu38z/Lt/F61U4iUeeU4UXcPeMULj5piObIRZKQCj2N1O9v5dW19TyzbCsLqnbSHnFGDOzHrGkVXDp+CCcOLQw6oogcBRV6CmtobufNml0srN7FwuqdrN3RCMCwolyuP7OcS8cPZdywAh16KJIiVOgpoiMSpbq+ieVbGlixpYF3N+1h+ZYGog45mSEmlQ3kr04tZeqoYpW4SIpSoScZd6dufyvV9Y3U1Dexbsd+lm9p4P1t+2hpjwLQLyvMuKGF3PjZ0Zx53CBOGVFEdoausSKS6lToCaiptYNtDc1s2dvC1r3NbN3bzKbdB6ipb2L9ziYaWzs+3LdfVpgThxYwc/IIThpWyPjSQsqL8wmHNAIXSTcxFbqZXQjcDYSBX7n7XQe9ng08DJwG7AKucvcN8Y2anNydlvYoDc3tHz72Hmhjz4E2dja2sbOxlV3dnnfsb2HvgfaPfI2QwZDCXCpK8rhywjAqSvKpKMmjoiSfIQU5hFTeIkIMhW5mYeAe4HygFlhsZnPd/f1uu90A7HH3UWY2A/gBcFVvBD4a0ajTEXUiUacjGqUj4rRHo7RHnI5IlPZI53J7JEpbR5TWjg+eI7R2rbe2R2huj9DcFqW5PUJLe4TmtghNbR00tXbQ1BqhsbXjw/V9zR20RaKHzZSfncGg/CyK87MZOagfk8oHMLQol2FFuQztegzun02GDiMUkR7EMkKfDFS5ew2AmT0OXA50L/TLgTu7lp8E/sPMzN09jlkBeGLxJmbPr8EdIu5E3YlG6Xz2zrL+4BF16IhGiUa7nuOcJiscIiczRE5mmPzsDPKyM8jLDjO0KKdrOYP+ORkU5mZSmJtJUW7Wn5f7ZVKcn01ulua2RSQ+Yin0YcDmbuu1wOmH28fdO8ysARgE7Oy+k5nNAmYBjBgx4lMFHpiXzQlDCgiZETIIm2FdyyEzwmEjbEY49OdHyIzMcOdyRsjICIfI6HotMxwiM/zBc+jD9ayMENkZ4a7nEFkZIbLCIXKzwuRmhsnJDGueWkQSSp9+KOrus4HZABMnTvxU4+XzKwdzfuXguOYSEUkFsUzMbgGGd1sv7dp2yH3MLAMopPPDURER6SOxFPpiYLSZlZtZFjADmHvQPnOBa7qW/xp4uTfmz0VE5PB6nHLpmhO/EXiezsMWH3D3lWb2HWCJu88F7gceMbMqYDedpS8iIn0opjl0d58HzDto2x3dlluAv4lvNBER+SR0cLOISIpQoYuIpAgVuohIilChi4ikCAvq6EIzqwc2BvLmh1fMQWe3JrhkyqusvSeZ8iZTVkjMvCPdveRQLwRW6InIzJa4+8Sgc8QqmfIqa+9JprzJlBWSL6+mXEREUoQKXUQkRajQP2p20AE+oWTKq6y9J5nyJlNWSLK8mkMXEUkRGqGLiKQIFbqISIpQoR+CmX3TzFab2Uoz+2HQeXpiZreYmZtZcdBZjsTMftT197rMzJ42s6KgMx3MzC40szVmVmVmtwad50jMbLiZvWJm73d9r94UdKaemFnYzN41s/8JOsuRmFmRmT3Z9f26ysw+E3SmWKjQD2Jm59J5j9ST3f1E4McBRzoiMxsOXABsCjpLDP4IjHP38cBa4LaA83xEtxuiXwRUAjPNrDLYVEfUAdzi7pXAFOAbCZ4X4CZgVdAhYnA38Jy7nwCcTHJkVqEfwt8Bd7l7K4C71wWcpyf/H/gnIOE/3Xb3F9y9o2t1EZ13v0okH94Q3d3bgA9uiJ6Q3H2bu7/TtbyfztIZFmyqwzOzUuAS4FdBZzkSMysEptF5nwfcvc3d9wabKjYq9I8bA5xlZm+a2atmNinoQIdjZpcDW9x9adBZPoXrgWeDDnGQQ90QPWELsjszKwNOBd4MNskR/ZTOwUc06CA9KAfqgQe7pod+ZWZ5QYeKRZ/eJDpRmNmLwLGHeOl2Ov9OBtL5K+wkYI6ZVQR1S70esv4zndMtCeNIed39D1373E7ndMGjfZktVZlZPvA74Fvuvi/oPIdiZpcCde7+tpmdE3SeHmQAE4BvuvubZnY3cCvwr8HG6llaFrq7n3e418zs74Cnugr8LTOL0nmBnvq+ytfd4bKa2Ul0jiSWmhl0Tl+8Y2aT3X17H0b8iCP93QKY2bXApcD0BLzvbCw3RE8oZpZJZ5k/6u5PBZ3nCM4ELjOzi4EcoMDMfuPuXwo416HUArXu/sFvO0/SWegJT1MuH/d74FwAMxsDZJF4V1vD3Ze7+zHuXubuZXR+E04Issx7YmYX0vkr92XufiDoPIcQyw3RE4Z1/kt+P7DK3X8SdJ4jcffb3L2063t1Bp03kk/EMqfrZ2izmR3ftWk68H6AkWKWliP0HjwAPGBmK4A24JoEHEkmq/8AsoE/dv1WscjdvxZspD873A3RA451JGcCVwPLzey9rm3/3HUPYDk63wQe7fqHvQa4LuA8MdGp/yIiKUJTLiIiKUKFLiKSIlToIiIpQoUuIpIiVOgiIilChS4ikiJU6CIiKeJ/AWIORW6+sYGvAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def linear_regression(xi):\n",
        "    result = w0\n",
        "    \n",
        "    for j in range(len(w)):\n",
        "        result = result + xi[j] * w[j]\n",
        "        \n",
        "    return result"
      ],
      "metadata": {
        "id": "PEIKiSH3N1kb"
      },
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def logistic_regression(xi):\n",
        "    score = w0\n",
        "    \n",
        "    for j in range(len(w)):\n",
        "        score = score + xi[j] * w[j]\n",
        "        \n",
        "    result = sigmoid(score)\n",
        "    return result"
      ],
      "metadata": {
        "id": "RTy2rEdEN59Y"
      },
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**3.10 Training logistic regression with Scikit-Learn**\n",
        "\n",
        "- Train a model with Scikit-Learn\n",
        "- Apply it to the validation dataset\n",
        "- Calculate the accuracy\n"
      ],
      "metadata": {
        "id": "ro5tFHnsOLnq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.linear_model import LogisticRegression\n"
      ],
      "metadata": {
        "id": "yUkPo_17N8wk"
      },
      "execution_count": 63,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = LogisticRegression(solver='lbfgs')\n",
        "# solver='lbfgs' is the default solver in newer version of sklearn\n",
        "# for older versions, you need to specify it explicitly\n",
        "model.fit(X_train, y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BczeBCj0_fyC",
        "outputId": "3cab294c-b094-45a2-9b52-14f18167ecee"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LogisticRegression()"
            ]
          },
          "metadata": {},
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.intercept_[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iATbA9Fq_vDd",
        "outputId": "12ddc4ad-42c7-4e07-e554-d61625304e41"
      },
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-0.10903461122930498"
            ]
          },
          "metadata": {},
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.coef_[0].round(3)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zQV0Kpnr__Ga",
        "outputId": "d025461a-bbbe-4384-bca1-1fb9ea7a8542"
      },
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 0.475, -0.175, -0.408, -0.03 , -0.078,  0.063, -0.089, -0.081,\n",
              "       -0.034, -0.073, -0.335,  0.317, -0.089,  0.004, -0.258,  0.141,\n",
              "        0.009,  0.063, -0.089, -0.081,  0.266, -0.089, -0.284, -0.231,\n",
              "        0.124, -0.166,  0.058, -0.087, -0.032,  0.07 , -0.059,  0.141,\n",
              "       -0.249,  0.215, -0.12 , -0.089,  0.102, -0.071, -0.089,  0.052,\n",
              "        0.213, -0.089, -0.232, -0.07 ,  0.   ])"
            ]
          },
          "metadata": {},
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y_pred = model.predict_proba(X_val)[:, 1]"
      ],
      "metadata": {
        "id": "ipujnPymADd_"
      },
      "execution_count": 67,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "churn_decision = (y_pred >= 0.5)"
      ],
      "metadata": {
        "id": "E6LlAdFAAF3u"
      },
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "(y_val == churn_decision).mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zhrSUO1GAMgN",
        "outputId": "73896a62-669b-43cb-c55a-fd0a2ba06a39"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8034066713981547"
            ]
          },
          "metadata": {},
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_pred = pd.DataFrame()\n",
        "df_pred['probability'] = y_pred\n",
        "df_pred['prediction'] = churn_decision.astype(int)\n",
        "df_pred['actual'] = y_val"
      ],
      "metadata": {
        "id": "t4OH7aArARUS"
      },
      "execution_count": 70,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_pred['correct'] = df_pred.prediction == df_pred.actual"
      ],
      "metadata": {
        "id": "DW7GhwjnAkHe"
      },
      "execution_count": 71,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_pred.correct.mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lG0d7JSJAn2F",
        "outputId": "9ca03690-054a-452a-f9cf-4066fe9fea52"
      },
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8034066713981547"
            ]
          },
          "metadata": {},
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "churn_decision.astype(int)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YzkgtDdUAp2Z",
        "outputId": "82dbdc88-20fb-4e4b-9258-2306b8593618"
      },
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 0, 0, ..., 0, 1, 1])"
            ]
          },
          "metadata": {},
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**3.11 Model interpretation**\n",
        "\n",
        "- Look at the coefficients\n",
        "- Train a smaller model with fewer features"
      ],
      "metadata": {
        "id": "zMeCzBwEBq4O"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "a = [1, 2, 3, 4]\n",
        "b = 'abcd'"
      ],
      "metadata": {
        "id": "jc69wX8OAr25"
      },
      "execution_count": 74,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dict(zip(a, b))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z0TVaSjNSfWX",
        "outputId": "89f54594-4ded-4eb2-ff8c-5406689cb503"
      },
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{1: 'a', 2: 'b', 3: 'c', 4: 'd'}"
            ]
          },
          "metadata": {},
          "execution_count": 75
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dict(zip(dv.get_feature_names(), model.coef_[0].round(3)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ugdXuYIHShnp",
        "outputId": "b09cd4ea-390a-4095-ad34-935748f2bfa0"
      },
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'contract=month-to-month': 0.475,\n",
              " 'contract=one_year': -0.175,\n",
              " 'contract=two_year': -0.408,\n",
              " 'dependents=no': -0.03,\n",
              " 'dependents=yes': -0.078,\n",
              " 'deviceprotection=no': 0.063,\n",
              " 'deviceprotection=no_internet_service': -0.089,\n",
              " 'deviceprotection=yes': -0.081,\n",
              " 'gender=female': -0.034,\n",
              " 'gender=male': -0.073,\n",
              " 'internetservice=dsl': -0.335,\n",
              " 'internetservice=fiber_optic': 0.317,\n",
              " 'internetservice=no': -0.089,\n",
              " 'monthlycharges': 0.004,\n",
              " 'multiplelines=no': -0.258,\n",
              " 'multiplelines=no_phone_service': 0.141,\n",
              " 'multiplelines=yes': 0.009,\n",
              " 'onlinebackup=no': 0.063,\n",
              " 'onlinebackup=no_internet_service': -0.089,\n",
              " 'onlinebackup=yes': -0.081,\n",
              " 'onlinesecurity=no': 0.266,\n",
              " 'onlinesecurity=no_internet_service': -0.089,\n",
              " 'onlinesecurity=yes': -0.284,\n",
              " 'paperlessbilling=no': -0.231,\n",
              " 'paperlessbilling=yes': 0.124,\n",
              " 'partner=no': -0.166,\n",
              " 'partner=yes': 0.058,\n",
              " 'paymentmethod=bank_transfer_(automatic)': -0.087,\n",
              " 'paymentmethod=credit_card_(automatic)': -0.032,\n",
              " 'paymentmethod=electronic_check': 0.07,\n",
              " 'paymentmethod=mailed_check': -0.059,\n",
              " 'phoneservice=no': 0.141,\n",
              " 'phoneservice=yes': -0.249,\n",
              " 'seniorcitizen': 0.215,\n",
              " 'streamingmovies=no': -0.12,\n",
              " 'streamingmovies=no_internet_service': -0.089,\n",
              " 'streamingmovies=yes': 0.102,\n",
              " 'streamingtv=no': -0.071,\n",
              " 'streamingtv=no_internet_service': -0.089,\n",
              " 'streamingtv=yes': 0.052,\n",
              " 'techsupport=no': 0.213,\n",
              " 'techsupport=no_internet_service': -0.089,\n",
              " 'techsupport=yes': -0.232,\n",
              " 'tenure': -0.07,\n",
              " 'totalcharges': 0.0}"
            ]
          },
          "metadata": {},
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "small = ['contract', 'tenure', 'monthlycharges']"
      ],
      "metadata": {
        "id": "dHdkPasISq0L"
      },
      "execution_count": 77,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_train[small].iloc[:10].to_dict(orient='records')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hpLrsXPcSuoy",
        "outputId": "37d973d9-65de-4fa6-f26c-d4ebc32302be"
      },
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'contract': 'two_year', 'monthlycharges': 115.5, 'tenure': 72},\n",
              " {'contract': 'month-to-month', 'monthlycharges': 95.25, 'tenure': 10},\n",
              " {'contract': 'month-to-month', 'monthlycharges': 75.55, 'tenure': 5},\n",
              " {'contract': 'month-to-month', 'monthlycharges': 80.85, 'tenure': 5},\n",
              " {'contract': 'two_year', 'monthlycharges': 20.1, 'tenure': 18},\n",
              " {'contract': 'month-to-month', 'monthlycharges': 30.5, 'tenure': 4},\n",
              " {'contract': 'month-to-month', 'monthlycharges': 75.1, 'tenure': 1},\n",
              " {'contract': 'month-to-month', 'monthlycharges': 70.3, 'tenure': 1},\n",
              " {'contract': 'two_year', 'monthlycharges': 19.75, 'tenure': 72},\n",
              " {'contract': 'month-to-month', 'monthlycharges': 109.9, 'tenure': 6}]"
            ]
          },
          "metadata": {},
          "execution_count": 78
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dicts_train_small = df_train[small].to_dict(orient='records')\n",
        "dicts_val_small = df_val[small].to_dict(orient='records')"
      ],
      "metadata": {
        "id": "Q4cU-DjNSxn1"
      },
      "execution_count": 79,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dv_small = DictVectorizer(sparse=False)\n",
        "dv_small.fit(dicts_train_small)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j4FhEFHyS7zl",
        "outputId": "b5da0dcf-5002-422f-faac-f5003b6b739a"
      },
      "execution_count": 80,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DictVectorizer(sparse=False)"
            ]
          },
          "metadata": {},
          "execution_count": 80
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dv_small.get_feature_names()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XUGBrcueS-bF",
        "outputId": "de64d3f6-ed98-4a14-c39b-ac488ce6cad1"
      },
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['contract=month-to-month',\n",
              " 'contract=one_year',\n",
              " 'contract=two_year',\n",
              " 'monthlycharges',\n",
              " 'tenure']"
            ]
          },
          "metadata": {},
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X_train_small = dv_small.transform(dicts_train_small)"
      ],
      "metadata": {
        "id": "c_ZOSDx6TCha"
      },
      "execution_count": 82,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_small = LogisticRegression(solver='lbfgs')\n",
        "model_small.fit(X_train_small, y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AW4DIkW9TGAI",
        "outputId": "9cce5c1b-2a72-4bd6-fd85-793c587bcd82"
      },
      "execution_count": 83,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LogisticRegression()"
            ]
          },
          "metadata": {},
          "execution_count": 83
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "w0 = model_small.intercept_[0]\n",
        "w0"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aai4wk5TTJeW",
        "outputId": "fd8f9d6d-e140-451c-b628-b818ba137144"
      },
      "execution_count": 84,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-2.476775663985754"
            ]
          },
          "metadata": {},
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "w = model_small.coef_[0]\n",
        "w.round(3)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XaKoO8VcTNvm",
        "outputId": "eb5d3d01-7efe-43e6-9579-71fa6d1dd2cc"
      },
      "execution_count": 85,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 0.97 , -0.025, -0.949,  0.027, -0.036])"
            ]
          },
          "metadata": {},
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dict(zip(dv_small.get_feature_names(), w.round(3)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5IzlFLphTQF9",
        "outputId": "9964222a-10cd-4212-bfcc-4fe27ac03994"
      },
      "execution_count": 86,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.\n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'contract=month-to-month': 0.97,\n",
              " 'contract=one_year': -0.025,\n",
              " 'contract=two_year': -0.949,\n",
              " 'monthlycharges': 0.027,\n",
              " 'tenure': -0.036}"
            ]
          },
          "metadata": {},
          "execution_count": 86
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "-2.47 + (-0.949) + 30 * 0.027 + 24 * (-0.036)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ry2XoS58TT74",
        "outputId": "691df7ad-507a-400c-8e8e-bc0cee798ad4"
      },
      "execution_count": 87,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "-3.473"
            ]
          },
          "metadata": {},
          "execution_count": 87
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sigmoid(_)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LHXfkxCfT5i1",
        "outputId": "7db6a346-15cc-43d5-92e2-081fe8c10066"
      },
      "execution_count": 88,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.030090303318277657"
            ]
          },
          "metadata": {},
          "execution_count": 88
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**3.12 Using the model**"
      ],
      "metadata": {
        "id": "FQ7KafyjUfmW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dicts_full_train = df_full_train[categorical + numerical].to_dict(orient='records')\n"
      ],
      "metadata": {
        "id": "gz2UkwAjT9K9"
      },
      "execution_count": 89,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dv = DictVectorizer(sparse=False)\n",
        "X_full_train = dv.fit_transform(dicts_full_train)"
      ],
      "metadata": {
        "id": "fyAVVj68Ut7N"
      },
      "execution_count": 90,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y_full_train = df_full_train.churn.values"
      ],
      "metadata": {
        "id": "HngBGC6zUv07"
      },
      "execution_count": 91,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = LogisticRegression(solver='lbfgs')\n",
        "model.fit(X_full_train, y_full_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7LDbGQL2U35w",
        "outputId": "5135503f-e7d0-44ff-d90a-ceefa54e53c9"
      },
      "execution_count": 92,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LogisticRegression()"
            ]
          },
          "metadata": {},
          "execution_count": 92
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dicts_test = df_test[categorical + numerical].to_dict(orient='records')\n"
      ],
      "metadata": {
        "id": "SZrzp-tSU8KS"
      },
      "execution_count": 93,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_test = dv.transform(dicts_test)\n"
      ],
      "metadata": {
        "id": "cXNfkEC9VA9L"
      },
      "execution_count": 94,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y_pred = model.predict_proba(X_test)[:, 1]"
      ],
      "metadata": {
        "id": "DX-RUqllVEYU"
      },
      "execution_count": 95,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "churn_decision = (y_pred >= 0.5)"
      ],
      "metadata": {
        "id": "7-wbxU30VJWw"
      },
      "execution_count": 96,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "(churn_decision == y_test).mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FF7A0QFZVKP6",
        "outputId": "9c6756d2-e7c6-4c5d-9de4-3833b74480e6"
      },
      "execution_count": 97,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.815471965933286"
            ]
          },
          "metadata": {},
          "execution_count": 97
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y_test"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xPQV6hDdVN5s",
        "outputId": "9fdde149-6b5f-4320-d71e-cc10a7833541"
      },
      "execution_count": 98,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 0, 0, ..., 0, 0, 1])"
            ]
          },
          "metadata": {},
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "customer = dicts_test[-1]\n",
        "customer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ztX-mUKvVR-_",
        "outputId": "c872f432-9282-41d1-a814-db90164dba0a"
      },
      "execution_count": 99,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'contract': 'month-to-month',\n",
              " 'dependents': 'yes',\n",
              " 'deviceprotection': 'yes',\n",
              " 'gender': 'female',\n",
              " 'internetservice': 'fiber_optic',\n",
              " 'monthlycharges': 104.2,\n",
              " 'multiplelines': 'yes',\n",
              " 'onlinebackup': 'no',\n",
              " 'onlinesecurity': 'yes',\n",
              " 'paperlessbilling': 'yes',\n",
              " 'partner': 'yes',\n",
              " 'paymentmethod': 'electronic_check',\n",
              " 'phoneservice': 'yes',\n",
              " 'seniorcitizen': 0,\n",
              " 'streamingmovies': 'yes',\n",
              " 'streamingtv': 'yes',\n",
              " 'techsupport': 'no',\n",
              " 'tenure': 17,\n",
              " 'totalcharges': 1743.5}"
            ]
          },
          "metadata": {},
          "execution_count": 99
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X_small = dv.transform([customer])"
      ],
      "metadata": {
        "id": "vx2i9kSQVZOS"
      },
      "execution_count": 100,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.predict_proba(X_small)[0, 1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4kSqr57gVeFd",
        "outputId": "edef54f9-fe3d-46be-d7e9-f2ae7a996be5"
      },
      "execution_count": 101,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5968852088243913"
            ]
          },
          "metadata": {},
          "execution_count": 101
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y_test[-1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eFAcicBFVgnE",
        "outputId": "78af9a24-99fb-490f-ff75-98b03a3c2dbc"
      },
      "execution_count": 102,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {},
          "execution_count": 102
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**4. Evaluation Metrics for Classification**\n",
        "\n",
        "In the previous session we trained a model for predicting churn. How do we know if it's good?"
      ],
      "metadata": {
        "id": "Safw5s_XmtnP"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**4.1 Evaluation metrics: session overview**\n",
        "\n",
        "Dataset: https://www.kaggle.com/blastchar/telco-customer-churn https://raw.githubusercontent.com/alexeygrigorev/mlbookcamp-code/master/chapter-03-churn-prediction/WA_Fn-UseC_-Telco-Customer-Churn.csv\n",
        "\n",
        "Metric - function that compares the predictions with the actual values and outputs a single number that tells how good the predictions are"
      ],
      "metadata": {
        "id": "NTden3xOmwlF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "import matplotlib.pyplot as plt"
      ],
      "metadata": {
        "id": "RJXQMvIBm42z"
      },
      "execution_count": 103,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.feature_extraction import DictVectorizer\n",
        "from sklearn.linear_model import LogisticRegression"
      ],
      "metadata": {
        "id": "Z8nat5sym5c5"
      },
      "execution_count": 104,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('data-week-3.csv')\n",
        "\n",
        "df.columns = df.columns.str.lower().str.replace(' ', '_')\n",
        "\n",
        "categorical_columns = list(df.dtypes[df.dtypes == 'object'].index)\n",
        "\n",
        "for c in categorical_columns:\n",
        "    df[c] = df[c].str.lower().str.replace(' ', '_')\n",
        "\n",
        "df.totalcharges = pd.to_numeric(df.totalcharges, errors='coerce')\n",
        "df.totalcharges = df.totalcharges.fillna(0)\n",
        "\n",
        "df.churn = (df.churn == 'yes').astype(int)"
      ],
      "metadata": {
        "id": "zov2Arpym75M"
      },
      "execution_count": 105,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_full_train, df_test = train_test_split(df, test_size=0.2, random_state=1)\n",
        "df_train, df_val = train_test_split(df_full_train, test_size=0.25, random_state=1)\n",
        "\n",
        "df_train = df_train.reset_index(drop=True)\n",
        "df_val = df_val.reset_index(drop=True)\n",
        "df_test = df_test.reset_index(drop=True)\n",
        "\n",
        "y_train = df_train.churn.values\n",
        "y_val = df_val.churn.values\n",
        "y_test = df_test.churn.values\n",
        "\n",
        "del df_train['churn']\n",
        "del df_val['churn']\n",
        "del df_test['churn']"
      ],
      "metadata": {
        "id": "x3iEL_cDnA-Z"
      },
      "execution_count": 106,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "numerical = ['tenure', 'monthlycharges', 'totalcharges']\n",
        "\n",
        "categorical = [\n",
        "    'gender',\n",
        "    'seniorcitizen',\n",
        "    'partner',\n",
        "    'dependents',\n",
        "    'phoneservice',\n",
        "    'multiplelines',\n",
        "    'internetservice',\n",
        "    'onlinesecurity',\n",
        "    'onlinebackup',\n",
        "    'deviceprotection',\n",
        "    'techsupport',\n",
        "    'streamingtv',\n",
        "    'streamingmovies',\n",
        "    'contract',\n",
        "    'paperlessbilling',\n",
        "    'paymentmethod',\n",
        "]"
      ],
      "metadata": {
        "id": "hZsefpnSnGZ4"
      },
      "execution_count": 107,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dv = DictVectorizer(sparse=False)\n",
        "\n",
        "train_dict = df_train[categorical + numerical].to_dict(orient='records')\n",
        "X_train = dv.fit_transform(train_dict)\n",
        "\n",
        "model = LogisticRegression()\n",
        "model.fit(X_train, y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JvBU0MTHnJUK",
        "outputId": "c8be75f4-8d37-4f0d-8b11-32866d101cfe"
      },
      "execution_count": 108,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LogisticRegression()"
            ]
          },
          "metadata": {},
          "execution_count": 108
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "val_dict = df_val[categorical + numerical].to_dict(orient='records')\n",
        "X_val = dv.transform(val_dict)\n",
        "\n",
        "y_pred = model.predict_proba(X_val)[:, 1]\n",
        "churn_decision = (y_pred >= 0.5)\n",
        "(y_val == churn_decision).mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "94_wEXe5nLn_",
        "outputId": "0c036952-5c50-42db-c42f-8453008d16b3"
      },
      "execution_count": 109,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8034066713981547"
            ]
          },
          "metadata": {},
          "execution_count": 109
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**4.2 Accuracy and dummy model**\n",
        "\n",
        "*  Evaluate the model on different thresholds\n",
        "*   Check the accuracy of dummy baselines\n"
      ],
      "metadata": {
        "id": "RSc6eoYlnQyx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "len(y_val)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2f19A82jnNn0",
        "outputId": "fe21db9c-02be-4e94-f560-606ad4a57cc7"
      },
      "execution_count": 110,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1409"
            ]
          },
          "metadata": {},
          "execution_count": 110
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "(y_val == churn_decision).mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ecT0x2kznfpl",
        "outputId": "92432483-112a-4be5-f88f-2de2fb90e30c"
      },
      "execution_count": 111,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8034066713981547"
            ]
          },
          "metadata": {},
          "execution_count": 111
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "1132/ 1409"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gEbU0Aj-nhqy",
        "outputId": "abb8eb73-d889-4a4a-d856-25b0e638d334"
      },
      "execution_count": 112,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8034066713981547"
            ]
          },
          "metadata": {},
          "execution_count": 112
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import accuracy_score"
      ],
      "metadata": {
        "id": "0_J3P8cbnjjy"
      },
      "execution_count": 113,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_score(y_val, y_pred >= 0.5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b-V_mu96nlZY",
        "outputId": "6da52421-0532-4a8f-8e01-871bd2cc08cb"
      },
      "execution_count": 114,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8034066713981547"
            ]
          },
          "metadata": {},
          "execution_count": 114
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "thresholds = np.linspace(0, 1, 21)\n",
        "\n",
        "scores = []\n",
        "\n",
        "for t in thresholds:\n",
        "    score = accuracy_score(y_val, y_pred >= t)\n",
        "    print('%.2f %.3f' % (t, score))\n",
        "    scores.append(score)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cW-IrnOUnm9P",
        "outputId": "2a015d26-fe7f-4813-a94b-f2cb749e3ed1"
      },
      "execution_count": 115,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.00 0.274\n",
            "0.05 0.509\n",
            "0.10 0.591\n",
            "0.15 0.666\n",
            "0.20 0.710\n",
            "0.25 0.739\n",
            "0.30 0.760\n",
            "0.35 0.772\n",
            "0.40 0.785\n",
            "0.45 0.793\n",
            "0.50 0.803\n",
            "0.55 0.801\n",
            "0.60 0.795\n",
            "0.65 0.786\n",
            "0.70 0.766\n",
            "0.75 0.744\n",
            "0.80 0.735\n",
            "0.85 0.726\n",
            "0.90 0.726\n",
            "0.95 0.726\n",
            "1.00 0.726\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plt.plot(thresholds, scores)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "11NRW0CCnpAD",
        "outputId": "0201b20e-9b77-4652-a5b6-e7411839f825"
      },
      "execution_count": 116,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f0c10932d10>]"
            ]
          },
          "metadata": {},
          "execution_count": 116
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAfOElEQVR4nO3de3xVZ53v8c8vVyABQkgINAkk3Etb6CVS206tvVF0tJ2ZOk7rXatVa7Xjbaa+nJfj6Zxz5jgzWnWGmbFTa723VWccHNFeaGsVLZBaoIUmIRAgAXIhae7k/jt/7F1M00A2sHdW9trf9+uVV/da+2nWb5HkmyfPs9azzN0REZHklxZ0ASIiEh8KdBGRkFCgi4iEhAJdRCQkFOgiIiGREdSBCwoKvKysLKjDi4gkpeeee+6YuxeO915ggV5WVkZlZWVQhxcRSUpmdvBk72nIRUQkJBToIiIhoUAXEQkJBbqISEgo0EVEQkKBLiISEjEFupmtN7NqM6s1s7vHeX+hmT1lZs+b2S4ze3P8SxURkVOZ8Dp0M0sHNgDXAw3AdjPb6O57RjX7G+ARd/83M1sFbALKElCvSMJ0HB/kuYNt7DnSSWZ6GrnTMsjNziAnK+MPr7Mj/83NzmBaZhpmFnTZIifEcmPRWqDW3fcDmNlDwE3A6EB3YFb09WzgSDyLFEmEtp4BttW1sbWulW11bew52snpPB4gPc3IyUr/Q9BHQ79kzgyWFOawZF4uSwtzKc6bTlqagl8SL5ZALwbqR203AJeOafNF4DEz+ziQA1wXl+pE4qips4+tdW1sq2tl6/429jZ3AzAtM42LF87hrmuXsbY8nwtL8xhx6Okfort/iO6+oT+87n/l9TDd/YP09A//oc3AEJ3HB/nF4aO09w6eOG52RhqLC3MjIV+Yy5J5kdeLC3KZnpUe1D+HhFC8bv2/FXjQ3b9sZpcB3zWz8919ZHQjM7sduB1g4cKFcTq0yPjq23pf1QM/0NoLQG52BpcsmsOfXlzMpeX5XFCcR1bGa6eTcrMzKDrDY7f1DLCvpZt9zd3sa+mmtrmbXQ0d/PyFo6/6K6A4b/qJgF86L5eV82dx3jmzmJapoJfTF0ugHwZKR22XRPeNdhuwHsDdf2dm04ACoHl0I3e/D7gPoKKiQs++k7jpHRjixcOd7KxvZ2dDO88faudw+3EAZk/PZG15Pu96/SLWluezasEsMtITe4FXfk4W+Tn5vK4s/1X7+waHOdDaw77mnkjgRz+217VxfHAYgIw0Y8X8mawuyePC0tmsLslj2bzchNcsyS+WQN8OLDOzciJBfgvwjjFtDgHXAg+a2bnANKAlnoWKvGJweITqxi521Lezq6GdXQ0d1DR1MRLtIhTnTefChXl8+KrFrC3PZ/m8mVNmDHtaZjor589i5fxZr9o/MuIc7exj9+EOdja0s7O+g5/vOsIPtx0CYHpmOucXz2JNSR6rS/O4sCSP0vzpmpSVV7FYHhIdvQzxq0A68IC7/x8zuweodPeN0Stb/gPIJTJB+lfu/tipPmdFRYVrtUWZyMiIU9faw65oyO1saGf3kU4GhiKjefk5Wawumc2akjzWRHuzBbnZAVcdHyMjzoHWHnY1dJz45fXiqHOfMyOT1SV5rCmJnHdF2RzyZmQFXLUkmpk95+4V474XS6AnggJdxuPu1DR189juRp6ta2VXQwddfUNApJd6Qcls1pTMZk1pHmtK8iiZk1q91Ff+OtnZ0M6u6C+4V/46mZaZxrtfv4gPX7UkNL/U5LUU6DKlDY84vz/0Mo/tbuSxPU0cbO3FDFYtmMWa6PDCmtI8ls7LJX2KDJ1MJb0DQ7zQ0MHDlfX89PnDZGek857LFnH7GxYzV8EeOgp0mXL6Bof57b5jPLa7iSdeauJY9wBZ6WlcvnQu61bN57pz5zFv1rSgy0w6+1q6+efNe/nvnUeYnpnOey8v4/YrFzMnR0MxYaFAlymh4/ggT1U189ieRp6ubqF3YJjc7AyuXjmPdauKeOOKQmZOywy6zFCobe7m65v38rNdR5iRmc77rijjQ1cu1hh7CCjQJTCNHX08vicylPK7fa0MjTiFM7O5flUR61YVcdmSuWRn6JrrRKlp6uJrm/fy811Hyc3O4ANXlHHbHy1m9gz94kxWCnSZNO7O7iOdbH6pmSermtjZ0AFAeUEO684r4obz5nNhSd6UuYwwVVQ1dvK1J/byixcbmTktgw9cUc4H/qic2dMV7MlGgS4J1TswxG/2HuPJqmaerGqmuasfM7iwNI/rzi3ihvOKWFKYm1JXo0xVe4508rXNNTy6u4lZ0zL44JWLef8VZRrqSiIKdIm7+rZenqxqZnNVM8/ub2VgaITc7AzesLyAa1ZGxsN16dzU9eLhDr76xF6eeKmJ2dMzuf0Ni3nf5WXkZMdrNRBJFAW6nLWh4RGer28/MZRS0xRZ2Kq8IIdrVs7j2pXzqCjLH3dNFJm6Xmjo4KtP1LC5qpm5OVl87OqlvOPShVpLZgpToMsZGRoe4Ze7G3l8TxNPV7fQcXyQjDRjbXk+16ycxzUr57G4MDfoMiUOfn/oZb78WDVbaltZMHsad127jLddUqL1Y6YgBbqclpER539eOMq9j9dQd6yH/Jwsrl4RCfArlxcwS+OtobWl9hj/+Gg1O+rbKS/I4ZPXL+ctFyzQJPYUokCXmLg7m19q5p8eq6aqsYsVRTP59LrlXHtuke7QTCFjvw9Wzp/JZ9at4Npz52liewpQoMuEfrsv0jN7/lA7ZXNn8Mnrl/PW1eeoZ5bCRkacn+06wr2P13CgtZeLFubx2RtWcPmSgqBLS2kKdDmp5w+9zD9Fx07nz5rGXddFxk4zNXYqUYPDI/zkuQa+tnkvRzv6uGLpXD6zbgUXLZwTdGkpSYEur1HV2MmXH6vh8T1NzM3J4o6rl/JOXd0gp9A3OMz3tx7iX5+qpbVngOtXFfHpdctfs7a7JJYCXU44cKyHe5+oYePOI+RmZ/DhNyzm/VeU6/pjiVlP/xDf2lLHN57ZT3f/EG9dfQ5vWb2A15XlaxGwSaBAF460H+efn9zLI5UNZKYb77+inA+/QYs1yZlr7x3gG8/s59u/PUDvQOTxeSvnz2RteT6Xls/ldeVzmDdTK2bGmwI9hXUcH+Trm/fy3WcP4u6889JF3HH1Ev2gSdz0Dw2zq6GDrftb2VrXxnMHXz4R8IsLcrh0cf6JkD8nb3rA1SY/BXqK2lbXxicf3sHRjuPcfHEJn7h2GaX5M4IuS0JucHiE3Uc62VbXytb9bWw70HbiqVMlc6aztjyf15fPZW15PovmztClkKdJgZ5iBodH+PrmvWx4qpbS/Bl87ZaLuLA0L+iyJEUNjzhVjZ1sq2s7EfBtPQMAFM3K5uKFcyLPRi2dzQXFs7VQ2AQU6CnkYGsPdz20gx317bztkhK+eON55GrCU6YQd6e2uZutdW1sq2tjR307h9p6ATCDJYW5rC6ZzYWleawuyePcBTO1Zv4oCvQU4O785+8P84X/fpH0NOP//tkFvGX1OUGXJRKTtp4BdjW0s6uhg5317exsaOdYd6QXn5lunLtgFmtK8lgdfUD4ksLUfb6sAj3kOo4P8vn/eoH/2XWUteX53PsXF1KsySdJYu7OkY4+dtW3s6OhnV31HbxwuIPu/shYfE5WOucXz6Z4znSM5Av2my8pPuM7bk8V6PpbPMm9MvHZ1NnHZ29YwUeuWpKyPRcJDzOjOG86xXnTedMFC4DIUgT7j3Wzs76DnQ3t7GzoYOv+toArPTNXLkvM8gkK9CQ1duLzxx+9XBOfEmppacbSeTNZOm8mN19SEnQ5U5ICPQmNnvj880tK+FtNfIoICvSk4u785PeH+dvoxOeGd1zMH69eEHRZIjJFKNCTREfvIJ//qSY+ReTkFOhJYEd9O3d87zmau/o18SkiJ6VAn+JePNzBu+/fSl5OJj/56OWs0cSniJxETE8xMLP1ZlZtZrVmdvc4799rZjuiHzVm1h7/UlNPbXM373lgG7OmZ/Lw7ZcpzEXklCbsoZtZOrABuB5oALab2UZ33/NKG3f/5Kj2HwcuSkCtKaW+rZd33b+VNDO+98FLtUqdiEwolh76WqDW3fe7+wDwEHDTKdrfCvwwHsWlqubOPt71za30Dgzx3dvWUl6QE3RJIpIEYgn0YqB+1HZDdN9rmNkioBx48iTv325mlWZW2dLScrq1poT23gHe/c1ttHT18+AH1nLuAj3eS0RiE+8nAd8C/Njdh8d7093vc/cKd68oLCyM86GTX3f/EO/91nbqWnu4/z0VXKyH8IrIaYgl0A8DpaO2S6L7xnMLGm45I32Dw3zw29t58XAHG95xMZcvTcxaDyISXrEE+nZgmZmVm1kWkdDeOLaRma0E5gC/i2+J4Tc4PMLHvv97tta18ZW3r+H6VUVBlyQiSWjCQHf3IeBO4FHgJeARd99tZveY2Y2jmt4CPORBrcebpIZHnE89spPNVc387z85n5suHHd6QkRkQjHdWOTum4BNY/Z9Ycz2F+NXVmpwd/7mpy/ws51HuPtNK3nnpYuCLklEkli8J0UlRu7O3/+iih9uq+djVy/hI1ctCbokEUlyCvSA/MuTtdz3zH7ee9kiPrNuRdDliEgIKNAD8K0tdXz58Rr+7OJi/vat52GmhbZE5Owp0CfZjyrr+V8/28MN5xXxDzevJk2rJopInCjQJ9GmF47y1z/ZxZXLCvj6rReRka5/fhGJHyXKJPlVTQt3PfQ8Fy2cwzfefQnZGelBlyQiIaNAnwRdfYN88uEdLCnM5YH3vY4ZWVqGXkTiT8kyCe57Zj9tPQM8+P7XMXt6ZtDliEhIqYeeYM2dfdz/6zresnoBq0v0gAoRSRwFeoLd+8RehkZG+OwNutZcRBJLgZ5Atc3dPFJZzzsvXcSiuXpIhYgklgI9gf7x0SqmZaRx5zVLgy5FRFKAAj1BnjvYxqO7m/jwVUsoyM0OuhwRSQEK9ARwd/5+UxWFM7P54JXlQZcjIilCgZ4AT7zUTOXBl/nL65bpmnMRmTQK9DgbGh7hS7+sYnFBDm+vKJ34fxARiRMFepz9+LkGapu7+av1K8jUWi0iMomUOHF0fGCYe5+o4aKFedxw3vygyxGRFKNAj6MHttTR1NnP5950rtY4F5FJp0CPk7aeAf796X1cd+481pbnB12OiKQgBXqc/MuTtfQMDPHX61cGXYqIpCgFehzUt/Xy3WcP8OeXlLKsaGbQ5YhIilKgx8GXH6smzYy/vH5Z0KWISApToJ+lFw938NMdR/jAH5WzYPb0oMsRkRSmQD9LX/plFXkzMvnIVUuCLkVEUpwC/Sz8em8Lv957jDuvXqonEYlI4BToZ2hkxPl/v6iiOG86775sUdDliIgo0M/Uz3YdYfeRTj5zw3KyM9KDLkdEJLZAN7P1ZlZtZrVmdvdJ2rzdzPaY2W4z+0F8y5xa+oeG+cdHq1m1YBY3rSkOuhwREQAmXNvVzNKBDcD1QAOw3cw2uvueUW2WAZ8DrnD3l81sXqIKngq+/+whGl4+znc+cAFpabrFX0Smhlh66GuBWnff7+4DwEPATWPafAjY4O4vA7h7c3zLnDo6+wb55yf3csXSuVy5rCDockRETogl0IuB+lHbDdF9oy0HlpvZFjN71szWj/eJzOx2M6s0s8qWlpYzqzhg3/jVPl7uHeTu9VqAS0SmlnhNimYAy4A3ArcC/2FmeWMbuft97l7h7hWFhYVxOvTkaers45u/qePGNedwQcnsoMsREXmVWAL9MDD60Tsl0X2jNQAb3X3Q3euAGiIBHypffaKG4RHnM+tWBF2KiMhrxBLo24FlZlZuZlnALcDGMW1+SqR3jpkVEBmC2R/HOgN34FgPj1Q28M5LF7Fw7oygyxEReY0JA93dh4A7gUeBl4BH3H23md1jZjdGmz0KtJrZHuAp4LPu3pqoooPwL0/VkpFm3HG1bvEXkakppkfSu/smYNOYfV8Y9dqBT0U/Qudgaw//9fxh3ntZGfNmTgu6HBGRcelO0RhseKqW9DTjw1ctDroUEZGTUqBPoL6tl//8/WHesXYhRbPUOxeRqUuBPoF/fbqWNDMtjysiU54C/RQaXu7lR5UN/MXrSpk/W71zEZnaFOin8K9P78MMPvpG9c5FZOpToJ/Ekfbj/KiynrdXlHJOnh4tJyJTnwL9JP7t6X0A3HH10oArERGJjQJ9HEc7jvPw9nredkkJxeqdi0iSUKCP4xu/2s+IO3e8Ub1zEUkeCvQxmjr7+MG2Q9x8cQml+VqzRUSShwJ9jH//1T6GR5yPaexcRJKMAn2U5s4+frD1EH96UbFWVBSRpKNAH+W+Z/YzNOLcqd65iCQhBXpUS1c/39t6kJsuPIeygpygyxEROW0K9Kj/+PV+BoZG1DsXkaSlQAeOdffz3d8d5MY157C4MDfockREzogCHbj/13X0DQ1z5zWhewyqiKSQlA/0tp4BvvO7A7x19TksnafeuYgkr5QP9Pt/vZ/jg8N8/BqNnYtIckvpQG/vHeDbvz3Amy9YwLKimUGXIyJyVlI60L/5mzp6Bob5hMbORSQEUjbQO3oHeXDLAd58wXxWzFfvXESSX8oG+je31NHVP8TH1TsXkZBIyUDvOD7It7bUccN5RZy7YFbQ5YiIxEVKBvqDWw7Q1TfEJ65V71xEwiPlAr2zb5Bv/mY/168q4rxzZgddjohI3KRcoH97ywE6+4a4S71zEQmZlAr07v4h7v9NHdeunMf5xeqdi0i4xBToZrbezKrNrNbM7h7n/feZWYuZ7Yh+fDD+pZ69Z/e10nF8kNuuLA+6FBGRuMuYqIGZpQMbgOuBBmC7mW109z1jmj7s7ncmoMa4qW7qAmB1SV7AlYiIxF8sPfS1QK2773f3AeAh4KbElpUY1Y1dlMyZTm72hL/HRESSTiyBXgzUj9puiO4b62Yz22VmPzaz0rhUF2fVjV2s0JotIhJS8ZoU/RlQ5u6rgceBb4/XyMxuN7NKM6tsaWmJ06FjMzA0wr6WbpbrNn8RCalYAv0wMLrHXRLdd4K7t7p7f3TzfuCS8T6Ru9/n7hXuXlFYWHgm9Z6xA609DI04KxXoIhJSsQT6dmCZmZWbWRZwC7BxdAMzWzBq80bgpfiVGB9VjZEJ0eUachGRkJpwdtDdh8zsTuBRIB14wN13m9k9QKW7bwQ+YWY3AkNAG/C+BNZ8Rmoau8hIM5bomaEiElIxXe7h7puATWP2fWHU688Bn4tvafFV1dhFeUEOWRkpdS+ViKSQlEm3mqYuTYiKSKilRKD39A9xqK2XlRo/F5EQS4lA39vcDaAeuoiEWkoEek30ChddsigiYZYSgV7V2MX0zHRK58wIuhQRkYRJiUCvaepieVEuaWkWdCkiIgmTEoFe1dilG4pEJPRCH+it3f0c6+5nhcbPRSTkQh/oNU2RK1wU6CISdqEP9OrGTgAtmysioRf+QG/qZs6MTApnZgddiohIQoU/0Bs7WV40EzNd4SIi4RbqQHd3apq6dUORiKSEUAf64fbjdPcP6ZZ/EUkJoQ70mqbILf+aEBWRVBDqQK9u1KJcIpI6Qh7onZwzexqzpmUGXYqISMKFO9CbunVDkYikjNAG+uDwCPuauzXcIiIpI7SBfrC1h4HhEU2IikjKCG2gV0UfaqEhFxFJFaEN9JrGLtLTjCWFuUGXIiIyKUIb6NVNXZTNncG0zPSgSxERmRThDfTGLg23iEhKCWWgHx8Y5mBbLyuKZgVdiojIpAlloO9t7sIdVszX+LmIpI5QBnp19AoXPUdURFJJaAM9OyONRXNzgi5FRGTShDPQm7pYVpRLepoeaiEiqSOmQDez9WZWbWa1Znb3KdrdbGZuZhXxK/H01TR1aUJURFLOhIFuZunABuBNwCrgVjNbNU67mcBdwNZ4F3k62nsHaOrs14SoiKScWHroa4Fad9/v7gPAQ8BN47T7O+BLQF8c6zttmhAVkVQVS6AXA/Wjthui+04ws4uBUnf/+ak+kZndbmaVZlbZ0tJy2sXGojr6lKKV8zXkIiKp5awnRc0sDfgK8OmJ2rr7fe5e4e4VhYWFZ3vocVU3djFrWgZFs7IT8vlFRKaqWAL9MFA6arskuu8VM4HzgafN7ADwemBjUBOjNU1drJw/CzNd4SIiqSWWQN8OLDOzcjPLAm4BNr7yprt3uHuBu5e5exnwLHCju1cmpOJTcHeqGrtYrglREUlBEwa6uw8BdwKPAi8Bj7j7bjO7x8xuTHSBp6Oxs4+uviFWaPxcRFJQRiyN3H0TsGnMvi+cpO0bz76sM3PioRa6wkVEUlCo7hStUaCLSAoLVaBXN3Yxf9Y0Zs/IDLoUEZFJF65Ab+piuR5qISIpKjSBPjzi7G3uZqUCXURSVGgC/UBrDwNDI7rlX0RSVmgC/ZUJUfXQRSRVhSbQqxq7MIOl83RTkYikptAEek1TF2Vzc5iWmR50KSIigQhNoFc3dun6cxFJaaEI9L7BYQ609uiSRRFJaaEI9NrmbkZcE6IiktpCEeh6SpGISEgCvaapi6yMNMrmzgi6FBGRwIQi0Ksau1hamEtGeihOR0TkjIQiAWuaulih8XMRSXFJH+gdvYMc7ehToItIykv6QK9p1hroIiIQgkB/5QoX9dBFJNWFItBnZmewYPa0oEsREQlU8gd69KEWZhZ0KSIigUrqQHf3yBouGm4REUnuQG/u6qfj+KAmREVESPJA14SoiMgfhCLQtYaLiEiyB3pTF4Uzs8nPyQq6FBGRwCV3oDd2aclcEZGopA304RFnb3OXhltERKKSNtAPtfXSNziiCVERkaiYAt3M1ptZtZnVmtnd47z/ETN7wcx2mNlvzGxV/Et9tRNXuKiHLiICxBDoZpYObADeBKwCbh0nsH/g7he4+4XAPwBfiXulY9Q0dWEGy4pyE30oEZGkEEsPfS1Q6+773X0AeAi4aXQDd+8ctZkDePxKHF91YxcL82cwIysj0YcSEUkKsaRhMVA/arsBuHRsIzP7GPApIAu4ZrxPZGa3A7cDLFy48HRrfZXqJk2IioiMFrdJUXff4O5LgL8G/uYkbe5z9wp3rygsLDzjY/UPDVN3rEeXLIqIjBJLoB8GSkdtl0T3ncxDwJ+cTVET2dfcw/CIq4cuIjJKLIG+HVhmZuVmlgXcAmwc3cDMlo3a/GNgb/xKfK3qpsiQvXroIiJ/MOEYursPmdmdwKNAOvCAu+82s3uASnffCNxpZtcBg8DLwHsTWXR1YzeZ6UZZQU4iDyMiklRiukTE3TcBm8bs+8Ko13fFua5TqmnqYklhLpnpSXtflIhI3CVlIuqhFiIir5V0gd7VN8jh9uOaEBURGSPpAr2mKXLLvyZERUReLekCvbqxG9BDLURExkq6QC/IzeL6VUWUzJkedCkiIlNK0i2Esu68+aw7b37QZYiITDlJ10MXEZHxKdBFREJCgS4iEhIKdBGRkFCgi4iEhAJdRCQkFOgiIiGhQBcRCQlzT/jznMc/sFkLcPAM//cC4Fgcy0kGOufUoHNODWdzzovcfdxneAYW6GfDzCrdvSLoOiaTzjk16JxTQ6LOWUMuIiIhoUAXEQmJZA30+4IuIAA659Sgc04NCTnnpBxDFxGR10rWHrqIiIyhQBcRCYkpHehmtt7Mqs2s1szuHuf9bDN7OPr+VjMrm/wq4yuGc/6Ume0xs11mttnMFgVRZzxNdM6j2t1sZm5mSX+JWyznbGZvj36td5vZDya7xniL4Xt7oZk9ZWbPR7+/3xxEnfFiZg+YWbOZvXiS983Mvh7999hlZhef9UHdfUp+AOnAPmAxkAXsBFaNaXMH8O/R17cADwdd9ySc89XAjOjrj6bCOUfbzQSeAZ4FKoKuexK+zsuA54E50e15Qdc9Ced8H/DR6OtVwIGg6z7Lc34DcDHw4knefzPwC8CA1wNbz/aYU7mHvhaodff97j4APATcNKbNTcC3o69/DFxrZjaJNcbbhOfs7k+5e29081mgZJJrjLdYvs4Afwd8CeibzOISJJZz/hCwwd1fBnD35kmuMd5iOWcHZkVfzwaOTGJ9cefuzwBtp2hyE/Adj3gWyDOzBWdzzKkc6MVA/ajthui+cdu4+xDQAcydlOoSI5ZzHu02Ir/hk9mE5xz9U7TU3X8+mYUlUCxf5+XAcjPbYmbPmtn6SasuMWI55y8C7zKzBmAT8PHJKS0wp/vzPqGke0i0RJjZu4AK4Kqga0kkM0sDvgK8L+BSJlsGkWGXNxL5K+wZM7vA3dsDrSqxbgUedPcvm9llwHfN7Hx3Hwm6sGQxlXvoh4HSUdsl0X3jtjGzDCJ/prVOSnWJEcs5Y2bXAZ8HbnT3/kmqLVEmOueZwPnA02Z2gMhY48YknxiN5evcAGx090F3rwNqiAR8sorlnG8DHgFw998B04gsYhVWMf28n46pHOjbgWVmVm5mWUQmPTeOabMReG/09duAJz0625CkJjxnM7sI+AaRME/2cVWY4JzdvcPdC9y9zN3LiMwb3OjulcGUGxexfG//lEjvHDMrIDIEs38yi4yzWM75EHAtgJmdSyTQWya1ysm1EXhP9GqX1wMd7n70rD5j0DPBE8wSv5lIz2Qf8PnovnuI/EBD5Av+I6AW2AYsDrrmSTjnJ4AmYEf0Y2PQNSf6nMe0fZokv8olxq+zERlq2gO8ANwSdM2TcM6rgC1EroDZAawLuuazPN8fAkeBQSJ/cd0GfAT4yKiv8Ybov8cL8fi+1q3/IiIhMZWHXERE5DQo0EVEQkKBLiISEgp0EZGQUKCLiISEAl1EJCQU6CIiIfH/Aey+fEa9ir66AAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from collections import Counter"
      ],
      "metadata": {
        "id": "fipnQIj4nsyf"
      },
      "execution_count": 117,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Counter(y_pred >= 1.0)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jYrbiXJLnvRU",
        "outputId": "5bcf4838-7fcb-42d3-e64c-b49dacfc0679"
      },
      "execution_count": 118,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Counter({False: 1409})"
            ]
          },
          "metadata": {},
          "execution_count": 118
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "1 - y_val.mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oT-DaBw0n0Bj",
        "outputId": "c93631fd-9376-477d-a210-8d7cc322de24"
      },
      "execution_count": 119,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7260468417317246"
            ]
          },
          "metadata": {},
          "execution_count": 119
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**4.3 Confusion table**\n",
        "\n",
        "*   Different types of errors and correct decisions\n",
        "*   Arranging them in a table"
      ],
      "metadata": {
        "id": "tvcvEEWwbtMl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "actual_positive = (y_val == 1)\n",
        "actual_negative = (y_val == 0)"
      ],
      "metadata": {
        "id": "JLpSp20wb5_P"
      },
      "execution_count": 120,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "t = 0.5\n",
        "predict_positive = (y_pred >= t)\n",
        "predict_negative = (y_pred < t)"
      ],
      "metadata": {
        "id": "Uhq2-EHfcRbu"
      },
      "execution_count": 121,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tp = (predict_positive & actual_positive).sum()\n",
        "tn = (predict_negative & actual_negative).sum()\n",
        "\n",
        "fp = (predict_positive & actual_negative).sum()\n",
        "fn = (predict_negative & actual_positive).sum()"
      ],
      "metadata": {
        "id": "JYN5UN1-cTX4"
      },
      "execution_count": 122,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tn"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xxxsRKVXd1zy",
        "outputId": "d2e88425-489a-4789-f8ec-f302f2ea31d5"
      },
      "execution_count": 123,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "922"
            ]
          },
          "metadata": {},
          "execution_count": 123
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "confusion_matrix = np.array([\n",
        "    [tn, fp],\n",
        "    [fn, tp]\n",
        "])\n",
        "confusion_matrix"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hUPpoa0od-Wa",
        "outputId": "584228b1-ff92-4a93-aa6e-0a865ef1f94a"
      },
      "execution_count": 124,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[922, 101],\n",
              "       [176, 210]])"
            ]
          },
          "metadata": {},
          "execution_count": 124
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "(confusion_matrix / confusion_matrix.sum()).round(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BonEUxVDeCBg",
        "outputId": "7edaa035-1662-41e9-934c-da8cfb289a66"
      },
      "execution_count": 125,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.65, 0.07],\n",
              "       [0.12, 0.15]])"
            ]
          },
          "metadata": {},
          "execution_count": 125
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**4.4 Precision and Recall**"
      ],
      "metadata": {
        "id": "aFrIlb5HeKBx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "p = tp / (tp + fp)\n",
        "p"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZntL2uyseJqE",
        "outputId": "cc929aa9-8995-4347-8f7e-b458d12190b7"
      },
      "execution_count": 126,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.6752411575562701"
            ]
          },
          "metadata": {},
          "execution_count": 126
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "r = tp / (tp + fn)\n",
        "r"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UXLYI1jPfqXp",
        "outputId": "59401a86-b9ca-4350-910d-51d167fb1c3e"
      },
      "execution_count": 127,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5440414507772021"
            ]
          },
          "metadata": {},
          "execution_count": 127
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**4.5 ROC Curves**\n",
        "\n",
        "TPR and FRP\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "hLZtzSkLpcdX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tpr = tp / (tp + fn)\n",
        "tpr"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mReaPOk4phQM",
        "outputId": "81e1a983-6d55-401d-bd94-b350b58d7621"
      },
      "execution_count": 128,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5440414507772021"
            ]
          },
          "metadata": {},
          "execution_count": 128
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "fpr = fp / (fp + tn)\n",
        "fpr"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zVlsldlypor2",
        "outputId": "c13214d3-7edf-4c74-a8c0-4643aa121d45"
      },
      "execution_count": 129,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.09872922776148582"
            ]
          },
          "metadata": {},
          "execution_count": 129
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "scores = []\n",
        "\n",
        "thresholds = np.linspace(0, 1, 101)\n",
        "\n",
        "for t in thresholds:\n",
        "    actual_positive = (y_val == 1)\n",
        "    actual_negative = (y_val == 0)\n",
        "    \n",
        "    predict_positive = (y_pred >= t)\n",
        "    predict_negative = (y_pred < t)\n",
        "\n",
        "    tp = (predict_positive & actual_positive).sum()\n",
        "    tn = (predict_negative & actual_negative).sum()\n",
        "\n",
        "    fp = (predict_positive & actual_negative).sum()\n",
        "    fn = (predict_negative & actual_positive).sum()\n",
        "    \n",
        "    scores.append((t, tp, fp, fn, tn))"
      ],
      "metadata": {
        "id": "el1AUq5dpq1Y"
      },
      "execution_count": 130,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "columns = ['threshold', 'tp', 'fp', 'fn', 'tn']\n",
        "df_scores = pd.DataFrame(scores, columns=columns)\n",
        "\n",
        "df_scores['tpr'] = df_scores.tp / (df_scores.tp + df_scores.fn)\n",
        "df_scores['fpr'] = df_scores.fp / (df_scores.fp + df_scores.tn)"
      ],
      "metadata": {
        "id": "4nDRESyUptBv"
      },
      "execution_count": 131,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.plot(df_scores.threshold, df_scores['tpr'], label='TPR')\n",
        "plt.plot(df_scores.threshold, df_scores['fpr'], label='FPR')\n",
        "plt.legend()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "wAIi8GUJpv7f",
        "outputId": "8de70ff2-ac38-4ba6-e292-cf3a98946f4b"
      },
      "execution_count": 132,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f0c10d7b350>"
            ]
          },
          "metadata": {},
          "execution_count": 132
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deVxV1f7/8dc6h1kmQRQVGVRQEXHCeUgzcypNG8ypLNOyzLoNN7/Zvbfs1m241e+WllmW2aBpg5k5VJY5DziPIAoqKoioqCgyrd8fG4tM5SgHNuecz/Px4AHssznns0XebNZe+7OU1hohhBCOz2J2AUIIIexDAl0IIZyEBLoQQjgJCXQhhHASEuhCCOEk3Mx64Ro1aujIyEizXl4IIRzSxo0bj2utQy73mGmBHhkZSWJiolkvL4QQDkkpdeBKj8mQixBCOAkJdCGEcBIS6EII4SRMG0MXQgh7KCgoID09nby8PLNLsSsvLy/CwsJwd3e3+Wsk0IUQDi09PR0/Pz8iIyNRSpldjl1orcnOziY9PZ2oqCibv67MIRel1EdKqWNKqR1XeFwppd5WSqUopbYppVpdQ91CCFEueXl5BAcHO02YAyilCA4Ovua/OmwZQ58B9L7K432A6JK3McB711SBEEKUkzOF+UXXc0xlBrrWejlw4iq7DABmasNaIFApVfuaK7HRnvU/kvjhYyzefpT9WWcpKpb2v0IIAfYZQ68LHCr1eXrJtqOX7qiUGoNxFk94ePh1vVj23vV0Sp9B65R2ZBOAh5uFBiG+xNTyJaaWH9E1jff1gnywWpzvt7YQomrJzs6mR48eAGRkZGC1WgkJMW7k3Lp1K82bN6ewsJAmTZrwySef4OPjg9VqpVmzZhQWFhIVFcWnn35KYGBguWup1IuiWutpwDSAhISE6zq17tSmLeyF2XeEsIXG7D12luTMMySmneS7LUd+36+ah5Ubm9Tilvja3BATgpe71T4HIYQQpQQHB7NlyxYAnn/+eXx9fXnqqacA8PX1/f2xYcOGMXXqVJ544gm8vb1/337vvfcyZcoUJk6cWO5a7BHoh4F6pT4PK9lWMYIbABBtPUZ0y55/euhMXgEpx86yN/Msmw+dZMnOTL7fegRfTzdualKTfvF16BJdQ8JdCFHpunTpwrZt2/6yvUOHDpfdfj3sEejzgXFKqdlAOyBHa/2X4Ra7CQwHZYUT+//ykJ+XOy3Dq9MyvDp3tanHiwOKWbM/mwVbj7JkVwbzthzBz9ONnrG16Bdfm87RNfB0k3AXwlm88P1Odh05bdfnjK3jz79ubVqu5ygsLGTRokX07v3n+SVFRUUsXbqUUaNGlev5Lyoz0JVSs4BuQA2lVDrwL8AdQGs9FVgI9AVSgHPAfXap7Eqs7kaon9hX5q5uVgtdokPoEh3Cv4viWL0vmwVbj7BkZwbfbD6Mv5cbLw1sxq3N61RoyUII13T+/HlatGgBGGfoF4P74vbDhw/TpEkTevbsebWnsVmZga61HlLG4xp4xC7V2Cq4AWSXHeiluVst3BATwg0xIbw0sBmrUo7zzi97eXTWZvZn5TK+R0OnnPokhCsp75m0vZUeK7/c9nPnztGrVy+mTJnC+PHjy/16jtnLJagBnEgFfX1TFj3cLHRvXJNZY9ozqFVd3vo5mfGzt7A9PYfz+UV2LlYIIS7Px8eHt99+mzfeeIPCwsJyP59j3vofVB/yz0BuFvjWvO6n8XSz8sadzWlY05fXFifx/dYjKAXhQT7cEBNCv2a1SYgMkumPQogK07JlS+Lj45k1axYjRowo13M5ZqCXzHQhe1+5Ah2Mu7Ee7taQW+PrsONwDsmZZ9lxJIc5iYeYueYANf08uadDBCM7ReHr6Zj/XEKIyvH888//6fOzZ89edr9Lt3///fd2eX3HTKig+sb7E/shooNdnrJekA/1gnzo08z4PPdCIb/sOcbXm9L574/JfLgyldFd6tMnLpSI4Gpy1i6EqHIcM9B/n7p4bRdGr0U1TzdubV6HW5vXYeuhU/y/n5N5fUkSry9JwsPNQv0a1fD3NtpaKmBQq7oMbnN9d78KIYQ9OGagW92hesQ1z3S5Xs3rBfLxfW1JzjzD1kOnSM48Q8qxs5wvMC6gnswt4Jmvt7P76Bme69cEN6tjXmsWQjg2xwx0KJnp8tebiypSTC0/Ymr5/WV7UbHmPwt38+HKVFKP5/LO0Jb4e9nelF4IIezBcU8lg+obgX6dUxftyWpRPHdLLP8ZZMxvv+O91Rw6cc7ssoQQLsZxAz24AeSfhbPHzK7kd0PahjPz/rZk5OQxYMoqEtOu1nVYCCHsy3EDPahk6mIlD7uUpWPDGsx7pBMB3u4M/WAds9YfRFeBvyKEEBXHarXSokWL39/S0tJYtmwZAQEBtGjRgiZNmvDCCy8A/Gl748aNf+/MaA8OHOgl6+xV4EyX61U/xJdvH+5I26gg/u+b7dw3YwMZOc61gK0Q4g8Xb+W/+BYZGQkY/Vu2bNlCYmIin332GZs2bfrT9s2bN7NgwQJWrVpllzocN9ADI8DiVmkzXa5VoI8HM+9vy/O3xrJ2fzY93/qNrzemy9m6EC6oWrVqtG7dmpSUlD9t9/b2/r1Jlz047iwXq5sR6lVsyKU0i0UxslMU3RrV5Km5W3ly7lYW7cjg5UFx1PTzMrs8IZzPogmQsd2+zxnaDPq8ctVdSndVjIqK4ttvv/3T49nZ2axdu5Z//OMfZGVl/b795MmT7N27l65du9qlVMcNdCiZ6VI1z9BLi6xRjS8f7MDHq1J5fUkSN7+1nAc6RxFbx5/omn7UDfTGIneeCuGwrtRVccWKFbRs2RKLxcKECRNo2rQpy5YtY8WKFTRv3py9e/fy+OOPExoaapc6HDvQgxvAgdXG1MUq3vrWalE80KU+3RrVZMLX2/jvj8m/Pxbi50nfuFBuaV6H1uHVJdyFuF5lnElXti5durBgwYIrbk9NTaV9+/bcddddv5/hl4eDB3pDKMiFM0fB3zEWqWhY05evxnYk51wBe4+dISnzDCuSjzN7wyE+WXOAZnUDmHl/W6pX8zC7VCFEBYuKimLChAm8+uqrzJo1q9zP57gXRQFqlTSzz9hhbh3XIcDHnYTIIIa1i2DqiNZs/EdPXr29GUmZZxj24TpOncs3u0QhRCV46KGHWL58OWlpaeV+LgcP9DjjfcZWc+uwA19PNwa3CeeDexJIyTrLsA/XkXOuwOyyhBA2uFyb3G7dul12uOXS7d7e3hw+fPj3qY7l4diB7uUP1aPsf1XbRDfEhPD+iNbszTzLkA/Wyvx1IYTNHDvQAWrHw9FtZldhV90b1WTaPa05kJ3LgCkr2Z6eY3ZJQggH4PiBHtoMTqZCnnOFXrdGNflqbEfcLBbufH813205LDclCXEFzvizcT3H5ASB3tx4n7nT3DoqQJPa/sx7pBNNavvz2OwtDHx3Nb8lZznlf14hrpeXlxfZ2dlO9XOhtSY7Oxsvr2u7AdGxpy2CcYYOxrBLREdza6kAIX6ezHmwA19tTGfyLync+9F6OjUM5oN7EvDxcPxvnxDlFRYWRnp6+p/uwHQGXl5ehIWFXdPXOH4i+IVCtRCnujB6KXerhSFtw7m9VRifrzvApAW7eOLLrbw7rJXchCRcnru7O1FRUWaXUSU4/pCLUhAa7xRTF8vi4Wbhvk5RTOzbhMU7M3htSZLZJQkhqhDHD3Qwhl2O7YFC17gZZ1TnKIa1C2fqb/uYvf6g2eUIIaoI5wj02vFQXABZe8yupFIopXi+f1O6RNdgwjfbue/j9WxLP2V2WUIIkzlHoIfGG++deBz9Uu5WC9NGJPB0r0ZsPnSK/pNXMXpmIkdOnTe7NCGESZwj0IMagHs1yHCuG4zK4u1h5ZHuDVnx9+48dXMMq1KO0+ut5cxJPORUU7iEELZxjkC3WCA0zunuGLWVn5c7426MZvFjXYmt48/fv9rGA58kcj6/yOzShBCVyDkCHYwLoxnbobjY7EpMEx7sw6zR7fnHLbH8knSMJ+duobhYztSFcBU2BbpSqrdSKkkplaKUmnCZx8OVUr8qpTYrpbYppfrav9Qy1G4O+Weq9JJ0lcFiUYzqHMWzfZqwcHsGb/wkUxuFcBVlBrpSygpMAfoAscAQpVTsJbs9B8zRWrcE7gbetXehZarX3nh/wD6rZzu6B7pEMaRtPab8uo+5iYfMLkcIUQlsOUNvC6RorfdrrfOB2cCAS/bRgH/JxwHAEfuVaKMa0VCtpgR6CaUUkwbE0alhMM9+u52vNqabXZIQooLZEuh1gdKneOkl20p7HhiulEoHFgKPXu6JlFJjlFKJSqlEu/ddUMro5ZK2ylhjVOButfDe8Na0iQziqblbeXXxHhlTF8KJ2eui6BBghtY6DOgLfKqU+stza62naa0TtNYJISEhdnrpUiI6wel0OHXA/s/toPy93Pnk/rYMaRvOe8v2MXpmIt9tOczuo6e5UCizYIRwJrY05zoM1Cv1eVjJttJGAb0BtNZrlFJeQA3gmD2KtFlkJ+N92iqoHlmpL12VuVstvDwwjoY1fXl10R6W7jG+LdU8rMy4vy1tIoNMrlAIYQ+2nKFvAKKVUlFKKQ+Mi57zL9nnINADQCnVBPACKr+XZUgT8K4u4+iXoZQx+2X7Czez+PEuvD2kJSF+njz6xWZO5LpGDxwhnF2Zga61LgTGAUuA3RizWXYqpSYppfqX7PYkMFoptRWYBYzUZtyqaLEYwy5pKyv9pR2Fp5uVxqH+9G9eh8lDW3EiN5+n5m6VsXUhnIBNY+ha64Va6xitdQOt9Usl2/6ptZ5f8vEurXUnrXVzrXULrfWPFVn0VUV0MsbQc2RWR1ni6gbw3C1N+GXPMaavTDW7HCFEOTnPnaIXlR5HF2Ua0T6C3k1DeXXxHr5Yd1B6wAjhwJwv0GvFgWcAHJBhF1sopXj1jnja1Q/i2W+3c89H66VjoxAOyvkC3WKFiA5yhn4NArzd+fT+drw4oCmJaSfp9dZyNqSdMLssIcQ1cr5AB2Mc/cQ+OH3U7EochsWiGNEhkiWPdyXEz5MxMxM5kJ1rdllCiGvgnIHeoLvxfu8Sc+twQOHBPnw0sg0auH/GBnLOFZhdkhDCRs4Z6LXiICAc9iw0uxKHFFmjGu8Pb83BE+cY+/lG8gtdtyWxEI7EOQNdKWjcF/Yvgwtnza7GIbWrH8wrg+JZvS+bge+uYk/GabNLEkKUwTkDHaBxPyi6APt+MbsSh3V76zDeH9GazNN53PrOSqb8mkJegfR/EaKqct5AD+8IXoGQJMMu5dGraShLHu/KzU1DeX1JEq1f/InHZ2/mp12ZEu5CVDG2NOdyTFY3iOkFyYuhqND4XFyXYF9PpgxtxbB2x5m/5QiLd2Ywb8sR/Dzd6Blbi37xtekcXQNPN6vZpQrh0pRZdwYmJCToxMTEin2RnfNg7r0w8geI7Fyxr+VCCoqKWZVynIXbj7J4Rwan8wrx83Lj5thQbomvTaeGNfBwc94//oQwk1Jqo9Y64bKPOXWgXzgDr9WHNqOh98sV+1ouKr/QCPcfth9lyc4MzuQVEuDtTq+mtegXX4dODYJxs0q4C2EvrhvoAJ/dAdl7YfwWY/aLqDAXCotYufc4P2w7yo+7Mjl7oZA2kdV5f0QCQdU8zC5PCKdwtUB3/lOnxn3hZBoc2Wx2JU7P081Kjya1eHNwCxKfu4lXb2/GtvQcBkxZyd7MM2aXJ4TTc/5AbzrImO2y7BWzK3EpXu5WBrcJZ/aY9pzPL2bQu6v5dO0BWUxDiArk/IHuHQidHzfaABxca3Y1LqdleHXmj+tE/ZBq/GPeDtq89DMjpq9j5d7jZpcmhNNx/kAHaDsGfGvBzy+A9PuudHUCvZn3SCd+GN+ZB7vWZ39WLvd8tI6PVqZK/3Uh7Mg1At2jGnR9Gg6uhn1Lza7GJSmlaFongL/3bsxPT3Tlpia1mLRgF8/N20FBkfSKEcIeXCPQAVrdC4HhsHQSFEuAmMnHw42pw1sztlsDPl93kPGzNsuZuhB24DqB7uYB3Z6Fo1th6yyzq3F5Fovimd6NmdCnMYt2ZDBjdZrZJQnh8Fwn0AHiB0O99rDk/+BMhtnVCODBrvW5qUlNXl64m23pp8wuRwiH5lqBbrHAgClQeAEWPCEXSKsApRT/vbM5Ib6ejPtiM6fzZEENIa6XawU6QI2G0P1ZSPoBdn5jdjUCCPTx4O0hLTl86jz3fbyBTQdPml2SEA7J9QIdoP0jUKcVLHwazmaZXY0AEiKDeP2OeFKP5zLo3dWM/Hg9Ww/JEIwQ18I1A93qZgy95OfC1/cb7XWF6Qa1CmPF37vzTO/GbD10igFTVjFqxga2p+eYXZoQDsE1Ax2gViz0exNSl8PSF8yuRpSo5unG2G4NWPHMjTzdqxGJB05y6+SVPPDJBnYclmAX4mqcv9tiWRY8AYnT4c4Z0HSg2dWIS5zJK2DGqjQ+WLGf03mF3Bxbi0e6NyQ+LAAl3TOFC3Lt9rllKcyHGX0hcxc8+BvUiDa7InEZp/MK+GhlKtNXpnImr5DIYB/6xdfm9lZh1A/xNbs8ISqNBHpZTh+BKe0hvB0Mm2t2NeIqcs4XsGj7URZsO8rqfcfxcLMw98GONAsLMLs0ISqFa/dDt4V/Hej6FOz9EVKk10tVFuDtzt1tw/nsgXasfOZGgqt5MuqTDRzNOW92aUKYzqZAV0r1VkolKaVSlFITrrDPXUqpXUqpnUqpL+xbZiVo9yAERsCPz0GxrGbvCOoEejN9ZALn8osYNSOR3AsyW0m4tjIDXSllBaYAfYBYYIhSKvaSfaKB/wM6aa2bAo9XQK0Vy80Tek6CY7tg86dmVyNs1DjUn3eGtmRPxmnGfbGJvAL5ZSxcly1n6G2BFK31fq11PjAbGHDJPqOBKVrrkwBa62P2LbOSxA4wer388m9jgWnhELo3qsmLt8WxLDmLu6et5diZPLNLEsIUtgR6XeBQqc/TS7aVFgPEKKVWKaXWKqV6X+6JlFJjlFKJSqnErKwqeIemUtD7ZcjNgrXvmV2NuAbD2kXw3rDWJGWc4bbJq9h15LTZJQlR6ex1UdQNiAa6AUOAD5RSgZfupLWeprVO0FonhISE2Oml7axua4jpDWvfhQtnza5GXIPecaHMfagDxRrumLqan3dlml2SEJXKlkA/DNQr9XlYybbS0oH5WusCrXUqkIwR8I6py5Nw/iRs+sTsSsQ1iqsbwHfjOtEgxJfRnyYybfk+WTxDuAxbAn0DEK2UilJKeQB3A/Mv2Wcextk5SqkaGEMw++1YZ+Wq1xYiu8Dqd4xWu8Kh1PL3Ys6DHegTF8rLC/cw4evtFMoyd8IFlBnoWutCYBywBNgNzNFa71RKTVJK9S/ZbQmQrZTaBfwKPK21zq6ooitFlyfhzFFZ3chBeXtYmTykFeO6N+TLxEM8NXcrRcVypi6cm9wpeiVawwc3GkMv4xKNDo3CIU35NYXXlyQxqGVdXr+zOVaL9IARjutqd4pKSl2JUsZZ+pfDYNuX0HKY2RWJ6/RI94YUF2ve+CmZC0XF3N6qLjG1/Kgb6C0NvoRTkUC/mkZ9Iawt/DgRonuCb02zKxLX6dEexjX6N39O5odtRwEIqubBxL5NGNSqrgS7cAoy5FKWrGSY2hka9Ya7ZppdjSinnHMF7D12hqTMM3yz6TAbD5zkpia1eHlQHDX9vMwuT4gySXOu8giJgW4TYNd3sHOe2dWIcgrwcSchMohh7SKY82AHnuvXhBV7s+j55nLeW7ZP+sEIhyaBbouO46F2C1j4FJw7YXY1wk6sFsUDXeqz8LEutKgXyKuL99DltV95/7d9nMuXYBeORwLdFhfXID1/En5+3uxqhJ01CPHlk/vb8vXYjjSt489/Fu2h62u/8uGK/ZzPl2ZfwnFIoNsqNA7ajjE6MWbsMLsaUQFaR1Tn01Ht+HpsBxqH+vPvH3bT9fVf+WhlqnRxFA5BLopei3Mn4O2WUKcFjJhnTG0UTmt96gne+imZNfuzqennycPdGjCiQ6TMYxemkoui9uITZFwg3b8M9v5kdjWigrWNCmLWmPbMHtOeqBrVeP77Xby2ZI/ZZQlxRRLo1yphFAQ1MOamFxWYXY2oBO3rB/Plgx0Y2i6c93/bz69JjtnuXzg/CfRr5eYBN78Ix5OlZ7qL+ectsTQO9ePJOVvJyJFFNETVI4F+PRr1hca3GDNe9v1idjWikni5W5k8tBV5BUWMn72Z/ELp4CiqFgn066EUDJwKIY1g7kg4nmJ2RaKSNKzpy79vi2N96gnavPQzf/9qK78lZ0nPdVElSKBfL08/GDIbLG4wa7AxR124hEGtwph5f1t6NK7Jwu0Z3PvRev63dK/ZZQkhgV4u1SNg8Odw8gAsec7sakQl6hoTwpuDW5D43E3c2rwOk39JYeeRHLPLEi5OAr28IjpA29HGQhgn08yuRlQyL3crLw5oSqCPB0/P3UaBrIwkTCSBbg8dxxtDLyveNLsSYYJAHw/+fVscu46eZuqyfWaXI1yYBLo9+NeG1vfCli/g1EGzqxEm6B0Xyi3xtXn7l70sSzomF0mFKSTQ7aXT48bsl5X/z+xKhEle6N+U0AAvRn68gcHT1rJmn2MvqyscjwS6vQTUhZbDjeZdOYfNrkaYINjXk5/+dgPP3xpL2vFchnywln99t0PO1kWlkUC3p85/A10Mv71idiXCJF7uVkZ2imL537tzX6dIPllzgBe+3yWhLiqFrClqT4Hh0H4srH4H4u6A+jeYXZEwiZe7lX/eEotVKT5cmYpSRusAWbtUVCQ5Q7e37hON5l3zH4X8XLOrESZSSjGxXxPu6xTJx6vSmDhvh0xrFBVKAt3e3L1hwGQ4dQCWvmh2NcJkSin+eUssY7s14It1Bxn58XpyzkmXTlExJNArQkRHY3WjdVPh4FqzqxEmU0rxTO/GvH5HPOtTTzDwvVUcyJa/3oT9SaBXlB7/gsB6MPc+mfUiALgzoR6fjWrHidx8HvgkURaiFnYngV5RPH2N5l35Z+HzOyFP+nwIaFc/mMlDWpGSdZbn5+80uxzhZCTQK1KtpnDXTDieBHPugcJ8sysSVUDn6BqM696QOYnpfLs53exyhBORQK9oDbpD/3eMdUi/ewSKZfV4AY/1iKZtZBATv93BvqyzZpcjnIQEemVoMdQYU98+x5jOWCxT11ydm9XC/4a0wMPNwgvf7zK7HOEkJNArS5cnoNuzsOVz+H68hLqgdoA3o7vUZ3lyFruPnja7HOEEbAp0pVRvpVSSUipFKTXhKvvdrpTSSqkE+5XoRLo9A12fNvq9LHwK5HZwlze8XQQ+HlY+WL7f7FKEEygz0JVSVmAK0AeIBYYopWIvs58f8Biwzt5FOpXuE6Hjo5A4HVa+ZXY1wmQBPu4MblOP+VuPcOTUebPLEQ7OljP0tkCK1nq/1jofmA0MuMx+LwKvAnl2rM/5KAU3TYK422HpC7D9K7MrEiYb1TkKDXy8KtXsUoSDsyXQ6wKHSn2eXrLtd0qpVkA9rfUPV3sipdQYpVSiUioxKyvrmot1GhYL3PYeRHSCeWMhbaXZFQkThVX3oV+z2sxaf4jTedIWQFy/cl8UVUpZgDeBJ8vaV2s9TWudoLVOCAkJKe9LOzY3Txj8GVSPhFlDIWO72RUJE43pWp+zFwplLF2Uiy2BfhioV+rzsJJtF/kBccAypVQa0B6YLxdGbeATBMO/Me4q/XQQZMt6lK4qrm4A/eJr884vKby8cDdFxXLBXFw7WwJ9AxCtlIpSSnkAdwPzLz6otc7RWtfQWkdqrSOBtUB/rXVihVTsbALrwYh5oIvg09vg9BGzKxIm+d/gFtzTIYJpy/fz4Kcbyb0gvV7EtSkz0LXWhcA4YAmwG5ijtd6plJqklOpf0QW6hJAYGPYVnDsBMwdIqLsoN6uFSQPimDSgKb8mHePWySvZdPCk2WUJB6LMWhorISFBJybKSfyfHFhtNPKqVgPumQ/VI8yuSJhkdcpxnpq7lYzTeYzp2oDHb4rGy91qdlmiClBKbdRaX3ZIW+4UrUoiOhpBfv4UfNQbju81uyJhko4Na7D4b125K6EeU3/bxz0fradQVjsSZZBAr2rCWsPIH6C4AKbfLFMaXZi/lzuv3B7PayULY7z1c7LZJYkqTgK9KgqNg/uXgE+wMaae+LHZFQkT3ZVQj8EJ9Xh32T6WJ7vw/RuiTBLoVVVwA3jgZ6jfDRY8DouekYZeLuz5/k2JrunL377cwrHTcjO2uDwJ9KrMOxCGzoH2Dxvrk84bC0Uylc0VeXtYmTK0Fbn5hTw2e4vMUxeXJYFe1Vms0OtluPE52DYbvrpPVj5yUdG1/Jg0II41+7OZ/EuK2eWIKkgC3REoZbTd7fUf2D0fvhwORdLzwxXd2TqMgS3r8r+lyazZl212OaKKkUB3JB0ehn5vwt4lsORZs6sRJlBK8eJtcUQGV+Ox2ZvJPnvB7JJEFSKB7mjajIIO42D9NNgw3exqhAl8Pd2YPLQVp84X8OiszeQVyDq1wiCB7oh6ToLom2HR3yF1udnVCBPE1vHnPwObsXpfNmM/28iFQgl1IYHumCxWuH06BDeEWUNg1f+gUP70djW3tw7jP4Oa8WtSFg9/tklCXUigOywvfxj+tbFIxk//hCntYPcCWafUxQxpG85LA+NYuucYYz/bJB0aXZwEuiMLCINhc4xgd/OEL4fBJ7fKYhkuZli7CF4aGMeypGPcMXWNrE3qwiTQnUHDm+ChVdD3v5C5A97vCgv+BvnnzK5MVJJh7SKYPrINh06cY8CUVWw5dMrskoQJJNCdhdUN2o6G8Zuh3UNG/5ePe0NOutmViUrSvVFNvnm4I55uFkZ8uE7O1F2QBLqz8a4Ovf9jtAzI3g/TusOhDWZXJSpJTC0/Pn+gHYXFmme/3Y5Z6x0Ic0igO6uYm43mXh7V4OM+sOwVmQnjIiKCq/F0r0YsS8ri602Hy/4C4TQk0J1ZzcYw+heIHQDL/gNTu8CBNWZXJSrByI6RJERUZ9L3O8mU7owuQwLd2fkEwR3TjTVLC84bZ+tLJkKB/JA7M4tF8dod8VwoLDa2toEAABFmSURBVOa5eTvMLkdUEgl0VxHdEx5eAwn3w5rJ8MGNkCE/6M6sfogvf+sZw0+7MlmVctzsckQlkEB3JZ6+cMubMHQu5GbBB91h21yzqxIVaGTHSOoGevPKoj0USw91pyeB7opibjbO1sPawjcPGK0DZDaEU/Jyt/LkzTFsP5zDD9uPml2OqGAS6K6qWg0Y8Q00HWi0Dlg8AYqlF4gzGtCiLo1D/fjvj0nkF8oyhs5MAt2VuXnC7R8Z7XjXTYVPB8JZWYTY2Vgtimf6NOZA9jlmbzhodjmiAkmguzqLBXq9BAOmwKF1MLUzHFhtdlXCzrrFhNC+fhBv/ZRM6vFcs8sRFUQCXRhaDv/jRqQZt8D3j8GpQ2ZXJexEKcW/b2uGRSmGTFtLmoS6U5JAF38IbQZjlhlTGzd/Du+0gh+ehOOyILEzaFjTl89HtyO/qJghH6zlQLaEurORQBd/5uUP/f5rNPlqMQw2zoDJrWF6L9g0E84eM7tCUQ6NQ/35/IF25BUUMfSDddI/3ckos5r3JCQk6MTERFNeW1yDM5mwbTZs/gyOJxvbQhpDZBdjGbz6NxgXV4VD2ZB2gjunrmFCn8Y8dEMDs8sR10AptVFrnXDZxyTQhU20hqNbYP8ySF0BB9dCQS54+Bnz2tuNhXptzK5SXIN7PlrPzsM5rHimOz4ebmaXI2x0tUC3achFKdVbKZWklEpRSk24zONPKKV2KaW2KaWWKqUiylu0qGKUgjotofPfjPnrz6Qad5w2vQ32/QrTb4KvR0v/dQfyWI9osnPz+XytTGV0FmUGulLKCkwB+gCxwBClVOwlu20GErTW8cBXwGv2LlRUMW6expn5gMnw+Hbo+jTsng/vJMB34yD5R2nXW8W1jqhOl+gavL98H+fz5aYyZ2DLGXpbIEVrvV9rnQ/MBgaU3kFr/avW+uJ6Z2uBMPuWKao0T1+48TkYtwHiBsHOefDFnfBaA/j2IUhbKa0FqqjxPaI5fjafz9cdMLsUYQe2BHpdoPSE5PSSbVcyClh0uQeUUmOUUolKqcSsLLkj0ekEhsNt78Lf9xntepveBnt+gBn94O0W8OM/YO9PcOGM2ZWKEm0ig+jYIJj3l+8nr0DO0h2dXactKqWGAwnA65d7XGs9TWudoLVOCAkJsedLi6rEzdNo1ztgMjyZBAOnQWAErH0PPr8DXomAb8fChbNmVyqAR2+MJuvMBb7eJNc/HJ0tl7YPA/VKfR5Wsu1PlFI3AROBG7TWMngqDB4+0Hyw8ZZ/zmgvkLwY1k+DwxvhrpnGykrCNO3rBxEfFsCHK1K5u004VosyuyRxnWw5Q98ARCulopRSHsDdwPzSOyilWgLvA/211nLnibg8Dx9o0B36vAoj5sH5E0ZP9rVTIV/uWjSLUooxXeuTejyXn3Zlml2OKIcyA11rXQiMA5YAu4E5WuudSqlJSqn+Jbu9DvgCc5VSW5RS86/wdEIY6t8AD66AsARY/Ay81RR+fsG4kUlUut5NQ6kX5M205fvMLkWUg9xYJMyltXGT0topsHsBeFeHQR9A9E1mV+ZyPlmdxr/m7+SrhzqQEBlkdjniCsp9Y5EQFUYpiOgAgz+DR9aBfx3jwukv/5YFNyrZnQlhVPdxZ+pv+80uRVwnCXRRdYQ0Mlr4thwGy1+HD3vAzm+hSBpIVQYfDzdGdIjk592ZLN6RYXY54jrIkIuomrbNgV9fhpOpEBAOsf3BJwi8AqFmEwjvYJzdC7vKvVDIiOnr2Jaew7vDWnFz01CzSxKXkOZcwjEVF0HSImP+evp6KMr/47HwjtD9WYjqYl59Tup0XgH3TF/PziM5vDesNTfF1jK7JFGKBLpwDgV5kHcKdn8PK96AM0ehdgtjpkxoPER2hmBpBWsPp/MKGPHhOnYdPc13j3Qmto6/2SWJEhLowvkUnDcW39g1HzK2Q/4ZsLhB3/9Cwn1mV+cUTubm0+PN32gQUo05D3ZAyRBXlSCzXITzcfeG9mPh/kUw4SA8ugnqd4cFj8PCp+VCqh1Ur+bBM70bsSHtJPO2/OXmcFEFSaALx2exGEMtQ7+EDuOMtgKf3gbH95pdmcO7s3U9mocF8PLCPZzJKzC7HFEGCXThPCxW6PUS3PYeHNkC77aHxf8H50+aXZnDslgUkwbEcfzsBf73s/yCrOok0IXzaTEUxm+ClsNh3VR4qxl8Ncro0y4dHq9Z83qBDE6ox8er01i3P9vscsRVyEVR4dwythuhnrQIzmWDm7cxp73lcIjobAzXiDKdzM3njqmrOZqTxyf3t6WNtAYwjcxyEaKoEA6the1fwY6v4cJpCKhndH+M7GrMZ/eTm2iu5tjpPO6etpbM03nMHNWW1hES6maQQBeitILzRiOwnd9A2iq4kAPKAq3uhe4TwVcWX7mSzJJQzzpzgbkPdaBJbZmfXtkk0IW4kuIiyNgGW2ZB4nRw94HOf4OITsaSer61ZFjmEhk5efSfvBJfLze+H9eZap62rJMj7EUCXQhbZCXDjxNh749/bHP3gcb9jDH3yK4S7iXW7Mtm2Idrua1lXd68q4XZ5biUqwW6/GoV4qKQGBg2F7L3wYn9cDINMncYHR+3zwX/MKgdb5y5V4+EqK5QM9Ylm4R1aBDM+B7R/L+f99KhfjB3JtQr+4tEhZNAF+JSwQ3+3BOm96uwZwHsmgfZ+yF1OeSXTH+sHgVNboVW90CNaHPqNcmjN0azdn82//xuJ+5WC7c2ryPrkZpMhlyEuFZaG43BkhcbF1dTl0NxAUT3gg4Pu9TQzLHTedz78QZ2Hz1Ng5BqjO8Rza3xdbBIsFcYGUMXoiKdzTIuqK7/AM4dB/dqEBpndIBsMQTqtja7wgpVXKxZtCOD/y1NJjnzLH3iQnnjrub4eMgAQEWQQBeiMhTkGa19DyfC0W1wdCsU5ELzIdDjX+Bf2+wKK1RxsWb6ylReXrSbpnX8+fCeNoQGeJldltORQBfCDHmnjb7ta981Wvu2uteYLRMaZ3ZlFWrp7kzGz9qMr5cbM+5rK3PV7UwCXQgznUg1Fr3e9Z0x1l67hXFnamCE8RaWYCyv50R2Hz3NfR9vIL+omC9Gt6NxqIS6vUigC1EV5GYb0x+3zYbMXVB0wdhu9YBGfaDFcGhwI1idY+w57Xgud09bS0FRMbPGtCemlp/ZJTkFCXQhqpriYsg9Zsx33/09bPvSaB7m6Q8RHSGyC4S3N+a5e/iYXe11Sz2ey+D311CsNR+NbEN8WKDZJTk8CXQhqrrCfNi7BFJ+htQVcGKfsV1ZoEaMsV5qi6FQp5XD3ci0L+sswz9cx7EzF3ike0PGdW+Ih5trTOusCBLoQjia00fg8MY/Zsuk/gaFecYZe5P+ULu5cdeqf12HCPiccwW8sGAn32w6TJPa/jzRM4auMTXwdLOaXZrDkUAXwtGdP2V0h9z8uRH0lPzc+gRDaDNjzntovBHywQ2N1ZuqoJ92ZTLx2+0cO3MBP083esbWYtyNDakf4mt2aQ5DAl0IZ3LhDGTuNM7eM0reju2GonzjcTdvqNm4ZBZNOATVN8K+VqyxuLbJCoqKWZVynB+2HWXRjgz8vdyYN64TNf1kzrotJNCFcHZFBZCVZIT70W2QtQdOHYScQ38EvbIaTcW8q4NXAPjVhpiboeFN4FHNlLJ3HM7hzqlriAn1Y/bo9nh7VM2/LKoSCXQhXFVxMZw6YCzFl7HN6CSZdwrycowZNudPgpuXEerNh0BML7C6V2qJP+7M4MHPNtInLpTJQ1pJH5gySKALIf6qqBAOrjYajO2aB2czwacGNLsDwtoYF16DGlRKo7EPlu/npYW7ianlS2xtf2JC/WgXFUTLetUl4C9R7kBXSvUG/gdYgQ+11q9c8rgnMBNoDWQDg7XWaVd7Tgl0IaqQokLYtxQ2f2Z0kSw9Hl+9ZCw+oJ5xR6tXQMlb4B8fe5d87Ol/XRdktdbMWJ3GsqQskjPPcDQnD4DaAV70bVabuxLq0ShUbkyCcga6UsoKJAM9gXRgAzBEa72r1D4PA/Fa64eUUncDA7XWg6/2vBLoQlRRhfnGGHzGdji2y1jo49QByEk3ZttQxkmgp/8fgR9Q1/hlEBhunP2XDv+L+3hU+8vUy5xzBfySlMkP246yPPk4+UXF9IuvzeM9ool28TtOy7tiUVsgRWu9v+TJZgMDgF2l9hkAPF/y8VfAZKWU0maN5wghrp+bhzH9sXb8Xx8rLjYW97g4Dp+XY4T8hdN/fPz79pPGL4EDq43Hr0RZ/wh6qwcAAcDAkreiUM3Jc/mcSiqgeI8mzQluSspu/Tit+z1g9+e1JdDrAodKfZ4OtLvSPlrrQqVUDhAMHC+9k1JqDDAGIDw8/DpLFkKYxmIBL3/j7VqcPwXnT/w19C/9xVBc8JcvtQI1AP/CYtKyczlxodAuh2ImD9+KacZWqV2AtNbTgGlgDLlU5msLIUzkHWi8lYMHEGOfapyWLX+7HAZKrwAbVrLtsvsopdww/mLKtkeBQgghbGNLoG8AopVSUUopD+BuYP4l+8wH7i35+A7gFxk/F0KIylXmkEvJmPg4YAnGcNZHWuudSqlJQKLWej4wHfhUKZUCnMAIfSGEEJXIpjF0rfVCYOEl2/5Z6uM84E77liaEEOJaOP78HyGEEIAEuhBCOA0JdCGEcBIS6EII4SRM67aolMoCDlznl9fgkrtQXYAcs2uQY3YN5TnmCK11yOUeMC3Qy0MplXil5jTOSo7ZNcgxu4aKOmYZchFCCCchgS6EEE7CUQN9mtkFmECO2TXIMbuGCjlmhxxDF0II8VeOeoYuhBDiEhLoQgjhJKp0oCuleiulkpRSKUqpCZd53FMp9WXJ4+uUUpGVX6V92XDMTyildimltimlliqlIsyo057KOuZS+92ulNJKKYef4mbLMSul7ir5Xu9USn1R2TXamw3/t8OVUr8qpTaX/P/ua0ad9qKU+kgpdUwpteMKjyul1Nsl/x7blFKtyv2iWusq+YbRqncfUB9jsZKtQOwl+zwMTC35+G7gS7PrroRj7g74lHw81hWOuWQ/P2A5sBZIMLvuSvg+RwObgeoln9c0u+5KOOZpwNiSj2OBNLPrLucxdwVaATuu8HhfYBGggPbAuvK+ZlU+Q/99cWqtdT5wcXHq0gYAn5R8/BXQQ6lLlg93LGUes9b6V631uZJP12KsIOXIbPk+A7wIvArkVWZxFcSWYx4NTNFanwTQWh+r5BrtzZZj1sDFxUoDgCOVWJ/daa2XY6wPcSUDgJnasBYIVErVLs9rVuVAv9zi1HWvtI/WuhC4uDi1o7LlmEsbhfEb3pGVecwlf4rW01r/UJmFVSBbvs8xQIxSapVSaq1SqnelVVcxbDnm54HhSql0jPUXHq2c0kxzrT/vZarURaKF/SilhgMJwA1m11KRlFIW4E1gpMmlVDY3jGGXbhh/hS1XSjXTWp8ytaqKNQSYobV+QynVAWMVtDitdbHZhTmKqnyG7oqLU9tyzCilbgImAv211hcqqbaKUtYx+wFxwDKlVBrGWON8B78wasv3OR2Yr7Uu0FqnAskYAe+obDnmUcAcAK31GsALo4mVs7Lp5/1aVOVAd8XFqcs8ZqVUS+B9jDB39HFVKOOYtdY5WusaWutIrXUkxnWD/lrrRHPKtQtb/m/Pwzg7RylVA2MIZn9lFmlnthzzQaAHgFKqCUagZ1VqlZVrPnBPyWyX9kCO1vpouZ7R7CvBZVwl7otxZrIPmFiybRLGDzQY3/C5QAqwHqhvds2VcMw/A5nAlpK3+WbXXNHHfMm+y3DwWS42fp8VxlDTLmA7cLfZNVfCMccCqzBmwGwBbja75nIe7yzgKFCA8RfXKOAh4KFS3+MpJf8e2+3x/1pu/RdCCCdRlYdchBBCXAMJdCGEcBIS6EII4SQk0IUQwklIoAshhJOQQBdCCCchgS6EEE7i/wMjv6JTMryDVQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Random model\n"
      ],
      "metadata": {
        "id": "BhRscK4xp0N5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "np.random.seed(1)\n",
        "y_rand = np.random.uniform(0, 1, size=len(y_val))\n"
      ],
      "metadata": {
        "id": "1FRAOo0Jpx_S"
      },
      "execution_count": 133,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "((y_rand >= 0.5) == y_val).mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bqt3fk3pp3NL",
        "outputId": "4e4d4c75-5237-4022-883d-863120d2adfe"
      },
      "execution_count": 134,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5017743080198722"
            ]
          },
          "metadata": {},
          "execution_count": 134
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def tpr_fpr_dataframe(y_val, y_pred):\n",
        "    scores = []\n",
        "\n",
        "    thresholds = np.linspace(0, 1, 101)\n",
        "\n",
        "    for t in thresholds:\n",
        "        actual_positive = (y_val == 1)\n",
        "        actual_negative = (y_val == 0)\n",
        "\n",
        "        predict_positive = (y_pred >= t)\n",
        "        predict_negative = (y_pred < t)\n",
        "\n",
        "        tp = (predict_positive & actual_positive).sum()\n",
        "        tn = (predict_negative & actual_negative).sum()\n",
        "\n",
        "        fp = (predict_positive & actual_negative).sum()\n",
        "        fn = (predict_negative & actual_positive).sum()\n",
        "\n",
        "        scores.append((t, tp, fp, fn, tn))\n",
        "\n",
        "    columns = ['threshold', 'tp', 'fp', 'fn', 'tn']\n",
        "    df_scores = pd.DataFrame(scores, columns=columns)\n",
        "\n",
        "    df_scores['tpr'] = df_scores.tp / (df_scores.tp + df_scores.fn)\n",
        "    df_scores['fpr'] = df_scores.fp / (df_scores.fp + df_scores.tn)\n",
        "    \n",
        "    return df_scores"
      ],
      "metadata": {
        "id": "XW-jkhiip8AM"
      },
      "execution_count": 135,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df_rand = tpr_fpr_dataframe(y_val, y_rand)\n"
      ],
      "metadata": {
        "id": "JqL51jMXp8n8"
      },
      "execution_count": 136,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.plot(df_rand.threshold, df_rand['tpr'], label='TPR')\n",
        "plt.plot(df_rand.threshold, df_rand['fpr'], label='FPR')\n",
        "plt.legend()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "_igM8Eupp_lb",
        "outputId": "194f3e5d-6818-428e-d625-0ecefa12f69d"
      },
      "execution_count": 137,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f0c108f9f90>"
            ]
          },
          "metadata": {},
          "execution_count": 137
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhNV9vH8e99MkqMIQQxz3MQaqipqKloq5RWSxvUVEqrD32eDvTtgNJWS5Waqqpa80ypqeYg5nkOSsQ8RBJZ7x8nbVWRkJPsnHPuz3XlunL23tn7Xg2/bmuvvZYYY1BKKeX8bFYXoJRSyjE00JVSykVooCullIvQQFdKKRehga6UUi7C06oL58iRwxQsWNCqyyullFPasmXLeWNM4L32WRboBQsWJDw83KrLK6WUUxKR4/fbp10uSinlIjTQlVLKRWigK6WUi7CsD10ppRwhLi6OyMhIYmJirC7FoXx9fQkODsbLyyvZP6OBrpRyapGRkWTKlImCBQsiIlaX4xDGGKKjo4mMjKRQoULJ/rkku1xEZLyInBORXffZLyIyQkQOicgOEan0EHUrpVSKxMTEkD17dpcJcwARIXv27A/9r47k9KFPBBo/YH8ToFjiVxfgm4eqQCmlUsiVwvxPj9KmJAPdGLMauPCAQ1oC3xu7DUBWEcn90JUk04kDEawf24dbMddT6xJKKeWUHDHKJS9w8o7PkYnb/kVEuohIuIiER0VFPdLFTm+cSfVT44kaEsrRzYsf6RxKKeUo0dHRhISEEBISQlBQEHnz5v3rs4gQEhJC2bJlad26NTdu3ADAw8Pjr+3Nmzfn0qVLDqklTYctGmPGGGNCjTGhgYH3fHM1SdVeGkRE3QmQcJtCC55nxzcduH7TtZ5uK6WcR/bs2YmIiCAiIoKuXbvSp0+fvz77+/sTERHBrl278Pb2ZvTo0QBkyJDhr+0BAQGMHDnSIbU4ItBPAfnu+BycuC3VhNR9lkx9NrMioC3lz85mxpBOfLvqMDdi41Pzskop9chq1arFoUOH/rW9evXqnDrlmMh0xLDFuUBPEfkJeAy4bIw544DzPlCWLFmo1+tbzk3z5uW939NzSUFqr6nLhy3L0KRcqnXhK6XSsYHzdrPn9BWHnrN0nsy837xMis4RHx/PokWLaNz4n+NLbt++zfLlywkLC0vR+f+UnGGLU4H1QAkRiRSRMBHpKiJdEw9ZCBwBDgFjge4OqSyZcrYaBvke40u/cTzmf4ZuU7bSa+o2Ll6PTcsylFLqX27evElISAihoaHkz5//r+D+c3tQUBBnz56lYcOGDrleknfoxph2Sew3QA+HVPMoPL2h9SQ8xtTha4/hVKwzik/XnGHd4fP0rFeUtlXz4+vlYVl5Sqm0k9I7aUf7s6/8fttv3LhBo0aNGDlyJL169Urx9VxjLpfMuaHNZOTqWTrt68Ki9rkonCMjH8zbQ92hK5m84TjxtxOsrlIppf7Bz8+PESNGMGzYMOLjU/4M0DUCHSD/Y9BxPsTfpNi8VkxrAlM6PUZwtgy8O3sXrUav59C5q1ZXqZRS/1CxYkXKly/P1KlTU3wusfeYpL3Q0FCTKgtcXDgKP7SCy5FQbwCmWg/m7T7Pe3N2cSP2Nv2eLEHY44Ww2VzvzTKl3NHevXspVaqU1WWkinu1TUS2GGNC73W869yh/ymgEIT9CsUawrIPkLFP0CLHGZb2qU2d4oF8tHAvQ5fut7pKpZRyONecbdE/O7SdAnvnwYK3YOwT5MyUhzFB5fitQB4GrjpLoRz+tAnNl/S5lFLKSbhmoP+pVHMoVBu2TYEzEciZHTxx/lfq+RiWzQ1lV9xblK3WCFxwYh+llPtx7UAH8M0C1f8eGi9X/+DWum+ptmEsmZc8T8y2CvjW6gWlW4JH8ieSV0qp9Mb1+tCTkikI30bvc6VrBJ9IZ85FRcGMMPiyApzYYHV1Sin1yNwv0BMF58rBkx3foWHcZwzONhDj4QNT2sAfO60uTSmlHonbBjpA5QIBDHkuhG/OFOOjHIMxPhlh8rMQfdjq0pRSTuTP6XD//Dp27BgrV64kS5YshISEUKpUKQYOHAjwj+0lS5bkrbfeclgdbh3oAC1D8tK7fjG+2xlHkwt9uXwjhujRTdm6eS1WjdFXSjmXP1/l//OrYMGCgH2GxYiICMLDw/nhhx/YunXrP7Zv27aN+fPns3btWofU4faBDvBGg2KMfKEStWvU5PNcH+MVd5WQ+c1YO7gl27ZutLo8pZST8/f3p3Llyv+aPjdDhgyEhISkq+lznZ6I0Kx8bpqVzw2U4taVxuye9QmVjk7Bd04jDiwqgX+JeuSt+CQUeNw+IZhSKv1Z1N/xz8GCykGTTx94yJ+zJwIUKlSIWbNm/WN/dHQ0GzZs4N133+XO1douXrzIwYMHqV27tkNK1UC/B5/MgZTrMJyYS/9h+9zheBxdQaGdY2DXN8T5B+FV7TWo3BH8AqwuVSmVDtxvVsU1a9ZQsWJFbDYb/fv3p0yZMqxcuZI1a9ZQoUIFDh48yBtvvEFQUJBD6tBAfwDfrLmo+PJgYuJu8+PafWxfPZvnri2mxvKBsHoolG0FFV+CfFX15SSl0oMk7qTTWq1atZg/f/59tx89epRq1arRpk2bv+7wU0L70JPB18uDDnXL0K93X/4v4GOaxn7K/sAnMbtmwvgn4esq9rdR9SGqUuohFCpUiP79+zN48GCHnE8D/SHkzpKB6d2qE1yyCo2OtKFb0FQia38GPhlhTnf4voUOeVRKPZSuXbuyevVqjh07luJzud70uWkgIcEwds0RRq08zOWbcTQsGcjHBbYQuOFjuB0L9d+Dat21G0apNKDT5/5N79Afgc0mvFanCGv+U4++DYuz4dhFnlhVmI1NF0GRJ2DJOzCtPdy8ZHWpSik3ooGeApl9vehVvxiLetcib9YMvDDtBJMLfgJPfgQHFsOYOnBmu9VlKqXchAa6AwRn82N6txrUKR7Iu3N2M+xaQ+i4EG7HwbhGsHtW0idRSj0yV3yr+1HapIHuIBl9PBn7cihtQoP56rdD/HwuD3RZaX8p4ZeOsOITSNCFqpVyNF9fX6Kjo10q1I0xREdH4+vr+1A/p+PQHcjDJnz0TDnOXI7hnZk7yRf2GNU7zof5fWDVpxC5CRoPhsDiVpeqlMsIDg4mMjLyH29gugJfX1+Cg4Mf6md0lEsquHwzjlbfrCPq6i1mda9B4Rz+sPk7WD4I4m5AlU5Q5z/6pqlS6qHpKJc0liWDF+M7VMHDJoRNCufijTio2hle3wqVXoZNY+DzsrCwn45bV0o5jAZ6Ksmf3Y8xL1Xm1MWbdP1hC7HxCZAxEJ76HLqtsy95Fz4BvqoMs7rCratWl6yUcnIa6KkotGAAQ1uXZ+PRC7wza+ffD21yloJnvoE+u6BmL9gxDcbUg7N7rC1YKeXU9KFoKmsZkpcjUdf5cvlBTkTfwN/HA4D8AX40LZeb0PoD8Sj2JEx/FcY+AU8Nh5AXLK5aKeWMNNDTwBsNihETf5v1h6OJib+NMbD+SDST1h8nZyYfmpbLTcum8wjZ9CYyuxscXwdNh4JXBqtLV0o5ER3lYpHrt+L5bd855u84zcr9UdyKTyBPJi+G51pEtcjxkKsctJkE2YtYXapSKh150CgXDfR04NqteJbvPcvciNMs33eOF7PtY2DCCDwx0Ho8FG1gdYlKqXQixcMWRaSxiOwXkUMi0v8e+/OLyAoR2SYiO0SkaUqLdicZfTxpGZKXcR2rMOGVKiy/HUK9q4M455kTM6U1rB+pc60rpZKUZKCLiAcwEmgClAbaiUjpuw77H/CzMaYi0BYY5ehC3UW9EjlZ0qc2VSpUoG70ACL8athnb5wRBlEHrC5PKZWOJeehaFXgkDHmCICI/AS0BO4cY2eAzInfZwFOO7JId5MlgxfD2lSgTN4sPLfAm0FZ8/PC3hnIrhlQrBHU6AkFa+l860qpf0hOoOcFTt7xORJ47K5jPgCWisjrgD9wz05fEekCdAHInz//w9bqVkSEsMcLUSiHH6//6MVErwaMLrONIkenwqTmUKAm1HsHCj5udalKqXTCUS8WtQMmGmOCgabAZBH517mNMWOMMaHGmNDAwEAHXdq1PVEyF7N61MQrc07qb6nO2/l+5Gb9TyD6EExsBpOfhWuuNSmRUurRJCfQTwH57vgcnLjtTmHAzwDGmPWAL5DDEQUqKJ4rE7N71KR3/WLM3BlNrVXFmBA6m7gGH9rHrI+tB3/stLpMpZTFkhPom4FiIlJIRLyxP/Sce9cxJ4D6ACJSCnug622jA3l72ujTsDize9SkRFAmBi4+So1VpZkXOh6TcBvGPQl77v61KKXcSZKBboyJB3oCS4C92Eez7BaRQSLSIvGwN4HOIrIdmAp0NK4023w6UjZvFqZ0qsa0LtUoEujP6ysNnXyGEhNQAn5+2T7hl1LKLemLRU7MGMOciNO8P3c3xN9kTuAYCl74nZv1BpGhTm+ry1NKpQKdD91FiQhPV8zL0j61CS2Sh4anuzD/9mNkWPEekz7qwsq9OnpUKXeid+guwhhD5MWbHDhzibxr3qbkH/M4nJCbTfk70+yFHmT2e7i1CZVS6ZPeobsBESFfgB/1y+Sh5GuTiX1uMpky+tMuchDnh4ay4/d5VpeolEplGuiuSATvsi3I+dZmjtQbhS9xlF/WnvARL3L9kg4+UspVaaC7MpuNwnVeJKBfOGuDXiIkeiExX4QybeYMDp27ZnV1SikH0z50N7Jry+9kX9iZbPFRvB73Oidz1uOp8rlpVj4PhXL4W12eUioZdD509bfr54md/Byef2xnbKbufBJVHRDK5MlM51qFaV4hDx42nfRLqfRKH4qqv/nnwPvVBdiKNeS1q19zoMBQJoQex8TH8ca0CJ78fBVzIk6RkKDvhSnlbDTQ3ZG3P7T9EZoNxzv+GvV2DWABPVkUuo3Mcp3eP0XQftxGTl64YXWlSqmHoF0u7i4hAQ4uhfVfw7E1GC9/DuRpyWvH6hBlsvLfZqVpVzUfonOvK5UuPKjLJTnzoStXZrNBicb2rzM7kA2jKLHzF37zns0vGdrw3qyb7DlzmQ9bltVQVyqd0y4X9bfc5eGZ0dBjI7YiT/D8lYlszDwAn82jGTLjd3S+NaXSNw109W/Zi0DbKfDyXLLkDOZdrx/ou7Ml+75sgTm72+rqlFL3oYGu7q9wHaTTMkz3DWwOep6gi1uJ/6Y2R+cOts/BrpRKVzTQVZIkZymqdx3F8vrzWC8VKbT1Y3Z9+gRbdu+zujSl1B000FWyiAjP1a5E1QGLWVv6fYrG7iXjtFZ0Gr2UTUcvWF2eUgodtqgeUezBFXj82JrdphDPx/SnQFAgzSvkoVm53BTUaQSUSjX6pqhyOO9i9fBoM5FycojleceSxTuBoUv2U/ezlXz920Gry1PKLWmgq0dX6imkxVfkiV7PtPi+bHv6Ik1KB/L5soPsOnXZ6uqUcjsa6CplKraHF34GrwxkW9yDry91p3GGvbz1y3Zi4xOsrk4pt6KBrlKueCN4bQ20noSHGL6+/SGNoibyzYr9VlemlFvRQFeOYbNBmafhtdVI+efp4zWDyms6s3P/AasrU8ptaKArx/L2h2dGc/3J4VSx7SP4x3pM+fYTjpy7anVlSrk8HbaoUs2lYzu4Nr07wdd2sjahDN/5deKIRyEA6pfMxYCmJfHy0HsKpR6GrlikrJOQwLV1Y/BaMQif29c54Vuc1X6NGHy6AuWL5mPUC5XJ4udldZVKOQ0dh66sY7OR8fGu+Ly5CxoPJn9WH9pf+IqNWf+Lx7HVPDNqLUfPX7e6SqVcgga6Sht+AVCtK3T9HcJ+xS9jVr73/IhXr43hmRHLmbzhuC57p1QKaaCrtJevKry2Cqq+RnsW8KvXW+yY9zUdvlury94plQIa6MoaXhmg6RB4eS45cuVlqNcYPjr1Kp9/MZipm07oYhpKPQINdGWtwnWQzr9Bu2kEBQYy3PYFfvNeo9u43zh96abV1SnlVJIV6CLSWET2i8ghEel/n2PaiMgeEdktIj86tkzl0kSgRGO8u60ioe5/ae65ifdPdmbAsK8YNG8P567GWF2hUk4hyUAXEQ9gJNAEKA20E5HSdx1TDBgA1DTGlAHeSIValavz8MRW921snX4lR0BWJnl8RKlN/WkxZC5DFu8jJk5XSVLqQZJzh14VOGSMOWKMiQV+AlredUxnYKQx5iKAMeacY8tUbiVvJby6r4XH+/Kc11p+9e7H6dWTeGrEGrafvGR1dUqlW8kJ9LzAyTs+RyZuu1NxoLiIrBWRDSLS+F4nEpEuIhIuIuFRUVGPVrFyD14ZoMH7SJdVZAoqwhfeo/j8Wj8Gjv6eL5Yd0IemSt2Dox6KegLFgLpAO2CsiGS9+yBjzBhjTKgxJjQwMNBBl1YuLagshC2DlqMo43eJmV7vkmvl23w6f6eGulJ3SU6gnwLy3fE5OHHbnSKBucaYOGPMUeAA9oBXKuVsNqj4IrZeWzE1etHOcwWhm3ozdMF2DXWl7pCcQN8MFBORQiLiDbQF5t51zGzsd+eISA7sXTBHHFinUuCTCXnyQ0zTYTT02EqNjd0ZNn+rhrpSiZIMdGNMPNATWALsBX42xuwWkUEi0iLxsCVAtIjsAVYA/Ywx0alVtHJvUrUTCS2/oYbHXppufoVxY0cQcyvW6rKUspzOtqicltm3gKuz3yJzzGn+sAXh90RfMtfobO+iUcpF6WyLyiVJyWZk7reTbdVGcOZ2ZjIve5u9XzzFxt2Hib+t65kq96N36Mol7Dl1mT1zhtLy7Cj+MAH0s/UhPldFiuXKRIXgLDxdMS++Xh5Wl6lUiukCF8ptxBzdiPmlI543z/Np1g+YfqkYl2/GkTOTD93qFqFd1fwa7MqpaaAr93L9PExqAReOYF6YxkbK8sWyA2w4coFcmX3oXrcoz1fJp8GunJL2oSv34p8DOsyFbAWRqW2pZtvHT12qM7VzNQoE+PP+3N3UHbqSSeuOcemGjo5RrkPv0JXrunYOJjaDSyeh3gCo1gNj82Dd4Wg+//UA4ccv4mkTHi+Wg6dD8tKiQh5sNrG6aqUeSLtclPu6dg7m9Yb9CyGoPLQYAXkqYoxh56nLLNhxhvk7znDq0k3aV8vPhy3LIqKhrtIvDXTl3oyBvXNhYT97wJdtBXX+A4HFE3cbPl28j29XHeHl6gUY2KKMhrpKtx4U6J5pXYxSaU4ESreEQnXg989h0xjYPRPKPw8NPkAyBdG/cUkSEgxj1xzFJsL7zUtrqCunow9FlfvIkBUaDoTeO6B6D9g1E76uCuETEGN4p2kpwh4vxMR1x3hjWoQuqKGcjga6cj8ZA+HJ/4Nu6yB3eZj/Bkxshpzdxf+alaJfoxLMiThNu7EbiLp6y+pqlUo2DXTlvnIUhQ7zoMXXELUXRtdC5vWmR5UsfPNiJfaeucLTI9dyIvqG1ZUqlSwa6Mq9iUCll6DXNnisK0RMgREVaXJmJLNfLMD12HhembiJyzfjrK5UqSRpoCsFkCEbNPkUuq2HYg1h/ShKTnucZfknYS4cpfuULcTphF8qndNAV+pOgcWh9QTovR2qdyfH6ZUs9elPmaMT+WB2hC6modI1DXSl7iVrPvuD0x6b8Cz2BO94TeWF7R35aeZ0qytT6r400JV6kCx5oe2PJLT+nmDv67Tb2Yl9I9vYpxNQKp3RQFcqKSLYyrQk01sR/BrYgYLnfiNuRGVYMwxu68NSlX5ooCuVTDbfTDzR7UuGFvuBZXHlYfkg+LYOROoUFip90EBX6iF42IR3XmjE0rKf0Tm2L9cuRcF3DWDp/yBeX0JS1tJAV+ohediEz1pXwK98C6pd+Zg9eZ6FdV/BmHpwdrfV5Sk3poGu1CPwsAnDWlegXoWiND3SisEBA7l56QxmTF2ImGp1ecpNaaAr9Yg8PWx83qYC/RqVYGlcCDWvfMy6uGIwuysD3+1DqXcXM2DmDq7G6INTlTZ0PnSlHMAYw/6zV1mx6wT1d/Wn+KU1zM/VjV4nahGU2Zchz1Xg8WI5rC5TuQBd4EKptHQ7DmZ2gd0zuZS/Ie9FPcHci/l58bECvNO0FP4+ugyBenS6SLRSacnDC1p9B3XfIWtUOCNuDmBtwP9xePNiGn+5mg1Hoq2uULkoDXSlUoPNA+r+B/rsgWbDyet9g6k+H9Mhbjrtxqzjg7m7uREbb3WVysVooCuVmrz9oEoYdFuHlHmWTnFTWJpzJLPX7aTpl2sIP3bB6gqVC9FAVyot+GS0d8M0G0ax61vYkOP/yBd/nNbfrmfAzJ2cunTT6gqVC9BAVyqtiECVTvDKInyJ5XvzDh+VimT6lpPUHbqC/83eyY7IS9yM1bVM1aPRUS5KWeHyKfjpBTiznUu1P2DI5fr8vPkk8QkGEciXzY/GZYPo06A4Gbw9rK5WpSMpHuUiIo1FZL+IHBKR/g84rpWIGBG558WUUomy5IVXFkGp5mRd/T4fZ1vA72/XY9SLlehdvxjFc2VizOojNB2xhi3HtZ9dJU+Sd+gi4gEcABoCkcBmoJ0xZs9dx2UCFgDeQE9jzANvv/UOXSngdjzMfR22/wg1XoeGH9q7ZoB1h87Tb/oOzly+Sfe6RenbsDg2m1hcsLJaSu/QqwKHjDFHjDGxwE9Ay3sc9yEwGIh55EqVcjcentByJFTpbJ/ga15ve8gDNYrmYEmf2rSqFMzXKw7R9YctXL+lQx3V/SUn0PMCdy7PEpm47S8iUgnIZ4xZ8KATiUgXEQkXkfCoqKiHLlYpl2SzQdOhUOtN2DoJprWH2BsAZPTxZMhz5XnvqdIs23uW1qPXc1pHxKj7SPEoFxGxAcOBN5M61hgzxhgTaowJDQwMTOmllXIdIlD/PWj6GRxYDN+3gOvRibuEVx8vxLgOVThx4QbNRqxh/o7TFhes0qPkBPopIN8dn4MTt/0pE1AWWCkix4BqwFx9MKrUI6jaGZ6fDH/shPGN4MrfwV2vZE5m96hJ/gA/ev64jR5TthJ9TRfVUH9LTqBvBoqJSCER8QbaAnP/3GmMuWyMyWGMKWiMKQhsAFok9VBUKXUfpZrDS7Pg6h8wvjFcPPbXrqI5MzKjWw37lL17/qDeZyv5ctlBrugUvYpkBLoxJh7oCSwB9gI/G2N2i8ggEWmR2gUq5ZYK1IAOc+DWFXuoR+3/a5enh40e9YqyoFctqhXOzufLDlBr8Ar+b/4eZm2LZNepy8TE6ctJ7khfLFIqPTu7G75/GuJuQJPBEPLiX8Ma/7Tr1GW+WHaQlfvPEZ9g//uc2deTTrUK80rNgmTy9bKicpVKdD50pZzZpZMwqysc/x1KPgXNR4B/9n8dFnc7gWPnr7P/7FVmbzvNsr1nyZLBi171ixH2eCELClepQQNdKWeXcBvWj4TfPgSfzPa79bKt/nW3fqcdkZcYumQ/aw6eZ3ibCjxbKTgNC1apRRe4UMrZ2TygZi/oshKyFYAZYTDlObhw9L4/Uj44KxM6VqFqwQD+N3sXh6OupVm5yhoa6Eo5k1xlIOxXaDwYjq+HryrD9DA4tfWeh3t62PiyXQg+njZ6TNmqD0tdnAa6Us7G5gHVukLPzVCtGxxYAmPrwbgnYetkuHX1H4fnzpKB4W1C2PfHVd6bs4vY+ASLClepTfvQlXJ2MVdg22QInwDRB8HLH8o+aw/7XGX+Omzw4n18s/IwebNmoOcTRXmucjBeHnpP52z0oahS7sAYOLnJHu67ZtiHOhauCzXfgCL1MMaw6kAUny87yPaTlyicw59fulYne0YfqytXD0EDXSl3c+MCbJkIm8bA1TNQ8SVo/Cn4ZMQYw7K95+jx41aqF87OhI5VdFpeJ6KjXJRyN34BUKsv9N4Bj/eFbT/A6JpwchMiQsPSuXj3qdKsOhDFmDVHrK5WOYgGulKuzNMbGrxvXx3JJNinEdg0FoD2j+Wnabkghi7Zr6siuQgNdKXcQYHq0PV3KNoAFr4F8/siCfF82qo8ebL60vPHbeyMvGx1lSqFNNCVche+WaDdVKjZG8LHweRnyJxwlW9erEyCMTw9ai3Dfz2gwxqdmAa6Uu7E5gENB8Ez39pHxIytR1mv0yx9ow4tQ/IwYvlBnh65lj2nr1hdqXoEGuhKuaMKbeGVhRB3E75rSJaTyxjeJoQxL1Xm3NVbtBz5OyOWHyTutt6tOxMNdKXcVXAodF4B2QvD1LYwPYwng+P5tU9tGpfNzfBfD/DMqLWsO3Qeq4Y3q4ej49CVcnexN+D34bDuK0CgRk+o0omFxwwD5+3m7JVbVC0UQJ8Gxale5N/T9qq0pS8WKaWSdukE/Po+7J4J4gFFGxBb/kWmXi7DyFXHOHf1FtULZ6dPw+JULRRgdbVuSwNdKZV85w9BxBSI+BGu/QGBJYmt9TaTL4fwzaqjnL92ixpFstO2an7ql8yJv4+n1RW7FQ10pdTDux0Pe+fAysFwfj/kLENsxVf46WZVRm6I4uyVW/h42mhQKhcftChDYCadEyYtaKArpR5dwm3YNRPWfgFnd4GnLwklmxNRsg+zDxumbT7J40Vz8F2HUOQBKygpx9C5XJRSj87mAeVb29807bIKKr6Ebf9CKi1+lkGVb9GvUQmW7zvH7IhTVlfq9jTQlVLJIwJ5QqDZZ9BpmX2emAlNeDXzZirlz8oHc/dw7mqM1VW6NQ10pdTDy1UaOq+E4CrYZnXhu/zLuBkXz7uzd+mYdQtpoCulHo1/dnhpFoS8SED4cObkm8ay3adpN3YDUzYeJ/raLasrdDv6UFQplTLGwIqPYPVQjgU8To9bPdgdbfCwCdULZ6dZ+dw0LhNENn9vqyt1CfpQVCmVekTgif/BU59T8OI65nsPYOXzGehapzCRF28wYOZOqny0jMkbjltdqcvTQFdKOUboq9BxIWISKDi3Ff08f2FFn5os6PU4NYvm4P05u1ix75zVVbo0DXSllOMUqA5d10KFF2DNZ8i09pQJ9Oab9pUolTszr0/dxr4/dGre1KKBrpRyLN/M8PRIaDYcDi6FKa3xMzGM61AFfx8PwiaG6/DGVKKBrpRKHVXC4JnRcHwtTH6GINtlxnWowhCAFAgAAAwfSURBVIXrsXT+fgsxcbetrtDlJCvQRaSxiOwXkUMi0v8e+/uKyB4R2SEiy0WkgONLVUo5nQptofUkOL0NvihL2fB3GNfEjx2Rl3jz5+0kJOiYdUdKMtBFxAMYCTQBSgPtRKT0XYdtA0KNMeWB6cAQRxeqlHJSpVtAj41Q6WXYOYMaS5uzJmgE13YvZtjSfVZX51KSc4deFThkjDlijIkFfgJa3nmAMWaFMeZG4scNQLBjy1RKObXsRaDZMOi7B+q/R97440zyHszT61qxdM4Uq6tzGckJ9LzAyTs+RyZuu58wYNG9dohIFxEJF5HwqKio5FeplHINfgFQ602k9w7iW47G38eD+lt7sGTsf4mP1z71lHLoQ1ERaQ+EAkPvtd8YM8YYE2qMCQ0MDHTkpZVSzsTTG8+K7cj15nr2Z6tLo1Nfs/7ztly8fNXqypxacgL9FJDvjs/Bidv+QUQaAP8FWhhjdBIHpVSSPHwzUrrXTHYW7Uat60uJGV6eX758i/mb9nD9VrzV5TmdJOdyERFP4ABQH3uQbwZeMMbsvuOYitgfhjY2xhxMzoV1Lhel1J1ObJ5PwurhFLy6hevGh/G2Z/Gs/SYv1yiky9zdIcUrFolIU+ALwAMYb4z5SEQGAeHGmLkisgwoB5xJ/JETxpgWDzqnBrpS6l4STm/n0uKPCTixmFW3yzPQqzePVyhJiaBMFM+VibJ5spDB28PqMi2jS9AppZyLMbBlIgmL3uYSmRkQ14klseUAIcDfm9dqF+al6gXw83a/O3edbVEp5VxEIPQVbGG/EpA5E9/aPuVg8P+xoNZxKubJwCeL9lF7yAq+X39MX066gwa6Uir9yhNifymp5Si8bDbKbB7AuNvvMqtTeYrnysR7c3bTbuwGTkTfSPpcbkADXSmVvnn6QMUXodtaaDUOzmyn4u/dmNKxAkOeK8+e01do/OVqxv1+1O3nh9FAV0o5BxEo95x9wq9ja5DpYbSpmJslfWoTWjCAD+fvoe7QlUzecJxbbvqSkga6Usq5lG8DTYbA/gXwSwfyeF5j0itVmNLpMYKzZeDd2bto8dVaoq663+swGuhKKefz2GvQ6GM4sARGVkG2T6Vmkez80rU6370cyokLN3jxuw2cd7OFqjXQlVLOqXoP6Po75CgBs7vBhKbI8XU0KJ2L8R2rcOLCDdp/t5FoNwp1DXSllPPKWRJeWQRPfQEXDsPEpvB9S6r7HmdchyocPX+ddmM3cPKCe4yC0UBXSjk3mw1CX4He2+3dMGd3w/hG1Ly5kgmvVOHM5RieGbWWLccvWl1pqtNAV0q5Bq8M9m6YnpshuCrMCKPG2anM6lYDfx9P2o3dwJyIf80r6FI00JVSriVDNnhpJpR5Bpb+j6Kb3mV2WDkq5svKG9Mi+Dn8ZNLncFLuNxGCUsr1efpAq/GQJR+s+4pse+fxQ513CPMoxX9m7MAmwnOVXW9hNb1DV0q5JpsNnvwQuqyAHMXwWtSXibFv0jvPPvpNj2Dm1kirK3Q4DXSllGvLU9E+Eqb1RGy3Y3kjehC/ZXyfeTMmMXuba/Wpa6ArpVyfiL1PvftGeHo0BfzjmeA1hEsz+jB361Grq3MYDXSllPvw8ISQdth6biauSlc6ei4h/+xWLFu/2erKHEIDXSnlfjy98Wo2mFvPTqC4xxmqLm7BkblDID7W6spSRANdKeW2fMo/S0KXlRzwKkHhrR8R+/Vj9vlhnJQGulLKrWXMXYKg7gvobRvAH5dj4Mc2MKsbxFyxurSHpoGulHJ7wQH+dOj4Gk3jhvCzXzvMjp9gdE04vs7q0h6KBrpSSgGV8mdj6POh/O9KSzowiOtxYCY0hYX9nOZuXQNdKaUSNSmXm4W9anE5RyWqRH/AskxPYzaN5daXVbgSMdvq8pKkga6UUncomjMjM7pWp0ejCrx9/QWeuTWQI9e9yDy7A8e/aQVXzlhd4n2JMcaSC4eGhprw8HBLrq2UUslhjCHq2i0On7lI5MKhNL/4PXj64N1oELbQjmDzSPOaRGSLMSb0Xvv0Dl0ppe5DRMiZyZfqxXPzbK9hfFd2Cltj82Nb2Je4r6rAzumQkH4WpNZAV0qpZPCwCT1bN+Jos6n0jO/DkQuxMCOMuK8fgzXDIOqA1SVql4tSSj2s05du8s2KA1zeMp1XbQsJsR2y7wgsBa3GQlC5VLv2g7pcNNCVUuoRRV68wcgVh1kTHkFDjy284TufjF7gEbYEshdJlWtqH7pSSqWC4Gx+fPJsOaa+1YqbIa/S+vp/uHw9hovfNuP8qbSfxVEDXSmlUihfgB+ftirPd2++yPdFhuN56xIXxzxF/++XsWjnGW7Gps2DU+1yUUopB/sj4lcC5rzIZeNHj1s92e1Vlk9blad5hTwpPneKu1xEpLGI7BeRQyLS/x77fURkWuL+jSJSMGUlK6WU8woKaYh31xXkCMjONN+PGZB5MX2nbWX+jtOpet0kA11EPICRQBOgNNBORErfdVgYcNEYUxT4HBjs6EKVUsqp5CqDvLYSKd2S9tcmsDpDP8J/HsySbYdS7ZLJuUOvChwyxhwxxsQCPwEt7zqmJTAp8fvpQH0REceVqZRSTsgnEzw3HlpPImeu3HzgOZFqs2uzfeF3qXK55AR6XuDkHZ8jE7fd8xhjTDxwGch+94lEpIuIhItIeFRU1KNVrJRSzkQEyjyNR5ffuPHSYg74V8Y/qHCqXMozVc56H8aYMcAYsD8UTctrK6WU1fyKVKfK2/NS7fzJuUM/BeS743Nw4rZ7HiMinkAWINoRBSqllEqe5AT6ZqCYiBQSEW+gLTD3rmPmAh0Sv38O+M1YNR5SKaXcVJJdLsaYeBHpCSwBPIDxxpjdIjIICDfGzAXGAZNF5BBwAXvoK6WUSkPJ6kM3xiwEFt617b07vo8BWju2NKWUUg9DX/1XSikXoYGulFIuQgNdKaVchAa6Ukq5CMtmWxSRKOD4I/54DuC8A8txBtpm96Btdg8paXMBY0zgvXZYFugpISLh95s+0lVpm92Dttk9pFabtctFKaVchAa6Ukq5CGcN9DFWF2ABbbN70Da7h1Rps1P2oSullPo3Z71DV0opdRcNdKWUchHpOtDdcXHqZLS5r4jsEZEdIrJcRApYUacjJdXmO45rJSJGRJx+iFty2iwibRJ/17tF5Me0rtHRkvFnO7+IrBCRbYl/vptaUaejiMh4ETknIrvus19EZETif48dIlIpxRc1xqTLL+xT9R4GCgPewHag9F3HdAdGJ37fFphmdd1p0OZ6gF/i993coc2Jx2UCVgMbgFCr606D33MxYBuQLfFzTqvrToM2jwG6JX5fGjhmdd0pbHNtoBKw6z77mwKLAAGqARtTes30fIfujotTJ9lmY8wKY8yNxI8bsK8g5cyS83sG+BAYDMSkZXGpJDlt7gyMNMZcBDDGnEvjGh0tOW02QObE77MAp9OwPoczxqzGvj7E/bQEvjd2G4CsIpI7JddMz4HusMWpnUhy2nynMOz/h3dmSbY58Z+i+YwxC9KysFSUnN9zcaC4iKwVkQ0i0jjNqksdyWnzB0B7EYnEvv7C62lTmmUe9u97ktJ0kWjlOCLSHggF6lhdS2oSERswHOhocSlpzRN7t0td7P8KWy0i5YwxlyytKnW1AyYaY4aJSHXsq6CVNcYkWF2Ys0jPd+juuDh1ctqMiDQA/gu0MMbcSqPaUktSbc4ElAVWisgx7H2Nc538wWhyfs+RwFxjTJwx5ihwAHvAO6vktDkM+BnAGLMe8MU+iZWrStbf94eRngPdHRenTrLNIlIR+BZ7mDt7vyok0WZjzGVjTA5jTEFjTEHszw1aGGPCrSnXIZLzZ3s29rtzRCQH9i6YI2lZpIMlp80ngPoAIlIKe6BHpWmVaWsu8HLiaJdqwGVjzJkUndHqJ8FJPCVuiv3O5DDw38Rtg7D/hQb7L/wX4BCwCShsdc1p0OZlwFkgIvFrrtU1p3ab7zp2JU4+yiWZv2fB3tW0B9gJtLW65jRoc2lgLfYRMBHAk1bXnML2TgXOAHHY/8UVBnQFut7xOx6Z+N9jpyP+XOur/0op5SLSc5eLUkqph6CBrpRSLkIDXSmlXIQGulJKuQgNdKWUchEa6Eop5SI00JVSykX8PzEFe9EMb8DYAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Ideal model\n"
      ],
      "metadata": {
        "id": "xhXzztmSqD__"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "num_neg = (y_val == 0).sum()\n",
        "num_pos = (y_val == 1).sum()\n",
        "num_neg, num_pos\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PX3e9XehqCLj",
        "outputId": "8b2e0f9a-ca0b-4edb-8732-9551f9df65af"
      },
      "execution_count": 138,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1023, 386)"
            ]
          },
          "metadata": {},
          "execution_count": 138
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y_ideal = np.repeat([0, 1], [num_neg, num_pos])\n",
        "y_ideal\n",
        "\n",
        "y_ideal_pred = np.linspace(0, 1, len(y_val))\n"
      ],
      "metadata": {
        "id": "QOs01krGqHl_"
      },
      "execution_count": 139,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "1 - y_val.mean()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gl17WcvmqKMD",
        "outputId": "d4bd29a6-1d88-4d67-c2aa-31d5e0fe6b65"
      },
      "execution_count": 140,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7260468417317246"
            ]
          },
          "metadata": {},
          "execution_count": 140
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "accuracy_score(y_ideal, y_ideal_pred >= 0.726)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ozHk15wfqMdI",
        "outputId": "bf303196-4518-4751-ef18-e6c62a093465"
      },
      "execution_count": 141,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.0"
            ]
          },
          "metadata": {},
          "execution_count": 141
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_ideal = tpr_fpr_dataframe(y_ideal, y_ideal_pred)\n",
        "df_ideal[::10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 394
        },
        "id": "3xpM3155qRK2",
        "outputId": "a7a1cd36-8e81-46dd-a7c0-5703a38b6e07"
      },
      "execution_count": 142,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     threshold   tp    fp   fn    tn       tpr       fpr\n",
              "0          0.0  386  1023    0     0  1.000000  1.000000\n",
              "10         0.1  386   882    0   141  1.000000  0.862170\n",
              "20         0.2  386   741    0   282  1.000000  0.724340\n",
              "30         0.3  386   600    0   423  1.000000  0.586510\n",
              "40         0.4  386   459    0   564  1.000000  0.448680\n",
              "50         0.5  386   319    0   704  1.000000  0.311828\n",
              "60         0.6  386   178    0   845  1.000000  0.173998\n",
              "70         0.7  386    37    0   986  1.000000  0.036168\n",
              "80         0.8  282     0  104  1023  0.730570  0.000000\n",
              "90         0.9  141     0  245  1023  0.365285  0.000000\n",
              "100        1.0    1     0  385  1023  0.002591  0.000000"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d022491a-5f1f-4167-8bff-2d7bdf2944e5\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>threshold</th>\n",
              "      <th>tp</th>\n",
              "      <th>fp</th>\n",
              "      <th>fn</th>\n",
              "      <th>tn</th>\n",
              "      <th>tpr</th>\n",
              "      <th>fpr</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.0</td>\n",
              "      <td>386</td>\n",
              "      <td>1023</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>0.1</td>\n",
              "      <td>386</td>\n",
              "      <td>882</td>\n",
              "      <td>0</td>\n",
              "      <td>141</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.862170</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>0.2</td>\n",
              "      <td>386</td>\n",
              "      <td>741</td>\n",
              "      <td>0</td>\n",
              "      <td>282</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.724340</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>0.3</td>\n",
              "      <td>386</td>\n",
              "      <td>600</td>\n",
              "      <td>0</td>\n",
              "      <td>423</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.586510</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>0.4</td>\n",
              "      <td>386</td>\n",
              "      <td>459</td>\n",
              "      <td>0</td>\n",
              "      <td>564</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.448680</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50</th>\n",
              "      <td>0.5</td>\n",
              "      <td>386</td>\n",
              "      <td>319</td>\n",
              "      <td>0</td>\n",
              "      <td>704</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.311828</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60</th>\n",
              "      <td>0.6</td>\n",
              "      <td>386</td>\n",
              "      <td>178</td>\n",
              "      <td>0</td>\n",
              "      <td>845</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.173998</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70</th>\n",
              "      <td>0.7</td>\n",
              "      <td>386</td>\n",
              "      <td>37</td>\n",
              "      <td>0</td>\n",
              "      <td>986</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.036168</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>80</th>\n",
              "      <td>0.8</td>\n",
              "      <td>282</td>\n",
              "      <td>0</td>\n",
              "      <td>104</td>\n",
              "      <td>1023</td>\n",
              "      <td>0.730570</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>90</th>\n",
              "      <td>0.9</td>\n",
              "      <td>141</td>\n",
              "      <td>0</td>\n",
              "      <td>245</td>\n",
              "      <td>1023</td>\n",
              "      <td>0.365285</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100</th>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>385</td>\n",
              "      <td>1023</td>\n",
              "      <td>0.002591</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d022491a-5f1f-4167-8bff-2d7bdf2944e5')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d022491a-5f1f-4167-8bff-2d7bdf2944e5 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d022491a-5f1f-4167-8bff-2d7bdf2944e5');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 142
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plt.plot(df_ideal.threshold, df_ideal['tpr'], label='TPR')\n",
        "plt.plot(df_ideal.threshold, df_ideal['fpr'], label='FPR')\n",
        "plt.legend()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "vMHjeWlYqT_Z",
        "outputId": "c8a278fc-f85d-4796-b23f-a41a2903b1de"
      },
      "execution_count": 143,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f0c104d1a10>"
            ]
          },
          "metadata": {},
          "execution_count": 143
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxV9Z3/8dcnGwkQ9rBlV1AW2cOStFpbEQFFXNlNVKpVRzsdp/0N/dlpbev8ptYuv+nUdmpdWJTVFRSKu1hZJAiigCJCgIAshk2WAAnf+eMmNIRgbsi999zl/Xw8eDT33sM9n2PCuyfnfs7na845REQk8sV5XYCIiASGAl1EJEoo0EVEooQCXUQkSijQRUSiRIJXO27Xrp3LycnxavciIhFp1apVXzrn0up6zbNAz8nJobi42Kvdi4hEJDPbeq7XdMlFRCRKKNBFRKKEAl1EJEp4dg1dRCQQTp48SWlpKeXl5V6XElDJyclkZGSQmJjo999RoItIRCstLSU1NZWcnBzMzOtyAsI5R1lZGaWlpeTm5vr99+q95GJmT5rZHjP7+Byvm5n9wcw2mdlaM+vfgLpFRBqlvLyctm3bRk2YA5gZbdu2bfBvHf5cQ58KDP+a10cAXav+3An8uUEViIg0UjSFebXzOaZ6L7k455aYWc7XbDIamO58c3iXm1krM+vknPuiwdX4Ye3aDzi2YhrvZd0Fps90RYIpNTmRiUOyaJqkq7ORIBDfpXRge43HpVXPnRXoZnYnvrN4srKyzmtnRz58ifwdU9mxbRP/p+J7VBJ/Xu8jIvVzDl76cAePFw6kY8tkr8sJS2VlZVxxxRUA7Nq1i/j4eNLSfDdyfvjhh/Tp04eKigq6d+/OtGnTaNq0KfHx8fTq1YuKigpyc3OZMWMGrVq1anQt5s8CF1Vn6C875y6p47WXgV855/5e9fgN4N+cc197G2heXp477ztFl/wG3vwldB8FNz4BCU3O731E5Gu9+clu7pu5mubJCTxeOJBeGS29LuksGzZsoHv37l6XAcCDDz5I8+bN+eEPfwhA8+bNOXz4MAATJ05kwIAB3H///Wc8X1RUxEUXXcQDDzxw1vvVdWxmtso5l1fX/gNxzWIHkFnjcUbVc8Fz2Q9h+K9gwwKYPQFOHAnq7kRi1Xe6deC5ewpIiIvj5r8s5fO9h70uKWJdeumlbNq06azn8/Pz2bEjMJEZiEsu84F7zWw2MBg4GKzr52cYcjckNYMF/wxTr4EJc6F5nfNqRKQRunVswfP3FHDpr9/iqfe28NB1vbwu6Zx+vmAd63ceCuh79ujcgp+N6tmo96ioqGDRokUMH35mf0llZSVvvPEGkydPbtT7V/OnbXEWsAy42MxKzWyymd1lZndVbbIQ2AxsAv4K3BOQyvzRvxDGPg17NsATV0LZ5yHbtUgs6dAimdF9OvP8Bzs4VH7S63IixrFjx+jbty95eXlkZWWdDu7q5zt27Mju3bu58sorA7I/f7pcxtfzugP+KSDVnI9uV0PRfJg5Fp4Y5jtTzxjgWTki0aqoIId5q0qZV1zK5G/6f7NLKDX2TDrQUlJSWLNmzTmfP3r0KFdddRWPPvoo3//+9xu9v+jo+8scBJNf812CmXYNbFzsdUUiUeeS9JYMyG7NjGUlnDpVfzOF1K9p06b84Q9/4Le//S0VFRWNfr/oCHSAdl18od6uK8waDx9M97oikahTmJ9NSdlR3vlsr9elRI1+/frRu3dvZs2a1ej3iq67BVI7wK2vwNwimH8fHNoJ3/o3iMK7yES8MOKSTjyUuoHpS0v49sXtvS4n7Dz44INnPK5uTayt9vMLFiwIyP6j5wy9WpNUmDAH+k6Et/8TFnwfKhv/q4yIQFJCHBMHZ/H2xr2UfKl24XATfYEOEJ8Iox+Fy37ku/SiXnWRgJkwKIt4M6YvO+dKaOKR6Ax08F1m+c5P4Jrfw6bXfL3qh3XdT6Sx2rdIZmSvTsxbtZ0jx/XbbziJ3kCvlnc7jH1GveoiAVRUkMNX5RW8sDq4N4VLw0R/oAN0G+nrVS8/6OtV37HK64pEIlr/rFZckt6C6ctK8GcelIRGbAQ6nNmrPlW96iKNYWYU5eewcfdhln1e5nU5UiV2Ah3Uqy4SQKP6dKZ100SmLi3xuhTPxcfH07dv39N/SkpKePvtt2nZsiV9+/ale/fu/PznPwc44/lu3bqdnswYCNHVh+4P9aqLBERyYjzjBmXxl3c+p3T/UTJaN/W6JM/UdYt/SUkJl156KS+//DJHjhyhb9++jBo1CuD088eOHaNfv35cf/31fOMb32h0HbF1hl6tule9zwT1qos0wqQh2Thg9vvb6902ljVr1owBAwacNT43JSWFvn37htX43MgUnwjX/QlapsOSR+CrXXDzVN81dhHxS3qrFC6/KI25xdv5wdCuJMR7fI64aArs+iiw79mxF4z41dduUj09ESA3N5cXXnjhjNfLyspYvnw5//7v/87evf9on96/fz+fffYZl112WUBKjc0z9Gpn9Kq/rl51kfMwblAWe746zpuf7PG6FM9UX3JZs2bNGWH+7rvv0q9fP4YNG8aUKVPo2bPn6ef79OlDeno6V111FR07dgxIHbF7hl5T3u3QvCM8e7uvV33Sc9D2Qq+rEokI3+nWnvapTZi9cjvDegYmmM5bPWfSoVZ9rfxcz2/ZsoUhQ4YwZsyY02f4jRHbZ+g11e5VL1Wvuog/EuPjuDkvg7c/3cPOA8e8Liei5ObmMmXKFB5++OGAvJ8CvSbNVRc5L+MGZnHKwdxifTjaUHfddRdLliyhpKSk0e+lQK+tdq/6qmleVyQS9jLbNOXSru2Yu3I7lTG4+EVdY3Ivv/zyOi+31H4+JSWFHTt2kJOT0+g6FOh1qe5Vv+ByX0vjW/8Jur1Z5GuNG5jFzoPlvLMxdj8c9ZoC/VxqzlV/51fqVRepx5U9OpCW2kRjdT2kQP86mqsu4rekhDgmDMri7U/3siXEi19E44Cw8zkmBXp9NFddxG8TB2eREGdMX1YSsn0mJydTVlYWVaHunKOsrIzk5OQG/T31oftLveoi9ape/OLZ4lJ+OOximjUJfsRkZGRQWlp6xh2Y0SA5OZmMjIwG/R0FekN0GwlFC2DmGF+v+oS5kDHA66pEwkpRQTbzP9zJ86t3cMuQ7KDvLzExkdzc3KDvJxLokktDZQ5Ur7rI1+if1dq3+MVSLX4Ragr086FedZFzql784rM9Wvwi1BTo56t2r/o7j6hXXaTKqD6dadMsSYtfhJgCvTGqe9V7j4W3HoLXf6ZQF6Fq8YuBmby+YTel+496XU7MUKA3VnwiXPc/vi6Y9/4LFv4ITp3yuioRz02s+kD06eXbPK4kdijQAyEuDq7+HRTcByv/Ci/eBRUnvK5KxFPprVIY1qMjs1duo/xkpdflxAQFeqCYwZW/9N2EtHYOzLwZyg95XZWIp4oKcjhw9CTz1+z0upSY4Fegm9lwM/vUzDaZ2ZQ6Xs8ys7fMbLWZrTWzkYEvNQKY+cYEjP4TbHkXpo6EQ194XZWIZ4Zc0IaLO6QyVS2MIVFvoJtZPPAoMALoAYw3sx61NvsJMNc51w8YB/wp0IVGlH4TYeJcKNvsuwFp70avKxLxhJlRWJDN+i8OsWrrfq/LiXr+nKEPAjY55zY7504As4HRtbZxQIuqr1sC+v2qy1C47RWoOAZPDoNty72uSMQT1/dLJzU5gWmawhh0/gR6OlBzGZLSqudqehCYZGalwELgvrreyMzuNLNiMyuOtrkLdercz3cDUkobmD4aNizwuiKRkGualMCYvEwWffQFuw+Ve11OVAvUh6LjganOuQxgJDDDzM56b+fcY865POdcXlpaWoB2Heba5PpCvWMvmHMLvP9XrysSCbnC/GwqneOZFWphDCZ/An0HkFnjcUbVczVNBuYCOOeWAclAu0AUGBWatYXC+XDxCFj4Q3j9Qd2AJDElu20zvn1xe2au2MaJCt2nESz+BPpKoKuZ5ZpZEr4PPefX2mYbcAWAmXXHF+gxcE2lAZKawpgZMOA2+Pvv4YXvqVddYkphfjZfHj7Ooo/V+RUs9Qa6c64CuBdYDGzA182yzsx+YWbXVm32r8AdZvYhMAu41alH6WzxCb6FMtSrLjHosq5p5LZrpvkuQeTXPHTn3EJ8H3bWfO6nNb5eD3wjsKVFqepe9dTOMP8+X6/6hHnQopPXlYkEVVycUZifzc8XrGdt6QF6Z7TyuqSooztFvdJvom+BjLLNvhWQ9n7qdUUiQXfTgAyaJcUzbalaGINBge6lrtW96uW+G5DUqy5RLjU5kRv6Z7Bg7U7KDh/3upyoo0D3WnWvetO26lWXmFBUkM2JilPMXrm9/o2lQRTo4UC96hJDurRP5Rtd2vLM8q1UVKqFMZAU6OFCveoSQ4ryc9h5sJzXN+z2upSookAPJ2f1qmuuukSnK7p3IL1ViloYA0yBHm7O6FWfrV51iUrxccYt+dks37yPT3bp5ztQFOjhqOZc9ZK/a666RKWxeZk0SYhTC2MAKdDDWb+JvkWo1asuUah1sySu65vOi6t3cPDoSa/LiQoK9HDXZSjcthAqjqtXXaJOYUE2x05WMm+VWhgDQYEeCTr3he++Bs3aqVddokrPzi0ZmNOa6cu2UnlKXV2NpUCPFK1z4PZX1asuUaeoIIdt+47yzsY9XpcS8RTokaR2r/prP4NTujFDIttVPTvSoUUTpurD0UZToEeamr3q7/1/eFG96hLZEuPjmDg4myUb9/L53sNelxPRFOiRSHPVJcqMH5RFUnwcM7SQdKMo0CNVzV71Le/CU+pVl8iVltqEq3t34tlVpRw+XuF1ORFLgR7pqueq71OvukS2wvxsDh+v4PkPSr0uJWIp0KOB5qpLFOiX1Zo+GS2ZtrQErWB5fhTo0aLmXPVp18L62ut4i4S/ooIcPt97hL9v+tLrUiKSAj2aVM9V79Qb5hbCise8rkikQa7u3Ym2zZI03+U8KdCjTc1e9UU/Uq+6RJQmCfGMH5TFG5/sZvu+o16XE3EU6NGoulc97/Z/9KpXaviRRIaJQ7KIM2PGcp2lN5QCPVrFJ8DVv/tHr/qcW+BkuddVidSrU8sUrurZgTkrt3PsRKXX5UQUBXo0q+5VH/kb2LgIZo6B47oTT8JfUX4OB4+d5KU1O7wuJaIo0GPBoDvguj9Dybvw9A1wbL/XFYl8rUG5bejWMZWpamFsEAV6rOg7AW56CnZ8AE8Oh4O6eUPCl5lRVJDDJ7u+4v0t+7wuJ2Io0GNJz+tg0nNwaCc8fiXsXud1RSLndF3fdFqmJDJtWYnXpUQMBXqsueBbcNsiwPnO1Lcs8boikTqlJMUzdmAmi9ft5ouDx7wuJyIo0GNRx0t8NyC16AxP3wgfPet1RSJ1umVINqec45nl27wuJSIo0GNVq0y4/W+QngfPTYalf/S6IpGzZLZpyhXd2jPr/W0cr1ALY338CnQzG25mn5rZJjObco5txpjZejNbZ2YzA1umBEVKa7jlBegxGl59AP72Y91VKmGnqCCHsiMneGWtxkPXp95AN7N44FFgBNADGG9mPWpt0xX4MfAN51xP4AdBqFWCITEZbpoKg++G5X+CZ2/TDUgSVr7ZpR0XpDVj2tISr0sJe/6coQ8CNjnnNjvnTgCzgdG1trkDeNQ5tx/AOafVXiNJXBwM/08Y9hCsf1G96hJWzIyi/Bw+LD3Imu0HvC4nrPkT6OnA9hqPS6ueq+ki4CIze8/MlpvZ8LreyMzuNLNiMyveu3fv+VUswWEGBffBjU/A9vfVqy5h5cYBGTRvkqCz9HoE6kPRBKArcDkwHvirmbWqvZFz7jHnXJ5zLi8tLS1Au5aA6nVTjV71obDrY68rEqF5kwRuGpDBy2t3sver416XE7b8CfQdQGaNxxlVz9VUCsx3zp10zm0BNuILeIlEp3vVDZ4aoV51CQu35GdzstIx6321MJ6LP4G+EuhqZrlmlgSMA2ovh/MivrNzzKwdvkswmwNYp4Rax0vgu69Bi3SYcYN61cVzF6Y159Ku7XhmxVZOVqobqy71BrpzrgK4F1gMbADmOufWmdkvzOzaqs0WA2Vmth54C/iRc64sWEVLiLTMgNsXQeagql71/wYNShIP3VqQw+5Dx1m8bpfXpYQl82qSWV5enisuLvZk39JAJ8vhhe/5OmAG3w1X/T9fZ4xIiFWeclz+m7fo1CKFuXfle12OJ8xslXMur67X9K9S6peY7JvUOPhuWPFn9aqLZ+LjjMIhObxfso/1Ow95XU7YUaCLf+LiYMSvYNh/qFddPDUmL5PkxDi1MNZBgS4NU3Cvr1e9dKV61cUTLZsmcn2/dF5cs4MDR094XU5YUaBLw53Rq6656hJ6hfk5HK84xZyV2+vfOIYo0OX85F7mm9YImqsuIde9UwsG5bZhxvKtVJ5S51U1Bbqcvw49q3rVNVddQu/WghxK9x/jrU80OqqaAl0ap2WG70w9Y6B61SWkhvXoQKeWyVqirgYFujReSmuY9Dz0uA5e/YnmqktIJMTHMXFwFu9+9iWb9hz2upywoECXwKjuVR9yj3rVJWTGDcoiKT6O6ctKvC4lLCjQJXBOz1VXr7qERrvmTbimTyeeW1XKV+UnvS7Hcwp0CbzqXnXNVZcQuLUghyMnKnlulX7OFOgSHL1uglue11x1CbreGa3om9mK6cu2cirGWxgV6BI8p3vVq+aqb37H64okSt1akMPmL4/w7qYvvS7FUwp0Ca7Tverp6lWXoBnZqxPtmjdheozPd1GgS/BV96prrroESVJCHBMGZfLmp3vYVnbU63I8o0CX0EhpdWav+qs/UahLQE0ckk28WUy3MCrQJXSqe9UHfQ+W/REW/DOcqvS6KokSHVokM/ySjswt3s7RExVel+MJBbqEVlwcjHgYLv1X+GCabyWkSvUPS2AUFeRwqLyCF1bXXsc+NijQJfTM4IqfwhU/g4/mwZxb4ETsXveUwMnLbk2PTi2YsWwrXi2v6SUFunjn0vth5G9g499g+rVwROuKS+OYGZOGZPPJrq9Yvf2A1+WEnAJdvDXoDhg7A3Z9BE9cCfu2eF2RRLhr+3amaVI8s9/f5nUpIadAF+91HwWFL8HRMl+o71ztdUUSwZo3SWBU784s+PCLmJvvokCX8JA1BCa/Bgkp8NTVsOl1ryuSCDZ+cBbHTlby0pqdXpcSUgp0CR9pF/nuKm17AcwcC2tmel2RRKg+GS3p1jGVWTF22UWBLuEltSPcuhByvgkv3g1LHtENSNJgZsaEwVms23mIj0oPel1OyCjQJfwkt4AJ86D3WHjzIXj5X6AyNm8UkfM3um86yYlxzFoZO2fpCnQJTwlJcP1f4Jv/AquegjmT1KsuDdIyJZGre3XmpdU7YubDUQW6hC8zGPrgP3rVp41Sr7o0SGF+dkwtfqFAl/BX3au++2P1qkuD9MmMrcUvFOgSGdSrLucplha/UKBL5Kjdq/6ZetWlfrG0+IVfgW5mw83sUzPbZGZTvma7G83MmVle4EoUqaFmr/qssbD6Ga8rkjAXS4tf1BvoZhYPPAqMAHoA482sRx3bpQL/DKwIdJEiZ6jZq/7SPepVl3rFyuIX/pyhDwI2Oec2O+dOALOB0XVs90vgYaA8gPWJ1E296tIAsbL4hT+Bng5sr/G4tOq508ysP5DpnHvl697IzO40s2IzK967d2+DixU5Q+1e9bmaqy7nVr34xYuro3e+S6M/FDWzOOB3wL/Wt61z7jHnXJ5zLi8tLa2xuxY5s1f900Waqy7nVL34xfRlJVG7+IU/gb4DyKzxOKPquWqpwCXA22ZWAgwB5uuDUQmp2nPV95d4XZGEGTPj1oIcPtn1FSu27PO6nKDwJ9BXAl3NLNfMkoBxwPzqF51zB51z7ZxzOc65HGA5cK1zrjgoFYucS81e9cfVqy5nu7ZvZ1o1TWRalLYw1hvozrkK4F5gMbABmOucW2dmvzCza4NdoEiDZA2Bya9CQrJ61eUsyYnxjB2Yyavrd7PzwDGvywk4v66hO+cWOucucs5d6Jz7j6rnfuqcm1/Htpfr7Fw8lXZxjbnqY9SrLmeYNDgb5xzPrNjqdSkBpztFJTpV96rnXqpedTlDZpumXNG9A7Pe3075yUqvywkoBbpEL/WqyzncWpDDviMneGXtF16XElAKdIlu6lWXOhRc2JYu7ZszbVl0tTAq0CX6qVddajEzivKzWVt6kNXbD3hdTsAo0CV21O5V11z1mHZD/wxSmyRE1RRGBbrEFs1VlyrNmiRwU14Gr3z0BXu+io4RVAp0iT2ne9U1Vz3W3TIkm5OVjlkrtte/cQRQoEtsqtmrrrnqMeuCtOZ866I0nlmxlRMVp7wup9EU6BK7as9Vf0e96rHo1oIc9nx1nMXrdnldSqMp0CW21exVf+shePkH6lWPMd+6KI3stk2jYr6LAl3kjF71qTBnknrVY0hcnHHLkGyKt+7n4x0HvS6nURToInBmr/rGv8EzN8Pxr7yuSkLk5rxMUhLjI36JOgW6SE2D7oAbH4dty2D6aDganXOz5UwtUxK5oX86L63Zyf4jJ7wu57wp0EVq63XTP25AmjYKvtrtdUUSAoX5ORyvOMXslZHbwqhAF6lLt6thwhzYt9l3A9KXm7yuSILs4o6p5F/QlqeXb6XyVGR2OynQRc7lwu/ArS/DiSO+UN++0uuKJMiKCnLYceAYr2+IzN/KFOgiXyd9gO+u0uSWvssvnyz0uiIJoqHd25PeKiViWxgV6CL1aXshTH4N2neHOROh+EmvK5IgSYiPY+KQLJZ+XsZnuyOvy0mBLuKP5mm+yy9dhvoWynjjl7qrNEqNG5hFUkIc05aVeF1KgynQRfyV1AzGzYL+hfDub+DFe6DypNdVSYC1aZbE6D6def6DHRwqj6zvrwJdpCHiE2DUH+Dy/wsfzoSZY3UDUhQqKsjh6IlK5hWXel1KgyjQRRrKDC7/N7j2v2Hz2zD1avWqR5lL0lsyILs1M5aVcCqCWhgV6CLnq38hjJ8NX36mXvUoVJifTUnZUd75bK/XpfhNgS7SGBcNq9Wr/r7XFUmAjLikE2mpTSKqhVGBLtJY1b3qKa3Uqx5FkhLimDg4i7c/3UvJl0e8LscvCnSRQKjuVe/QU73qUWTCoCwS4ozpy7Z6XYpfFOgigdKsHRQtgC5X+nrV33xIveoRrn2LZEb26sS84u0cOR7+C58o0EUCKakZjJvp+8B0ySPw0j+pVz3CFRVk89XxCp5fvcPrUuqlQBcJtNO96j+GNc9U9aof9roqOU/9s1pzSXoLpi8twYX5b1wKdJFgMIPLp9ToVR+pXvUIZWYU5efw2Z7DLPu8zOtyvpZfgW5mw83sUzPbZGZT6nj9fjNbb2ZrzewNM8sOfKkiEeiMXvWhvv+ViDOqT2faNEtiapi3MNYb6GYWDzwKjAB6AOPNrEetzVYDec653sCzwK8DXahIxDrdq34UnhimXvUIlJwYz7iBmby+YTel+8N3AXF/ztAHAZucc5udcyeA2cDomhs4595yzlUf5XIgI7BlikS4s+aqv+J1RdJAk4b4Ljw8vXybx5Wcmz+Bng7UXGSvtOq5c5kMLKrrBTO708yKzax4797IuZ1WJCBOz1XvAXMmwconvK5IGqBzqxSG9ejI7JXbKD9Z6XU5dQroh6JmNgnIAx6p63Xn3GPOuTznXF5aWlogdy0SGWrOVX/lfs1VjzBFBTkcOHqS+R/u9LqUOvkT6DuAzBqPM6qeO4OZDQUeAK51zh0PTHkiUUhz1SPWkAvacHGHVKaFaQujP4G+EuhqZrlmlgSMA+bX3MDM+gF/wRfmewJfpkiUqdmrrrnqEcPMKCzIZt3OQ6zaut/rcs5Sb6A75yqAe4HFwAZgrnNunZn9wsyurdrsEaA5MM/M1pjZ/HO8nYhUO6tXXXPVI8H1/dJJTU4IyxbGBH82cs4tBBbWeu6nNb4eGuC6RGJH/0Jo3hHmFflG8E56Htp18boqOYemSQmMyctk2tISdh8qp0OLZK9LOk13ioqEA81VjyiF+dlUOsczK8KrhVGBLhIu0gfAd1+rMVddverhKrttM759cXtmrtjGiYpTXpdzmgJdJJy0uaDGXHX1qoezwvxsvjx8nEUff+F1Kacp0EXCTc256upVD1uXdU0jt12zsPpwVIEuEo5Oz1Uv8vWqv3QvVIb/AguxJC7OKMzPZvW2A6wtPeB1OYACXSR8xSfAqP+Cb02BNU/Dc5Oh4oTXVUkNNw7IoGlSPNOWhscSdQp0kXBmBt/+MQx7CNa/6LuufvKY11VJlRbJidzYP4MFa3dSdtj7G+QV6CKRoOA+uOb38Nmr8MzNcCw8fsUX3xJ1JypOMXvl9vo3DjIFukikyLsdbvgrbFsGT42EQ+E5ICrWdGmfyje6tOWZ5VupqPS2hVGBLhJJet8ME+fBgW3w+JWwZ4PXFQlQlJ/DzoPlvLbe29ENCnSRSHPhd+C2hXDqJDx5FZS853VFMe+K7h1Ib5XieQujAl0kEnXq7bsBqXkHmHEdrHvB64piWnyccUt+Niu27OOTXYc8q0OBLhKpWmfD7Yuhc3+Ydxss+5PXFcW0cQMzaZIQx/Rl3rUwKtBFIlnTNlD4InS/Bhb/GBY/AKfCZ7ZILGnVNInr+qbzwgc7OHjUmwVLFOgikS4xBW6eBoPuhGV/hOe/CxXe90THosKCbI6drGTeKm9aGBXoItEgLh5G/BqG/hw+fg6evhHKD3pdVczp2bklA3NaM33ZVipPhX7+jgJdJFqYwTd/ANc/5utVf3KEetU9UFSQw7Z9R3lnY+hX41Sgi0SbPmNh4rNVvepD1aseYlf17EiHFk2Y6sF8FwW6SDS68NtVveqV8IR61UMpMT6OiYOzWbJxL5/vPRzSfSvQRaJVp96+FZBS1aseauMHZZEUH8eMELcwKtBFolmrrDN71Zf/2euKYkJaahOu7t2JZ1eVcvh46ObYK9BFol11r3q3q+FvU+DVn6hXPQQK87M5fLyC5z8oDdk+FegisSAxBcZMh4F3wNL/Vq96CPTLak2fjJZMW1qCC9ESggp0kStNHhkAAAY2SURBVFgRFw8jH4GhD/6jV11z1YOqMD+Hz/ce4b1NZSHZnwJdJJaYwTf/pWqu+nJ4agQc3OF1VVHrmj6daNssKWRTGBXoIrGo95iquerb4QnNVQ+WJgnxjB+UxRuf7Gb7vqNB358CXSRWne5Vr6jqVf+71xVFpYlDsogz4+nlwW9hVKCLxLJOveG7r1f1ql8PHz/vdUVRp1PLFK7q2YHZK7dz7ERlUPelQBeJdTV71Z+9XXPVg6AoP4eDx07y0prgfl6hQBcRzVUPskG5bejWMZWpQW5hVKCLiI/mqgeNmVFUkMMnu75iZcn+oO3Hr0A3s+Fm9qmZbTKzKXW83sTM5lS9vsLMcgJdqIiEQF1z1Q94s1hDtLmubzotkhOYFsQWxnoD3czigUeBEUAPYLyZ9ai12WRgv3OuC/B74OFAFyoiIVI9V/2Gv0LpSvhjHrz5H3DiiNeVRbSUpHjGDszkb+t28cXBY0HZR4If2wwCNjnnNgOY2WxgNLC+xjajgQervn4W+KOZmQvV/a4iEni9x0BWPrz+ICz5Nax8HJq397qqiPajylPcnHCEjxffS6cx9wT8/f0J9HSg5u9cpcDgc23jnKsws4NAW+DLmhuZ2Z3AnQBZWVnnWbKIhEyrTLjpCd919eInoKLc64oiWhLQvuVJuvS7OCjv70+gB4xz7jHgMYC8vDydvYtEiqzBvj/SaK2C+N7+fCi6A8is8Tij6rk6tzGzBKAlEJppNCIiAvgX6CuBrmaWa2ZJwDhgfq1t5gNFVV/fBLyp6+ciIqFV7yWXqmvi9wKLgXjgSefcOjP7BVDsnJsPPAHMMLNNwD58oS8iIiHk1zV059xCYGGt535a4+ty4ObAliYiIg2hO0VFRKKEAl1EJEoo0EVEooQCXUQkSphX3YVmthc43yU82lHrLtQYoGOODTrm2NCYY852zqXV9YJngd4YZlbsnMvzuo5Q0jHHBh1zbAjWMeuSi4hIlFCgi4hEiUgN9Me8LsADOubYoGOODUE55oi8hi4iImeL1DN0ERGpRYEuIhIlwjrQY3Fxaj+O+X4zW29ma83sDTPL9qLOQKrvmGtsd6OZOTOL+BY3f47ZzMZUfa/XmdnMUNcYaH78bGeZ2Vtmtrrq53ukF3UGipk9aWZ7zOzjc7xuZvaHqv8ea82sf6N36pwLyz/4RvV+DlyAb+WmD4Eetba5B/ifqq/HAXO8rjsEx/xtoGnV13fHwjFXbZcKLAGWA3le1x2C73NXYDXQuupxe6/rDsExPwbcXfV1D6DE67obecyXAf2Bj8/x+khgEWDAEGBFY/cZzmfopxends6dAKoXp65pNDCt6utngSvMzEJYY6DVe8zOubecc0erHi7Ht4JUJPPn+wzwS+BhIBoWtfTnmO8AHnXO7Qdwzu0JcY2B5s8xO6BF1dctgZ0hrC/gnHNL8K0PcS6jgenOZznQysw6NWaf4RzodS1OnX6ubZxzFUD14tSRyp9jrmkyvv+Hj2T1HnPVr6KZzrlXQllYEPnzfb4IuMjM3jOz5WY2PGTVBYc/x/wgMMnMSvGtv3BfaErzTEP/vdcrpItES+CY2SQgD/iW17UEk5nFAb8DbvW4lFBLwHfZ5XJ8v4UtMbNezrkDnlYVXOOBqc6535pZPr5V0C5xzp3yurBIEc5n6LG4OLU/x4yZDQUeAK51zh0PUW3BUt8xpwKXAG+bWQm+a43zI/yDUX++z6XAfOfcSefcFmAjvoCPVP4c82RgLoBzbhmQjG+IVbTy6997Q4RzoMfi4tT1HrOZ9QP+gi/MI/26KtRzzM65g865ds65HOdcDr7PDa51zhV7U25A+POz/SK+s3PMrB2+SzCbQ1lkgPlzzNuAKwDMrDu+QN8b0ipDaz5QWNXtMgQ46Jz7olHv6PUnwfV8SjwS35nJ58ADVc/9At8/aPB9w+cBm4D3gQu8rjkEx/w6sBtYU/Vnvtc1B/uYa237NhHe5eLn99nwXWpaD3wEjPO65hAccw/gPXwdMGuAYV7X3MjjnQV8AZzE9xvXZOAu4K4a3+NHq/57fBSIn2vd+i8iEiXC+ZKLiIg0gAJdRCRKKNBFRKKEAl1EJEoo0EVEooQCXUQkSijQRUSixP8CsXM/s4M3VP4AAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Putting everything together\n"
      ],
      "metadata": {
        "id": "YsGrG8riqY4a"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "plt.plot(df_scores.threshold, df_scores['tpr'], label='TPR', color='black')\n",
        "plt.plot(df_scores.threshold, df_scores['fpr'], label='FPR', color='blue')\n",
        "\n",
        "plt.plot(df_ideal.threshold, df_ideal['tpr'], label='TPR ideal')\n",
        "plt.plot(df_ideal.threshold, df_ideal['fpr'], label='FPR ideal')\n",
        "\n",
        "# plt.plot(df_rand.threshold, df_rand['tpr'], label='TPR random', color='grey')\n",
        "# plt.plot(df_rand.threshold, df_rand['fpr'], label='FPR random', color='grey')\n",
        "\n",
        "plt.legend()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "opl6POV4qZ_j",
        "outputId": "d3d3f980-faf3-4879-f245-b0719a876ea2"
      },
      "execution_count": 145,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f0c104f97d0>"
            ]
          },
          "metadata": {},
          "execution_count": 145
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXyMVxfA8d/NHkHsQkSC2IkgQhBr1S7UvjQJqlRR1Wopb6lq0VJFldpp7fsSqT1SaolUGlvtscSWoog9cd8/HtHQkGBmnpnJ/X7efNI8M+ae5I3jzn3OPVdIKVEURVEsn43eASiKoiiGoRK6oiiKlVAJXVEUxUqohK4oimIlVEJXFEWxEnZ6DZwnTx7p5eWl1/CKoigWKTo6+m8pZd60HtMtoXt5ebFv3z69hlcURbFIQogzz3tMLbkoiqJYCZXQFUVRrIRK6IqiKFZCJXRFURQroRK6oiiKlUg3oQshZgkhrgghDj7ncSGEmCiEOCGEiBVCVDJ8mIqiKEp6MjJDnwM0esHjjYHijz/eBaa8fliKoijKy0q3Dl1KGSmE8HrBU4KAeVLrw7tbCJFDCFFASnnRQDE+Zdy4VbieWc3eYkHkzZ+fnDlzYiOEMYZSlEwvm5M9nasVJouDbltWlJdgiP+X3IFzqb4+//jafxK6EOJdtFk8hQsXfqXBHM5u4p1cK8iZcIWBF3qQzN9o6Vyk/I9U/6EoymuQElb/Gc+M4Cq4uTrpHY6SDpP+syulnAZMA/Dz83ulkzWKNZzMZ6OK8HX9L6mc/SZTE6oQe+gvDh06xNmzZ588L2vWrDRr1oz27dvTqFEjnJzUL6OivKytf12m74L9BE3ewYzgKpQv5Kp3SMoLGKLKJR7wSPV1ocfXjKJ4cRi142P25BqN151YRlc4w/pVSzlz5gw3btxg9+7dzJw5k44dO7J582ZatWpFvnz56NKlC2vWrOHevXvGCk1RrE69UvlZ3rs6djY2tP3pd04mJOodkvIChkjoa4Dgx9Uu1YAbxlo/B/DyAltbWHvlPWgxCU5uhTnNIDGB7NmzU7VqVbp168a0adO4ePEiGzdupF27doSHhxMUFET+/PkJDg5m3bp13L9/31hhKorVKOWWnRW9q/NIwuydp/UOR3mBjJQtLgR2ASWFEOeFEN2FEL2EEL0eP2U9cAo4AUwHehstWsDeXkvqx48DlYKh/S9w5QjMbABXTz71XDs7Oxo0aMCMGTO4dOkSv/76K61bt2bt2rU0b94cNzc3Fi9ebMxwFcUq5M/uRFCFgqz4I56b9x7qHY7yHOkmdCllRyllASmlvZSykJRyppRyqpRy6uPHpZTyfSllMSlleSml0VsoFi/+OKEDlGoKIWvg3g2Y+Sacj07zz9jb29OwYUNmzZrF5cuXCQsLo3Tp0nTo0IERI0agDstWlBcLqe7FnQfJLN13Xu9QlOewyJ2ixYvDiRPaHXgAPPyh+yZwcIG5zeDYhhf+eQcHB5o0acK2bdsIDg5m2LBhdOrUiejoaO7cuWP8b0BRLFA5d1cqe+bk511xPHqkJkDmyCITurc33LoFV66kupjHW0vqeYrDwo7wx7x0X8fR0ZE5c+YwatQoFi1ahJ+fH1mzZsXb25s+ffoQGRlJcnKy8b4RRbEwwQGexF29w/bjCXqHoqTBIhN68eLa5yfLLimy5YfQMChaB9b0hYjRqabxaRNCMGjQIE6dOsWyZcsYPnw45cqVY9asWdSuXRsPDw+++uorbt26ZYxvRVEsSuNyBcibzZF5v8fpHYqSBotO6CdOpPGgYzbotBh8O0PEKFjbD5KT0n3NIkWK0Lp1az7//HNWrVrFlStXWLRoEb6+vgwdOhQvLy9GjRrFsWPH1KxdybQc7GzoXLUwEccSiPv7tt7hKM+wyITu6amVLv5nhp7C1h6CJkOtgdrSy6JO8ODlfvmyZs1K+/btWb9+PXv37qVatWp89tlnlCxZEhcXFypUqEDt2rWpXbs2derUYebMma//jSmKBejkXxhbIZi367knoSk6sciEbm8PRYq8IKEDCAH1hkKz8XBi05Na9VdRpUoVwsLCOHjwILNnz6ZPnz4UKlQIGxsbbGxsuHr1Ku+88w4ffPABSUnpvxtQFEuWL7sTTcoXYGn0OW7fV7/v5sRiO+6kVLqky68bZHWDZd20WvUuyyF3sVcas2zZspQtW/Y/15OTk/nkk0/47rvvOHbsGIsWLcLVVW2RVqxXSHUv1vx5gZX74+lSzVPvcJTHLHKGDlqly/Hj6d7z1JRq8nStenzateqvytbWlnHjxjFt2jQ2b95MjRo1OH1a7ahTrFelwjko556debvi1B4OM2KxCb14cUhMhMuXM/gHUteqz0m/Vv1V9OjRgw0bNhAfH4+/vz87d+40+BiKYg6EEIQEeHHsciK7Tl7VOxzlMYtO6JDBZZcUr1Cr/rLq1avHnj17yJkzJ/Xq1WP69OlqBqNYpeYVCpIziz1zVAmj2bDYhO7trX1+4Y3RtLxCrfrLKlGiBLt376ZWrVq8++67NG3alPh4ozWgVBRdONnb0sG/MJuPXOb8dbXD2hxYbEL38gI7u1dI6PBvrXqFTi9Vq/4ycuXKxYYNG5g4cSIRERGULVuWefPmqdm6YlW6VPNEAov2nkv3uYrxWWxCt7PTShdfasklNVt7aPljqlr1ji9dq54eGxsb+vbtS2xsLOXKlSMkJISWLVty6dIlg46jKHpxz+FMnRJ5WbLvHEnJj/QOJ9Oz2IQO/1a6vLKnatU3v1at+ot4e3uzfft2xo0bx8aNGylbtixfffUVYWFhxMXF8eiR+ougWK4O/oW5cus+W/+6kv6TFaOy6ISe0kb3tVcx/LpB+/nP7atuCLa2tgwYMID9+/dTunRphg4dSrNmzShSpAju7u707duXHTt2qOSuWJx6pfKRL5sji6LUsoveLDqhlygBt2/DhQsGeLFna9Wf01f9tYcpVYodO3Zw7do1duzYwdSpU6levTozZswgMDAQf39/rl5VZWCK5bC3taGtXyEijl7hwj939Q4nU7PohO7jo33+808DveBL9lV/HTlz5qRGjRr07NmT5cuXc+XKFWbMmMHBgwd54403uHbtmtHGVhRD61ClMI8kLNmnZul6suiEXqGC9nn/fgO+6LO16tFzDfjiz5ctWza6d+/O6tWrOXLkCG+88QbXr183ydiK8ro8cmUhsHgelkSdI1kdfqEbi07o2bNDsWIQE2PgF05dq762H2wbZfBa9edp2LAhK1eu5NChQ9SrV0/VrysWo0OVwly4cY/tx9TNUb1YdEIHqFjRwDP0FKn7qm8fbZRa9edp3Lgxq1ev5sSJE/j7+xMdbZz1fEUxpAZl8muHX6i2urqx+ITu6wsnT8KNG0Z4cQP0VX9VjRo1YufOndjZ2REYGMjChQvVpiTFrDnY2dDJvzARRxM4rQ6/0IXFJ/SKFbXPsbFGGsCAfdVflo+PD3v37qVChQp06tSJgIAANmzYoBK7YrY6Vy2MnY1g3q44vUPJlCw+ofv6ap+NsuySmglq1dOSP39+IiMjmTZtGhcvXqRRo0Y0aNCA27fVDEgxPymHXyzbd14dfqEDi0/oBQpAvnxGuDGallJNIGSt0WvVn2Vvb0+PHj04fvw4EyZMYOvWrQQHB6tNSIpZCqnuya37SazYr27om5rFJ3QhjHhjNC0eVUxWq/4sBwcH+vXrx7hx41ixYgWfffaZycZWlIyqVDindvjF7+rwC1Oz+IQO2rLLoUPw4IGJBtSpVj1F//796dWrF2PGjGHGjBkmHVtR0pNy+MXxK+rwC1OzioResSI8fAiHD5tw0Gdr1bd/a7JadSEEEydO5M0336RHjx40bdqUffv2mWRsRcmI5hUKksvFQR1+YWJWkdBTboyaZB09tZRadZ/2sG0kbB5msqRub2/PypUr+frrr9m9ezdVqlShZcuWnDuntl4r+nOyt6VDFQ91+IWJWUVCL14cXFxMuI6emq09tJyqVcHsnADrB4KJblZmyZKFwYMHc/r0aUaOHMnmzZspV64cs2fPVmuXiu46V/ME4JfdZ3WOJPOwioRuY6P1ddEloacE0PQ7qN4XoqbDql6QZKoFfciePTtDhgwhNjaWihUr0q1bN1q0aMGdO2pmpOjHPYczb5ZxY1HUWe49TNY7nEzBKhI6aMsuMTEmmxz/lxDQ4EttE1LsYljQFu7dNGkIRYsWZevWrYwfP56wsDBCQkJUaaOiq5DqXvxz5yFrYgzR41pJT4YSuhCikRDiqBDihBBiUBqPFxZCbBNC7BdCxAohmhg+1BerVAlu3XqNI+kMQQitTUDQj3D6N5jTBG5eNGkINjY29O/fn2+//ZZly5bxv//9z6TjK0pq1YrmomT+bMxRJYwmkW5CF0LYApOBxkAZoKMQoswzTxsKLJFSVgQ6AD8aOtD01KihfY6MNPXIaajYGTovgauntA1ICcdMHsKAAQPo0aMHX3/9NXPmzDH5+IoCWkVWcHVPDl+8SfQZ1Q7a2DIyQ/cHTkgpT0kpHwCLgKBnniOB7I//2xUw+furkiUhf37Yvt3UIz+H9xvQNQyS7sKsN+HsbpMOL4Rg8uTJ1K9fn3fffZe5c01bK68oKVpVdCebkx1zVRdGo8tIQncHUtfCnX98LbXhQBchxHlgPdA3rRcSQrwrhNgnhNiXkGDYBldCQK1aWkI3m3d2BStqG5Ccc8G8IDiy1qTD29vbs3z5cgIDAwkNDWXw4MFqTV0xuSwOdrTz8yD8wEUu37yndzhWzVA3RTsCc6SUhYAmwM9CiP+8tpRympTST0rplzdvXgMN/a9ateDcOYiLM/hLv7pcRbSk7lYeFr8Ne6ebdHhXV1d+/fVX3n33XUaPHk1QUBALFy4kNjaW+/fvmzQWJfMKDvAkWUrm71EljMaUkYQeD3ik+rrQ42updQeWAEgpdwFOQB5DBPgyatfWPpvNsksKl9wQvAZKNob1H8Pm4SZ9G2Fvb8/UqVMZP348GzdupFOnTlSoUIE8efKwY8cOk8WhZF6euV2oWzIfC/ac5UGSepdoLBlJ6FFAcSFEESGEA9pNzzXPPOcsUB9ACFEaLaGbpml4KmXLQq5cZpjQARyyQLufoXJX2DEeVvY0aa26EIL+/ftz8+ZNYmNjWbhwIW5ubnTo0IG///7bZHEomVdwgCd/J94n/KBpK78yk3QTupQyCegDbACOoFWzHBJCjBBCtHj8tI+AHkKIP4GFQKjUoUbJxubfdXSzZGunHZShY626o6Mj5cuXp0OHDixZsoSEhARCQ0PV2rpidLWK56VIHhfV38WIMrSGLqVcL6UsIaUsJqX86vG1z6WUax7/92EpZQ0pZQUppa+UcqMxg36R2rXh9GltLd0smUGteoqKFSvy3XffERYWxvjx43WJQck8bGwEwQGe7D/7D7Hn/9E7HKtkNTtFU5jtOvqzKnaGTim16g0g4aguYfTu3Zu33nqLQYMGMW3aNLX5QzGqNpUL4eJgy9zfVQmjMVhdQvfxAVdXC0joAMVTatXvaRuQTFyrDtra+syZM6lduzY9e/akUaNGqmOjYjTZnOx5q1Ih1sZe4GqiqrIyNKtL6La2EBhoIQkd/q1Vz5Jbl1p1gBw5crBx40YmT57Mjh07KFeunKp+UYwmpLonD5IesShKTRwMzeoSOmjLLsePwwVL6Qekc606aD1gevfuzYEDB3Bzc6Nly5acPGmag7CVzMU7XzZqeOdm/u4zJCWrm/GGZJUJvUED7XNYmL5xvBSda9VTFC1alLCwMKSUNGvWjOvXVf8NxfBCAry4cOMem49c1jsUq2KVCd3HBzw9YfVqvSN5Sf+pVTdtX/UU3t7erFy5kpMnT9KmTRsemOywViWzqF86P+45nFUJo4FZZUIXAoKCYPNmSEzUO5qX9FSt+iJdatUBatWqxYwZM9i6dSvVqlXjwIEDJo9BsV62NoK3AzzZfeoaf10y/e+3tbLKhA7QsiXcvw8bdauIfw2pa9XjduhWqx4cHMzKlSuJj4+ncuXKfP3119y9e9fkcSjWqb2fB452NqqE0YCsNqEHBkLOnBa47JJaxc7aIdQ61qq3bNmSgwcP0qpVK4YMGUK+fPno0qULa9as4d491TlPeXU5XRxo6evOqv3x3LjzUO9wrILVJnQ7O2jaFNatg6QkvaN5Dd5vQNf1kHRft1r1vHnzsnjxYrZu3UqHDh0IDw8nKCiI/PnzExwczLp161TnRuWVBFf35O7DZJZGqxJGQ7DahA7aOvq1a7Bzp96RvKaCvvDOJnDJo1utOkDdunWZPn06ly5dIjw8nDZt2rB27VqaN29O/vz5CQ0NZf369eomqpJhZQu6UsUrJ/N2nSH5kdql/LqsOqE3bAgODha+7JIipxd026hrrXoKe3t7GjVqxMyZM7l8+TJhYWG0atWKVatW0bRpU9zc3OjevTsbNmwgyaLfHimmEFLdi7PX7rD92BW9Q7F4Vp3Qs2WD+vW1hG4VLUqerVXfNAx07pLo4OBAkyZNmD17NpcvX2bt2rU0a9aMpUuX0qhRI+rWrava8yov1LCsG/mzOzJH3Rx9bVad0EFbdjl1CqKj9Y7EQFLXqu/8HlbpU6ueFkdHR5o1a8a8efO4cuUKM2bMYN++ffj7+3P48GG9w1PMlL2tDZ2rehJ5LIGTCZZWZ2xerD6ht2+vVbsMH653JAZkBn3V0+Pk5ET37t2JiIjgzp07BAQEMGXKFDVbV9LU0b8wDrY2/KwOkn4tVp/Qc+SATz/V2gBY/M3R1J7tqz5bv77qL1K1alWioqIoWbIkvXv3xs3NjYYNG7J582a9Q1PMSN5sjjT1KcCy6PMk3lf3XV6V1Sd0gD59wM0NBg+2krX01FL6ql/Tt6/6i3h4eLBnzx7279/PJ598wtGjR2nYsCETJkxQ/deVJ4IDPEm8n8SKP87rHYrFyhQJ3cUFhg6F336z0J2j6TGDvurpEULg6+vL119/zaFDh2jRogX9+/end+/ePHyoNpUoULFwTioUcmXu73HqH/pXlCkSOkCPHuDlBZ99pnthiHGk7qs+twUcfvYcb/Ph4uLC8uXLGTRoEFOnTqVjx47qL7ACaCWMJxNus+OEutfyKjJNQndwgC++gD/+gHnz9I7GSFL6qhfwgSXBsGea3hE9l42NDaNGjWLMmDEsX76cSZMm6R2SYgaa+hQgt4uD6u/yijJNQgfo0gVq1IAPP4SL5nf/0DBS16qHDzSLWvUXGThwIM2bN+fjjz9m3759eoej6MzRzpaO/oXZ8tdlzl27o3c4FidTJXQbG5g1C+7dg/fes8IbpClSatX9uv1bq55snuvUQgjmzJmDm5sb7du358aNG3qHpOisc7XC2AjBz7vVLP1lZaqEDlCiBIwYoe0eXbJE72iMyNYOmn73b6364rfhoXl2R8yVKxcLFy7kzJkzNGnShN27ze+mrmI6BVydaVg2P4ujznH3QbLe4ViUTJfQQVtyqVJFK2e8Ys3tI1Jq1ZuMhWPhsKAd3DfPnXg1atRg9uzZHDt2jICAAJo0aUJUVJTeYSk6CQnw4sbdh6yOidc7FIuSKRO6nZ229JKYCB07Wnh73Yzw7wEtp0Dcb/DLW3DXPM8Jffvttzl9+jSjR49m7969+Pv707x5c6Ktpm+DklH+RXJRyi0bc1QJ40vJlAkdoFw5mDIFtm7VShmtnm8naDMb4v+AWY3ghnlu3siaNSuffvopp0+f5uuvv2bnzp34+fnRokUL9u/fr3d4iokIIQip7sVfl26x9/Q1vcOxGJk2oQOEhmo3R7/9FpYu1TsaEyjbErosh5sXYEYDuHxI74ieK1u2bAwePJi4uDi+/PJLfvvtNypVqkSrVq2IiopSs7ZMoKWvO67O9szdFad3KBYjUyd0gO+/h2rVoGtXOGp+u+YNr2ht6BoOSG2mfjpS74heKHv27AwdOpS4uDiGDx/Otm3b8Pf3p0SJEgwZMoRjx47pHaJiJM4OtrSv4sGGQ5e5eEOdZZsRmT6hOzjAsmXauvqAAXpHYyJu5bQNSNkLwi+t4cAyvSNKl6urK8OGDSMuLo7p06fj5eXF6NGj8fX1VWvsVuztap48kpL5u8/qHYpFyPQJHcDdXev1sn69lfZ6SUsOD+j2K7j7wfLu8PsPekeUITly5OCdd95h06ZNxMXFkTdvXpo3b8758+Z5T0B5PR65slC/VD4W7j3L/SRVwpieDCV0IUQjIcRRIcQJIcSg5zynnRDisBDikBBigWHDNL6+faFIEfjoI0jOLL83zjnh7ZVQJgg2DoFfB5v1rtJneXh4sG7dOhITE2nevDmJieZZkqm8npDqXly9/YCwWGvd3m046SZ0IYQtMBloDJQBOgohyjzznOLAYKCGlLIs0N8IsRqVoyN88w0cPKiVNGYa9k7QZg5UfQ92/wjLuprtBqS0lC9fnsWLFxMbG0v79u25e1ettVqbmt55KJrXhbm/x+kditnLyAzdHzghpTwlpXwALAKCnnlOD2CylPI6gJTSIrfrtG6t9XoZOhRu3dI7GhOysYFGo+DNkXB4lVnXqqelcePG/Pjjj4SHh1O3bl0uXbqkd0iKAQkhCAnw4s/zN4g594/e4Zi1jCR0d+Bcqq/PP76WWgmghBBipxBitxCiUVovJIR4VwixTwixLyEh4dUiNiIhYPx4bffo99/rHY2JCQHV+0LrmXBur1nXqqelZ8+eLF++nAMHDuDv78+ff/6pd0iKAbWuXIisjnZqlp4OQ90UtQOKA3WAjsB0IUSOZ58kpZwmpfSTUvrlzZvXQEMbVpUq0KyZltAz5ZJs+TapatXfgEsH9Y4ow1q1asWOHTt49OgRNWrUYO3atXqHpBhIVkc72lQuxLrYCyTcuq93OGYrIwk9HvBI9XWhx9dSOw+skVI+lFKeBo6hJXiL9NlncO0aTJ+udyQ6eVKrLmB2Y7OvVU+tYsWK7N27l1KlShEUFMTYsWPVJiQr8XaAJw+TJQv3qhLG58lIQo8CigshigghHIAOwLPH4axCm50jhMiDtgRzyoBxmlRAANStC2PHwv3MOhlwKwfvbILs7vDzWxZRq56iYMGCREZG0rp1awYOHEiPHj1IsvqGPdavWN6sBBbPw/w9Z3iYbDnVWKaUbkKXUiYBfYANwBFgiZTykBBihBCixeOnbQCuCiEOA9uAgVLKq8YK2hQ++wwuXLDi040ywrUQdAsHD//HteqTLKaJfJYsWVi8eDFDhgxh5syZhIaGkpxp6lGtV2h1Ly7fvM+GQ+rGd1qEXm9H/fz8pDmfUCMlVK2qLb389Ze2kzTTengPVvbUKmCqvgcNv9YqYyzEqFGj+Oyzz3j77beZPXs2tra2eoekvKLkR5I6Y7dRILszS3oF6B2OLoQQ0VJKv7Qes5y/lSYmhDZLP3kSfvlF72h0Zu+kdWqs+h7smWJxteqDBw/myy+/5Oeff6ZTp06EhYVx5swZtbZugWxtBMHVvNgbd43DF27qHY7ZUQn9BVq00NbTP/oILl/WOxqd2dhA49Hw5lcWWas+dOhQvvzyS5YuXUqzZs3w8vIiX758zJs3TyV2C9POzwMnextVwpgGldBfIOUM0tu3tdONFKB6H61W/XyUxdWqDx06lKtXr7Jjxw6mTp1KiRIlCAkJoWXLlmozkgVxzWJPq4rurIqJ5587D/QOx6yohJ6OUqVg+HCtI+Myyyn0MK6natXNu6/6s3LmzEmNGjXo2bMnkZGRjBs3jo0bN1KmTBnGjBmj+sFYiOAAL+4nPWJx1Ln0n5yJqISeAR9/DJUrw/vvw1WLrt0xoCK1tG6NYBF91dNia2vLgAEDiImJoWrVqgwaNIgiRYrw7bffcvv2bb3DU16gdIHs+BfJxc+7z5D8SC2ZpVAJPQNSziC9dg0GD9Y7GjOSv+zjWnXL6auelpIlSxIeHs7vv/9OpUqV+OSTTyhatCjfffcdd+7c0Ts85TlCq3tx/vpdtv1lka2jjEIl9Azy8dHW0WfOhNhYvaMxI66FtJl6oSoWV6v+rICAADZs2MDOnTvx8fHho48+olixYkyYMEF1cTRDb5bJTwFXJ3VEXSoqob+E//0PXF21qhcLzVnG4ZwTuqyAMi1h41CL66v+rOrVq7Np0yYiIyMpXbo0/fv3p1ixYkyaNEltTjIjdrY2dK5amN+O/82JK+reB6iE/lJy5YJhw2DzZggP1zsaM5NSq16tt0XWqqclMDCQrVu3EhERQYkSJejXrx+fffaZ3mEpqXTwL4yDrQ3zdsXpHYpZUAn9Jb33HhQvrs3SHz7UOxoz86SvumXWqj9P7dq1iYiIoGfPnnzzzTeEq3/NzUaerI40q1CA5dHnuXVP/YVUCf0lOTjAt99q7QAmTNA7GjOVUqtugX3VX2T8+PH4+PgQHBxMfPyzDUcVvYRW9+L2g2SWR1vH79nrUAn9FbRoAS1bwqBBsGmT3tGYqfJt4O0VFtlX/XmcnZ1ZvHgxd+/epVOnTjx4oDa1mAOfQjnw9cjBvF1neJTJSxhVQn8FQmhdGEuXhnbt4NgxvSMyU09q1R/3VT+1Xe+IXlupUqWYMmUKkZGRuLm50b17dzZs2KDaB+gstLoXp/6+zW8n/tY7FF2phP6KsmWDtWu1GvXmzeG65S8VG8eTWnV3i65VT+3tt99mw4YNNGvWjKVLl9KoUSNGjBihd1iZWpPyBciT1ZF5mby/i0ror8HLC1auhNOntZukynOk1KpbYF/153nzzTeZN28eV65coUOHDowcOZKYmBi9w8q0HOxs6OTvwdajVzh7NfNuBlMJ/TXVrKm1BJg3T0vsynM453i6Vn3jUItP6gBOTk5MnjyZ3Llz07VrVx6q0ifddK7mia0QmbqEUSV0Axg4UFt6GTVK70jMXEqtun9P2PUDrP0AHln+Rp1cuXIxZcoUYmJiGDNmjN7hZFr5szvRqJwbS/ad486DzHnkoEroBlCwIPToAXPmwJkzekdj5mxsoPEYCPwI/pirnYSUbPmz2latWtG+fXtGjBjBr7/+qm6S6iSkuhc37yWxcn/mLCtVCd1APv1Uq35RE7QMEALqfw71h8GBpbD4bVeUihcAACAASURBVHhg+euekyZNolChQjRu3Jg6deoQERGhd0iZjp9nTsoUyM7PuzLniVQqoRtIoULQrZvWvOu82t+QMYEDoMlYOPYrzGsBty27N3HevHk5fPgwEydO5Pjx49StW5e+fftmysSiFyEEXap58telW+w/94/e4ZicSugGNGiQ1pPqiy/0jsSC+PeA9j/DpQMwswFcs+w7y05OTvTt25eTJ0/ywQcf8MMPP/DBBx+opG5CLXwLksXBlkV7z+odismphG5Anp7Qvz/MmAFbt+odjQUp3RyCV8Odq1pSv7Bf74hem7OzM+PHj2fAgAFMmjSJDz/8UCV1E8nqaEdzn4Ks/fNipuvvohK6gY0YoTXveucd7SxSJYMKV4Pum8DOGWY3hROb9Y7otQkhGDt2LB988AETJkzgvffeU2WNJtKxamHuPkxmdcwFvUMxKZXQDczZWVtHP30ahgzROxoLk7eEtqs0d1FY0B5iFugd0WsTQjB+/HgGDRrETz/9ROPGjbmuthUbXYVCrpRyy8bCTLbsohK6EQQGaqcbTZwIO3fqHY2FyeYGoevBqyaseg8iv7X4DUhCCEaNGsXs2bOJjIwkICCAkydP6h2WVRNC0KlqYQ5duMmB8zf0DsdkVEI3klGjtDX19u1V1ctLc8oOnZaCT3vYOhLWfQjJlr9RJDQ0lM2bN5OQkECLFi3UQdRGFuTrjpO9DQujMs8sXSV0I8maFdasgVu3oEkTuJF5JgmGYecArX6Cmh9C9GxY3MUqatVr1arF4sWLOXLkCP369dM7HKvm6mxP0/IFWb0/PtPcHFUJ3YjKl4fly+HIEWjTBlT77JckBLwx/N9a9bnNLb5WHeCNN95gyJAhzJo1i19++UXvcKxacIBnpjr8QiV0I3vjDa2McfNmbeOROmP4FaTUql8+aBW16gDDhg0jMDCQXr16cfToUb3DsVoVPDLX4RcqoZtASIi2pj5/vlbO+OiR3hFZICurVbezs2PBggU4OjrywQcf6B2OVctMh1+ohG4igwZpO0jnzIF331VJ/ZU8W6t+3LJr1QsVKsRHH33Ehg0biI2N1Tscq5WZDr/IUEIXQjQSQhwVQpwQQgx6wfNaCyGkEMLPcCFaj88/h6FDtTr1Pn0svhpPH6lr1Re2h/3z9Y7otbz33nu4uLgwduxYvUOxWpnp8It0E7oQwhaYDDQGygAdhRBl0nheNuADYI+hg7QmI0bAxx/DlCmqM+MrS12rvrq3Rdeq58yZk3feeYeFCxdy7tw5vcOxWpnl8IuMzND9gRNSylNSygfAIiAojed9CYwB7hkwPquT0mK3QwcYPBgWLtQ7IgtlRbXqKX1eJkyYoHcoViuzHH6RkYTuDqSeOpx/fO0JIUQlwENKGfaiFxJCvCuE2CeE2JeQkPDSwVoLGxttLb1WLQgNhe3b9Y7IQj1bq77EMvuqe3p60q5dO6ZNm8YNtWHBaFIOv1i133r7u7z2TVEhhA3wHZDuMclSymlSSj8ppV/evHlfd2iL5uioHTBdtCgEBcGff+odkYVKXat+NNxi+6oPHDiQW7duqbV0I0o5/GLerjir7XyZkYQeD3ik+rrQ42spsgHlgAghRBxQDVijboymL1cu2LABsmWDhg3hxAm9I7Jgz/ZVvx6nd0QvpWLFirRr146RI0cycOBAktWGBYMTQhBa3Yu/Lt1iz+lreodjFBlJ6FFAcSFEESGEA9ABWJPyoJTyhpQyj5TSS0rpBewGWkgp9xklYitTuDBs2qRtOGrQAOIz51GIhpG6Vn2G5dWqz58/n/fff5+xY8fy1ltvkZiYqHdIVqeFb0FyZLFnrpWWMKab0KWUSUAfYANwBFgipTwkhBghhGhh7AAzg1KlIDwc/v5b21mqkvprKFwNum8EOyeLq1W3s7Pjhx9+4IcffiAsLAw/Pz92796td1hWxcnelvZVPNh4+DIX/rmrdzgGl6E1dCnleillCSllMSnlV4+vfS6lXJPGc+uo2fnL8/OD9eu1zoy1akFcnN4RWbC8JVP1VW9ncbXq77//Phs3buTOnTvUqFGDQYMGce+eKh4zlC5VPZFSMn/PGb1DMTi1U9SMBAbCli1w/TrUrAmqxcdrSKlVLxJokbXq9erV48CBA3Tr1o0xY8bQsGFDkpKst9zOlDxyZaF+6fws3HuOew+t616FSuhmxt8fIiLg4UOoUUOVNL4WC69Vd3V1Zfr06cyaNYvIyEiGDRumd0hWI7S6F9duPyAs9qLeoRiUSuhmyMdHO+koTx5tTX3aNL0jsmBWUKvetWtXunfvzqhRo9i4caPe4ViF6sVy450vK3N3WVcJo0roZsrbG3bv1hJ6z57wwQeqodcrs4Ja9YkTJ1KmTBm6dOnCxYvWNavUgxCCkABPYs/fYP+5f/QOx2BUQjdjOXLAunXQv792PmloKKhl1NfwbK26BfVVz5IlC0uWLCExMZHOnTurOnUDeKtSIbI52llVF0aV0M2crS189x2MHAk//6z1gFEnH70GC+6rXqZMGSZPnsy2bdv46quv9A7H4rk42tHGrxBhBy5y5ZZ1VBGphG4BhIAhQ2D8eO1Iu7fe0m6aKq/oSa265fVVDw0NpUuXLnzxxRdEREToHY7Fe7uaJw+TJQv3WEenS5XQLUj//lrb3bAwGDBA72gsXOpadQvqqy6E4Mcff8Tb25tOnTqRmZvcGULRvFmpXSIv8/ec4UGS5d+kUgndwvTqBR99BD/8AFOn6h2NhXu2r/p2y6hVz5YtG0uWLOHatWt06NBBbTp6TaHVvbhy6z4bDl3SO5TXphK6BRozBpo0gb59Yds2vaOxcKlr1beNhHX9LaJWvUKFCkybNo2tW7fSunVr7t+/r3dIFqt2ibx45s5iFf1dVEK3QLa22sEYJUpAixbw7beg/j6/hqdq1efA4i4WUaseHBzMtGnTWL9+PW3atFFJ/RXZ2AjerubJvjPXORhv2f3oVUK3UNmzw6+/Qu3a8MknULYsrFplESsG5il1rfqxX2F+W7h/S++o0tWjRw+mTp3KunXraN26terQ+Ira+nngbG9r8UfUqYRuwTw8tDr1X3/VDsxo1Qrq11eHZbwW/x7Qegac3QXzguCO+ffN7tmzJ1OnTiU8PJyaNWuqs0lfgauzPW9Vcmd1zAWu37bcumCV0K1Aw4ZaEv/hB+1zpUrw3ntwx/xXDcxT+Tb/bkCa2xxuXdY7onT17NmTdevWcerUKfz9/dm7d6/eIVmc4AAv7ic9YlGU5f6DqBK6lbCzg/ff10496tcPfvpJ696oJmuvqFRT6LQYrp3SNiD9bf7HSTVu3Jhdu3bh5OREgwYN1Ez9JZV0y0ZA0dz8svsMyY8sc+1S6NWYxs/PT+7b93Tb9IcPH3L+/HlVhvUanJycKFSoEJs22dOhA2TJoq2tV6umd2QWKj4a5rcD+Qg6LQGPKnpHlK6TJ0/i4+ND7dq1CQsLQwihd0gW49eDl+j1SzQ/vV2ZhmXd9A4nTUKIaCll2kd8Sil1+ahcubJ81qlTp2RCQoJ89OjRfx5T0vfo0SOZkJAgT506JaWU8tAhKYsVk9LeXsrhw6W8d0/nAC3V3yek/L6ClF/ml/JImN7RZMj3338vATlnzhy9Q7EoD5OSZfVRW2THabv0DuW5gH3yOXnVrJZc7t27R+7cudWM4hUJIcidO/eTdzhlysCePdCmDQwfDhUrwo4d+sZokXIXg+6bIF9pWNwZ9s3SO6J09e3blxo1atC/f38uXLigdzgWw87Whs7VCvP7yascv2z+VU7PMquEDqhk/pqe/fnlzg0LFmjH2925ox1v9/HHoFa1XlLWvBC6Drzf0A7K2PKlWdeI2tjYMGvWLO7du0fv3r31DseidKhSGAc7G+buitM7lJdmdgldMY7GjeHgQa11wLhx2slIsbF6R2VhHFygw0KoFAy/jYVVvSHZfLuklShRgi+++ILVq1ezZcsWvcOxGLlcHAiqUJAVf8Rz8575/v+bFpXQU7l69Sq+vr74+vri5uaGu7v7k6+FEPj6+lKuXDnatm3Lncc1gba2tk+uN2/enH/+Md9m+Vmzwo8/as29rlyBKlW02bvyEmztoPlEqPMZ/LkAFrQ36w1I/fr1o3Dhwnz66ac8UiekZFhIdS/uPEhm6b7zeofyUlRCTyV37tzExMQQExNDr169+PDDD5987eLiQkxMDAcPHsTBwYGpjztjOTs7P7meK1cuJk+erPN3kb4mTeDAAQgIgM6dtdYBZrx6YH6EgDqfQotJcCoC5jQ121p1JycnRo4cSXR0NEuXLtU7HItRzt2Vyp45+XlXHI8sqITRTu8Anqd///7ExMQY9DV9fX35/vvvX/t1AgMDiU1jvSIgICDN6+Yob17YsAGCg7XWAefPawdp2NrqHZkFqRQMWd1gaYhWq95lBeTx1juq/+jUqRNjx45lyJAhtGrVCgcHB71DsgjBAZ58sCiG7ccTqFsyn97hZIiaob+kpKQkwsPDKV++/FPXk5OT2bJlCy1atNApspfn6Kg1+RowQDvirmFDbSlGeQkl3tRulj64rSX1c+a3Q9PW1pbRo0dz8uRJpk+frnc4FqNxuQLkzeZoUV0YzXaGboiZtCHdvXsXX19fQJuhd+/e/anr8fHxlC5dmgYNGugZ5kuzsdFukpYrB717g68vLF6s7TJVMsi9snYC0vw2WquANrOhVBO9o3pKo0aNqFOnDsOGDePNN9+kePHieodk9hzsbOhctTDfbz5O3N+38crjondI6VIz9AxKWSuPiYlh0qRJT962plw/c+YMUkqLWENPS9eusHu3duO0bl3o2RPOntU7KguSUquev6xZ1qoLIZgyZQo2NjbUrVuXEyfMv5WBOejkXxg7G8G8XWf0DiVDVEI3kCxZsjBx4kTGjRtHUpL5H5CQlgoVYN8+rbRx9mwoXlzrD3PsmN6RWQiXPBCyFrwbaLXqW0ea1d3mUqVKsWXLFu7fv0/dunU5efKk3iGZvXzZnWhSvgBL953j9n3z/3utEroBVaxYER8fHxYuXKh3KK8se3ata+OJE9qsfdo0KFkSataEmTPhsnkWc5gPBxfosEC7YRr5Lax+36xq1cuXL8+WLVu4e/cu9erVU/3TMyCkuie37iexYn+83qGky6yacx05coTSpUvrEo81MeTP8dIl+PlnmDUL/vpLu1amjLYs06SJ1n/d0dEgQ1kXKWH7GIgYBcXqQ7t54JhV76ie2LFjB4GBgYwZM4ZPPvlE73DMmpSS5j/s4P7DR2z8sJbuu9lf1JxLzdCVF3Jzg4ED4fBhbTlm9GjtYI05c6BpU638sWNHbf1dSUUIqDMoVa16E7OqVa9ZsyYNGzZk7Nix3L59W+9wzJoQgpAAL45fSWTXyat6h/NCGUroQohGQoijQogTQohBaTw+QAhxWAgRK4TYIoTwNHyoip6EgMqV4dNPtROSrl7Vdpy2awebNmmblLp0Uf3X/6NSMHRcBH8fh5lvaJ/NxLBhw0hISHiySU55vuYVCpLLxYE5Zl7CmG5CF0LYApOBxkAZoKMQoswzT9sP+EkpfYBlwDeGDlQxL46O2pLLjBkQFwdDh8Ly5dp6+zvvaM3A1JnFjz2pVb8DM980m1r1gIAAGjRowDfffPOklYWSNid7WzpU8WDzkcucv26+P6uMzND9gRNSylNSygfAIiAo9ROklNuklCnf5W6gkGHDVMxZ1qzw5ZfaGnv79rBkyb/LMSEhsH27WRV76COlVt3JVatV/ytM74gA+Pzzz7ly5Qo//fST3qGYvS7VtIWHX3abbz1vRhK6O5D6jfT5x9eepzsQntYDQoh3hRD7hBD7EhISMh6lYhE8PbVyx4QEbYberp12WlKdOuDtrbUYCA+HW+bby8q4nvRVLwOLu0DUTL0jombNmtSrV49vvvmGu3fv6h2OWSuYw5k3y7ixKOos9x4m6x1Omgx6U1QI0QXwA75N63Ep5TQppZ+U0i9v3ryGHFoxI46OWrveGTPg4kWtSqZIEfj+e22ZJmdOCA2FTFkxl7qvetgAs+ir/r///Y9Lly4xb948XeOwBCHVvfjnzkPW/Gmeh4ZkJKHHAx6pvi70+NpThBBvAEOAFlJKi109TWmHm/IRFxdHREQErq6u+Pr6Urp0ab744guAp66XKlWKjz/+WOfozU+WLNrN0s2b4Z9/tBuo77+vJXl/f616JtMxs77qtWvXxs/Pj3HjxpGcbJ4zT3NRrWguSubPxtzf49Cr5PuFnnc2XcoHWr+XU0ARwAH4Eyj7zHMqAieB4um9XspHWmeKHj582GDn7r0qFxeX/1zbtm2bbNq0qZRSysTEROnt7S2jo6Ofun7nzh1ZsmRJuWPHDpPGmxZz+DmmZ8sWKfPlkzJLFiknTJAyMVHviHTw6JGU20ZJOSy7lPNaSXnvpm6hLF68WAJyxYoVusVgKX7ZHSc9P10no05f1WV8XnCmaLrNuaSUSUKIPsAGwBaYJaU8JIQY8fiF16AtsWQFlj4uuj8rpXyttoP9+4OBu+fi66u97X8dLi4uVK5cmRMnTpAv378tNZ2dnZ806VLSV68e7N8Pb78NH3wAX3yh9Y/p10+rfc8UUmrVsxeEtf21vuqdlkK2/CYP5a233qJIkSJ8++23tGrVyuTjW5JWFd0ZHf4Xc36Pw88rl97hPCVDa+hSyvVSyhJSymJSyq8eX/v8cTJHSvmGlDK/lNL38Yfl9JB9Rkr3RF9f3zR/sa9evcru3bspW7bsU9evX7/O8ePHqVWrlqlCtXgFC2pLMb/9pt04HT1a6/r46696R2ZiT9WqN4C/Td84y87OjgEDBrBr1y527txp8vEtSRYHO9r5efDrwUtcvmlmh/M+b+pu7A9LW3LJnj279PX1lZUqVZJTpkx56rqPj490dnaWgwcPNnW4aTKHn+OrOHxYSh8fKYWQcuhQKZOS9I7IxM7vk3JMUSlHe0l5do/Jh09MTJS5c+eWLVq0MPnYlibu70TpNWidHLfxqMnH5gVLLmrrfwYFBgayf/9+oqOj6dWr11PX//zzTw4dOsTMmTMNfspSZlK6tNZCoGtXGDkSqlWDpUvBQptXvjz3yvDOJnDOoUutuouLC++//z5r1qxh5cqVJh3b0njmdqFuyXws2HOWB0nmc1arSugGUqRIEQYNGsSYMWP0DsWiOTtrXR1/+QWuX9dq2b294aOPYNQomDpVW6IxxwIDg8hVNFVfddPXqg8cOJCAgADatWvH6tWrTTq2pQkO8OTvxPuEH7yodyhPqIRuQL169SIyMpK4uDi9Q7F4nTvD0aOwciV4eWktfT/7DN57D2rVgtq1ISJC7yiNJHVfdRPXqmfNmpXw8HAqV65M27ZtWbt2rUnGtUS1iuelSB4Xs+rvotrnWiFr/Tneu6fN2lesgK+/hgsXtIZhVatCxYpakreqk9WSk7SE/sdc8O0CzSeArWlOjbxx4wYNGjQgJiaGqKgoKlSoYJJxLc3snaf5Yu1h1vSpgU+hHCYZU7XPVayCkxMUKKBtTDpxQitBzZJF26TUo4fWp33aNL2jNCBbOy2J1x4EMb/A8u6Q9MAkQ7u6uhIeHo6rqyt9+vQxz000ZqB15UJkcbBl7u/mcUSdSuiKRXJ21urXIyO1HajHjkGDBlote9++VnQjVQioOxjeHAmHV2nr6g9N03Mld+7cjB49mh07djB//nyTjGlpsjvZ07pSIdbGXuBqov4b5FVCVyyejY221LJ2rXbz9IcftOR+9KjekRlQ9b7QbDwc3wjz28Ldf0wybNeuXalSpQoDBw7k5s2bJhnT0oRU9+RB0iMWRel/GIBK6IrVsLWFsWO105Sio7VNSh9+qK27WwW/bvDWdDi7C2Y3gZvGbxBlY2PD5MmTuXz58pMeRsrTvPNlo4Z3bubvPkNSsr4ljCqhK1YnJASOH4du3WDiRK2tb6dOsGyZFXR49GkLnZfCP2dhRgO4csToQ1apUoXu3bszYcIEIiMjjT6eJQoJ8OLCjXtsOqzvMYMqoStWKX9++Okn+OMPaNtW6/LYti3ky6f1j9m2DR6Zz36Ql1OsHnRdD48ewqyGEGf8rfqjR4+mePHiNGnShB07dhh9PEtTv3R+3HM4617CqBJ6KlevXn3Sx8XNzQ13d/cnXwsh8PX1pVy5crRt2/bJkV0p7XbLlStH8+bN+eeftNc2q1evnub10NBQli1b9krxDh8+nLFjx77Sn80sKlTQNipdvKjVrQcHw5o1WnOwIkW06pgFC7THLUoBH20DUtb88HNLOGTcnZ25c+dm69atuLu707hxY37//XejjmdpbG0Ebwd4suf0Nf66pN+9BpXQU8mdOzcxMTHExMTQq1cvPvzwwydfu7i4EBMTw8GDB3FwcHhysK6zs/OT67ly5WLy5Mlpvrb6C6AvOzutTn3qVLh0CebP17pvLl2qbWIqVAh69YIrV/SO9CXk9IRuG6BgJVjaFXb9aNThChQowLZt2yhQoACNGjUiNjbWqONZmg5VPHC0s2HeLv1KGE2zS+EVfLH2EIcvGPZfujIFszOsedn0n5iOwMDANH+ZAwICnvtLnjVrVhITE5FS0rdvXzZt2oSHhwcODg5PnhMdHc2AAQNITEwkT548zJkzhwIFCjB9+nSmTZvGgwcP8Pb25ueffyZLliyv/X1kVs7O2pp6p06QnKy1aZ47F6ZMgYULYfBgbTeql5fWytfGnKc9WXJB8CpY0QM2DIab8dDgS6MFXbBgQbZt20aVKlVo164d+/btI2vWrEYZy9LkyOJAS193Vv4Rz6cNS+Gaxd7kMZjzr6pZSkpKIjw8nPLlyz91PTk5mS1bttCixYs7B69cuZKjR49y+PBh5s2b92Tm/vDhQ/r27cuyZcuIjo6mW7duDBkyBNB6VUdFRfHnn39SunRpZs7U/yxKa2Frq+02nTgRDhyAmjW1hF6jBri7Q7Zs2gx+yxYzXnO3d4a2c8H/Xdj1A6x4B5KMVxPt7u7OggULOH78OO+//77RxrFEwdU9ufswmaXR+pQwmu0M3RAzaUNK6ZMO2gy9e/fuT12Pj4+ndOnSNGjQ4IWvExkZSceOHbG1taVgwYLUq1cPgKNHj3Lw4MEnfz45OZkCBQoAcPDgQYYOHco///xDYmIiDRs2NNa3mamVKgVhYVqFzIkTcPo0/PknLFmirbN7eGgtBry8oGhRbR2+XDlt74/ubGyh8TeQ3R02D4PEK9BhPji5GmW4OnXq8PnnnzN8+HDq1q1LaGioUcaxNGULulLFKyfzdp2ha40i2NqY9pfDbBO6uUlZK3/e9Tt37tCwYUMmT55Mv379Xvr1pZSULVuWXbt2/eex0NBQVq1aRYUKFZgzZw4RVtuVyjwUL/50T5gJE2DVKm29/fhx2Lr13/LHYsXgrbege3coWVKfeJ8QAmr2h2wFYHVvmNUYuizTTkQygqFDhxIREcH777+Pvb09HTp0wNbW1ihjWZKQ6l70WbCf7ceuUK+UaU+fUksuBpIlSxYmTpzIuHHjSHrBvvNatWqxePFikpOTuXjxItu2bQOgZMmSJCQkPEnoDx8+5NChQwDcunWLAgUK8PDhQ7UFWwdOTtChAyxfDrGxcPMmnD+v3WD19tZ6ypQqBc2amcnSTIX20HnZ41r1N4xWq25ra8uCBQsoXrw4Xbp0oVy5cixcuJBHuv8A9NWwrBv5szsyR4f+LiqhG1DFihXx8fFh4cKFz31Oq1atKF68OGXKlCE4OJiAgAAAHBwcWLZsGZ9++ikVKlTA19f3yfr6l19+SdWqValRowalSpUyyfeiPJ8Q2vp6z57acXnnz8Pw4bB3L7zxBmTPrq3B9+kDUVE6BVms7uNa9WSYabxa9QIFCvDHH3+wZMkSbG1t6dSpE+3ateP27dtGGc8S2Nva0LmqJ5HHEjiZYOKdbM87ysjYH+Z6BJ01UD9Hfdy9K+X8+VL26ydlYKCULi5SgpTBwVLGx+sU1PUzUk7yk3JEHikPrjDqUMnJyXLs2LFSCCErVaokz58/b9TxzNmVm/dk8c/Wy2GrDxr8tVFH0CmK8Tk5aaWQEyZoXSAvXIBPP4VFi7Q1+f79tSUbk8pR+Ola9d1TjDaUjY0NH330EWvWrOHYsWP4+/tn2lr1vNkcaepTgGXR50m8b7rWnyqhK4qRZM8Oo0fDkSMQFAQ//qjtXPXzg4EDta/Xr4erV40cSEqteqmm8Osg2DjUqAv9zZo1Y+fOnQghqF+/PgcOHDDaWOYsOMCTxPtJrPjjvMnGVAldUYysaFGt7PHCBW32LgRMmqQd1NG0KRQsqPWZWb/eiH3c7Z2h3Tyo0gN+n2T0WnUfHx8iIiJwdHSkfv36T27wZyYVC+ekQiFX5v4eZ7IDQlRCVxQTyZMH+vXTbpTeuaMl+MhI6N1b6zPTtCnkzg3Nm8N338GePdrzDMbGFpp8C28Mh4PL4ZfWRu2r7u3tzbZt27Czs6NevXo8e+RkZhAc4MXJhNvsPGHst2EaldAVRQc2NtpxeoGBMH48xMdrZ6V26KAdzPHRR1CtmrZTtWxZbTYfFWWAs6KFgJofPu6rvhtmN4Yb8Qb5ntJSvHhxtm3bhoODA9WqVWPYsGE8eGCaY/TMQbMKBcjt4mCyLowqoSuKGXBwgFattJa/x45ppZArVsCQIVpXyFmzwN8ffHy0Esk1a+DcuddI8D7tHvdVPwczjdtXvWTJksTGxtK5c2dGjBiBv78/a9as4f59/Y9sMzZHO1s6+hdmy1+XOXfNkG+30qYS+jNS2uGmfMTFxREREYGrqyu+vr6ULl36ycktqa+XKlWKjz/+OM3X3Ldv33N3j3p5efH333+/Uqx16tTJlG9jMwN3dy3BjxgB69Zp7X2nTgUXF+1aUBAULqz1d2/QAD75RFunP3JEaziWIU9q1ZMe16obr895zpw5mTt3fsKRrwAACyFJREFULqtXr+bKlSsEBQWRL18+goODOXbsmNHGNQedqxXGRgh+2W2CjUbPq2c09oe51qG7uLj859q2bdtk06ZNpZRSJiYmSm9vbxkdHf3U9Tt37siSJUvKHTt2vNR4np6eMiEh4ZVirV27toyKivrPdXP4OSrGc/OmlDt2SDlpkpTdu0tZqZKUDg5azTtI6ewspZ+flG3bSjlwoJQ//STlnj1S3rnznBdMXat+YLnR43/w4IEMDw+XXbt2ldmyZZMeHh7y4sWLRh9XT+/9sk/6DN8g79xPeu3X4gV16ObbyyV8EFwycLmTW3loPPq1XsLFxYXKlStz4sQJ8uXL9+S6s7PzkyZdz4qIiGDs2LGsW7eOq1ev0rFjR+Lj4wkICHjq7vcvv/zCxIkTefDgAVWrVuXHH3/E1taW9957j6ioKO7evUubNm3U2Y6ZXLZs2k7UGjX+vfbwoTY7379f+zh8WGsLvHo1pCxZ29pqFTe5ckGOHFp1TdOm0KhRYVy6bYCFHWFZN7h1CQJ6Gy1+e3t7GjVqRKNGjejTpw+BgYEEBQWxbds2q20LHRLgxfoDl1gdE08H/8JGG0ctuTwjpXuir68vrVq1+s/jV69eZffu3ZQt+3Q3yOvXr3P8+HFq1ar1wtf/4osvqFmzJocOHaJVq1acPXsWgCNHjrB48WJ27txJTEwMtra2T/q2fPXVV+zbt4/Y2Fi2b9+eaTdrKM9nb6+tr4eEaL1lNm7U1uLv3oWTJ7U+NIMHQ6VK4OqqHZy9ejW0aaNV37TqnIs1OVfxqGQzra/6hiEmaUpTqVIlFixYQFRUFCEhIVbbB8a/SC5KuWVjjpFLGM13hv6aM+lX9byuir/99hsVK1bExsaGQYMGUbZsWSIiIvjtt9+oUKECx48fp3///ri5ub3w9SMjI1mxYgUATZs2JWfOnABs2bKF6OhoqlSpAmj/sKS8A1iyZAnTpk0jKSmJixcvcvjwYXx8fAz5bStWysZGm5UXLap1hUwtKQl++w1WrtQ6Sa5a5Uz+fHNZ0nUQtXb9wM34i2TtMgUbB0ejxhgUFMS3337Lxx9/jI+Pz5MjHWvVqkW1atWwMesTRjJGCEFIdS8GrzhAVNx1/IvkMso4GUroQohGwATAFpghpRz9zOOOwDygMnAVaC+ljDNsqPoKDAxk3bp1z71++vRpqlWrRrt27Z70TX8ZUkpCQkIYNWrUU9dPnz7N2LFjiYqKImfOnISGhnLv3r1X/j4UJYWdHdStq3189x1s2ACzZ9vSYPw3fODnzjcNhrG91xW+jJ2CY34PPD21OvkcObSPnDn/+9/Zs2tLOy9rwIAB2NvbEx4ezvbt25+8Oy1UqBBt27alW7dulCtXzsA/AdNq6evOqPVHmPt7nH4JXQhhC0wGGgDngSghxBop5eFUT+sOXJdSegshOgBjgPbGCNhcFSlShEGDBjFmzJgXdlusVasWCxYsYOjQoYSHh3P9+nUA6tevT1BQEB9++CH58uXj2rVr3Lp1i5s3b+Li4oKrqyuXL18mPDycOnXqmOi7UjILOzttPb1pU3jwQHD4cH8idxSkuuxLWCE/5h7vx5fL+xOf4JJuqWT27P8meQ8P7UAQLy/ImzftfwhcXLQZbL9+/Z5Ug12/fp2wsDCWLFnC5MmTGT9+PO3atWPYsGGUKVPG6D8PY3B2sKV9FQ9m7Yzj4o27FHB1NvgYGZmh+wMnpJSnAIQQi4AgIHVCDwKGP/7vZcAPQgghjblYZIZ69erF2LFjiYuLw8vLK83nDBs2jI4dO1K2bFmqV69O4cLaDZIyZcowcuRI3nzzTR49eoS9vT2TJ0+mWrVqVKxYkf+3d3chUtVhHMe/P3W3ITINTYjc1TUUFSOzAZWgF4wQL5TIZA17AWtRsRuvBCPEIDCoi0ioJSMLCq2LWKgILEWQtAzNt8VwS21L0ra0JHDHnaeLc6pxXZ3jvJwzZ+b5wGHPmfPfmeeZM/Psmf//7PynTp1KS0sL9xaOhDlXBc3NwQTazFwC5+bC9vV0DHuZjplvYTeNI58PLo3MDwQ/B/KXrxfuy+WgPwf5HqDn8sf5M1wQDB8WnNkXzv40B5hzB9iquxgYGGBg4Bj2+uMcrYkpokrzyHDjsdH9bHmpnbUbX6j4/atYzZW0GJhvZs+E208As81sdUGbw2Gb3nC7J2zz26D76gA6AFpbW+85efLy6zK7u7uZNm1a2Uk1On8eXcWd2gv7NsOl0rr7+nPQfzEs8P3/F/pcf/gzXL/WmGg+n+fChQvkcrkSk6gRzcZfExey9PkVJf26pG/NLDvUvlgHRc2sE+gEyGazDXX27lyqtc4OlhI1h4urrijDxz8DLQXb48PbhmwjaQQwimBw1DnnXEyiFPRvgMmS2iQ1A+1A16A2XcBT4fpi4MtS+88brNu94vz5c65xFS3oZnYJWA18DnQD28zsiKQNkhaGzTYDYyQdB9YAa0sJJpPJ0NfX50WpRGZGX18fmUwm6VCccwkoOihaLdls1gZ/sVQul6O3t9evsy5DJpNh/PjxNDU1JR2Kc64KamZQtJimpiba2tqSDsM551Ip/f9T65xzDvCC7pxzdcMLunPO1YnEBkUlnQVKncJjLFDaND/p5Tk3Bs+5MZST8wQzu3WoHYkV9HJI2ne1Ud565Tk3Bs+5MVQrZ+9ycc65OuEF3Tnn6kRaC3pn0gEkwHNuDJ5zY6hKzqnsQ3fOOXeltJ6hO+ecG8QLunPO1YmaLuiS5ks6Jum4pCu+wVHSDZK2hvv3SpoYf5SVFSHnNZKOSjoo6QtJE5KIs5KK5VzQ7lFJJin1l7hFyVnSkvBYH5H0ftwxVlqE13arpB2S9oev7wVJxFkpkt6WdCac0W2o/ZL0Wvh8HJQ0q+wHNbOaXIDhBLMQTiKY7OQ7YPqgNquAN8L1dmBr0nHHkPODwI3h+spGyDlsNxLYBewBsknHHcNxngzsB24Jt8clHXcMOXcCK8P16cCJpOMuM+f7gFnA4avsXwB8BohgCtW95T5mLZ+h/zc5tZn1A/9OTl1oEbAlXP8ImCeleAbZCDmb2Q4z+zvc3EMwg1SaRTnOAC8CG4F6+G7lKDk/C2wysz8AzOxMzDFWWpScDbg5XB8F/BJjfBVnZruA36/RZBHwrgX2AKMl3VbOY9ZyQb8d+Klguze8bcg2FkzEcR4YE0t01REl50LLCf7Cp1nRnMOPoi1m9kmcgVVRlOM8BZgiabekPZLmxxZddUTJeT2wTFIv8CnwXDyhJeZ63+9F1dT3obvoJC0DssD9ScdSTZKGAa8CTyccStxGEHS7PEDwKWyXpDvN7FyiUVXXUuAdM3tF0lzgPUkzzCyfdGBpUctn6I04OXWUnJH0ELAOWGhmF2OKrVqK5TwSmAHslHSCoK+xK+UDo1GOcy/QZWY5M/sR+J6gwKdVlJyXA9sAzOwrIEPwJVb1KtL7/XrUckGPdXLqGlE0Z0l3A28SFPO096tCkZzN7LyZjTWziWY2kWDcYKGZ7Rv67lIhymv7Y4KzcySNJeiC+SHOICssSs6ngHkAkqYRFPSzsUYZry7gyfBqlznAeTM7XdY9Jj0SXGSUeAHBmUkPsC68bQPBGxqCA/4hcBz4GpiUdMwx5Lwd+BU4EC5dScdc7ZwHtd1Jyq9yiXicRdDVdBQ4BLQnHXMMOU8HdhNcAXMAeDjpmMvM9wPgNJAj+MS1HFgBrCg4xpvC5+NQJV7X/q//zjlXJ2q5y8U559x18ILunHN1wgu6c87VCS/ozjlXJ7ygO+dcnfCC7pxzdcILunPO1Yl/ABoAxJ3k5yDbAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(5, 5))\n",
        "\n",
        "plt.plot(df_scores.fpr, df_scores.tpr, label='Model')\n",
        "plt.plot([0, 1], [0, 1], label='Random', linestyle='--')\n",
        "\n",
        "plt.xlabel('FPR')\n",
        "plt.ylabel('TPR')\n",
        "\n",
        "plt.legend()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 352
        },
        "id": "TW43h8xnqs_y",
        "outputId": "07d2540d-4ac0-4175-ef39-39f3374ddcd3"
      },
      "execution_count": 146,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f0c104a4150>"
            ]
          },
          "metadata": {},
          "execution_count": 146
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 360x360 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUoAAAE9CAYAAABtDit8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU1d3H8c/JvgfIwhYgQfYdDKuCIEoR2VREUHYVq3Wp1qqtdam11Wrbp48WH4uVVVbFBRVUFBBKEQiy7wEChCX7vk/mPH/cSCOSBTIzd2bu7/168WLuzM3M7ybkyzn33HuO0lojhBCiZj5mFyCEEO5OglIIIeogQSmEEHWQoBRCiDpIUAohRB0kKIUQog5+ZhdwpaKjo3V8fLzZZQghvMzOnTsztdYxl3vN44IyPj6epKQks8sQQngZpdSpml6TrrcQQtRBglIIIeogQSmEEHXwuHOUl1NRUUFqaiqlpaVml+IRgoKCiIuLw9/f3+xShPAIXhGUqamphIeHEx8fj1LK7HLcmtaarKwsUlNTSUhIMLscITyCV3S9S0tLiYqKkpCsB6UUUVFR0voW4gp4RVACEpJXQL5XQlwZpwWlUmqeUipdKbW/hteVUuoNpVSyUmqvUqqPs2pxBaUUU6ZMubhts9mIiYlh9OjRV/Q+8fHxZGZmNngfIYTjOLNFuQAYWcvrtwDtq/7MBv7PibU4XWhoKPv376ekpASAdevW0bJlS5OrEkI4gtMGc7TWm5RS8bXsMg5YpI0p1r9TSjVSSjXXWp93Vk3ONmrUKD7//HMmTJjAsmXLmDx5Mps3bwYgOzubWbNmceLECUJCQpg7dy49evQgKyuLyZMnc/bsWQYOHEj1Geffe+893njjDcrLy+nfvz9vvfUWvr6+Zh2eEC6htaaiUlNRaaei0k55pd3Ytv13u9xmv7hPeaW96jVju7K8mFLtz9heLQkPcsyVHWaOercEzlTbTq167idBqZSajdHqpHXr1i4p7mpMmjSJl156idGjR7N3715mzZp1MShfeOEFevfuzccff8z69euZNm0au3fv5ve//z3XX389zz//PJ9//jnvvvsuAIcOHWLFihVs2bIFf39/HnroIZYsWcK0adPMPEThxbTWlNnslJRXUlJRSZnNTpmtkrIK++Uf2+yUXdyvptf/+7j8YtD9NwQrbJdsV4Xi1QqnmIUBr/KdvQuZ7d72iqCsN631XGAuQGJiYq3fxd9/eoCD5/Id+vldWkTwwpiude7Xo0cPUlJSWLZsGaNGjfrRa//+979ZtWoVADfeeCNZWVnk5+ezadMmPvzwQwBuvfVWGjduDMA333zDzp076du3LwAlJSXExsY68rCEl9BaU1BmIz2/lLT8MtILSknPL6OwzHYx9C7+XVFJcXklpVXPXXxc9achS2j5+SgC/XwI9Pc1/vbzIdDPl4CqxwF+PoQF+eHv60OArw/+vgp/Xx/8/S7Z9jX2/dG2rw/+fpds//A1VV8fWJFHqzVTCMw8RdtbnyWscbDDvsdmBuVZoFW17biq5zza2LFjefLJJ9m4cSNZWVlX/T5aa6ZPn84rr7ziwOqEp9Facy6vlDPZxaTlGwGYll9KWkFZ1bYRjiUVlT/5WqUgxN+X4ABfgvx9CQnwJdjfeBwVGkBw4x8/HxzgZ/zt73Pxa34IOyMAfQjw9SXQ3+cyz/vg52viRTTlRTDvbsg6Ane9R6OOtQ2PXDkzg3I18LBSajnQH8hzxPnJ+rT8nGnWrFk0atSI7t27s3HjxovPDx48mCVLlvDcc8+xceNGoqOjiYiIYMiQISxdupTf/e53rF27lpycHACGDx/OuHHjePzxx4mNjSU7O5uCggLatGlj0pEJZ8ssLOPohQKOpBVwNK2AIxcKOJZWSEGZ7Uf7Bfv70iwyiJjwQLrHNeKm8ECaRgQRG2H83TTCeC00wNc6l4L5h0D7EXDTi9DuJoe/vdOCUim1DBgKRCulUoEXAH8ArfXbwBpgFJAMFAMznVWLK8XFxfHoo4/+5PkXX3yRWbNm0aNHD0JCQli4cCFgnLucPHkyXbt2ZdCgQRfPwXbp0oWXX36ZESNGYLfb8ff3Z86cORKUXiKvpIK9qbnsOp3L7jO57E3NJbOw/OLrjUP86dA0nNv7tKRDs3Dio0KrQjCQsEA/6wRgXfLPQ1k+xHSE4c877WOUp63rnZiYqC+dj/LQoUN07tzZpIo8k3zPXO9CXilvrD/GthNZHM8ouvh8u9gwesY1okuLCDo2DadDszBiwgIlDOuSlwoLxwAKfrEdfBvW7lNK7dRaJ17uNY8YzBHCk9ntmmU7TvPqmsOUV9q5vl00t/VuSa9WjenRKpIIB43MWkrOKVg4GkpyYcqqBodkXSQohXCik5lFPLNqL9tOZjOwbRSv3tGdNlGhZpfl2bKOw8KxUF4I0z6Bls6/qU+CUggH0FpTWGYjo6CMzMJyMgrKOHwhn7mbThDg58Of7+jOxMRW0p12hI2vgq0Epn8KzXu45CMlKIWoJ1ulndScEo5nFHI8o5Dk9EJOZBRxIb+UzMIySivsP/man3VtykvjutE0IsiEir3UmL9D/jmIbu+yj5SgFOIytNbsSc1j/eF0jqUVcDyjkJTMYsor/xuG0WEBtI0OI7FNY2LCA4kJDyQ6LPDi49jwIJqEBph4FF7kwj5Y/0e44x0IDHdpSIIEpRAXVdo1O1Ky+WL/Bb48cIHzeaX4KIiPCqVtTBjDOsVyTUxY1Z9QGoVICLrEuV2waDwEhEJRphGULiZB6SC+vr50794dm81GQkICixcvplGjRg1+3wULFpCUlMQ//vEPB1QpSsorySwsI6OwjIyCsqpzimWk5pSw8Ug6mYXlBPr5MKRDDE+O6MhNnZsSGSKj0qY5swPeuwOCI41zko3jTSlDgtJBgoOD2b17NwDTp09nzpw5PPvssyZXZT15xRV8uCuVFTvOcDq7+Eev2bW+7HlEgKjQAAZeE8XIbs0Y1jGW0ED51TDd6W3w3u0QGmOEZKNWdX+Nk8i/BicYOHAge/fuBWD79u089thjlJaWEhwczPz58+nYsSMLFixg9erVFBcXc/z4cW677TZee+01AObPn88rr7xCo0aN6NmzJ4GBgQCkpKQwa9YsMjMziYmJYf78+bRu3ZoZM2YQHBzMrl27SE9PZ968eSxatIitW7fSv39/FixYYNa3wiW01uw6k8vSbaf5bO85Sivs9IyL5O5+rak+yKyUIjLY/+I5xJiq84lNQgPwN/M+ZXF5YTHQ8lq47W2IaGFuLVprj/pz7bXX6ksdPHjwJ8+5WmhoqNZaa5vNpidMmKDXrl2rtdY6Ly9PV1RUaK21Xrdunb799tu11lrPnz9fJyQk6NzcXF1SUqJbt26tT58+rc+dO6dbtWql09PTdVlZmR40aJD+xS9+obXWevTo0XrBggVaa63fffddPW7cOK211tOnT9d33XWXttvt+uOPP9bh4eF67969urKyUvfp00fv2rXrJ/W6w/esofJLyvWi/5zUP/ufb3Wbpz/TXZ5bq3/z4V69LzXX7NJEQ6Qd1Npud/nHAkm6htzxzhbl/Ft/+lzX8dDvfigvhiV3/vT1XndD73ugKAtWXjLn48zP6/zIkpISevXqxdmzZ+ncuTM333wzAHl5eUyfPp1jx46hlKKiouLi1wwfPpzIyEjAuLf71KlTZGZmMnToUGJiYgC46667OHr0KABbt269OCXb1KlTeeqppy6+15gxY1BK0b17d5o2bUr37t2Nw+7alZSUFHr16lXnMXiKjIIy5mxIZmXSGYrLK+nSPII/3taNcb1aEiZdZs929CtYMQWG/Rau/6XZ1Vwk/6oc5IdzlMXFxfzsZz9jzpw5PProozz33HMMGzaMjz76iJSUFIYOHXrxa37oUoMxGGSz2S7zzvXzw3v5+Pj86H19fHwa9L7upLDMxtxNJ/jX5hOU2eyM79WSqQPb0DMuUi7k9gaHPoP3Z0DTrtDHvSao9s6grK0FGBBS++uhUfVqQdYkJCSEN954g/Hjx/PQQw+Rl5d3ce2c+pwr7N+/P4899hhZWVlERETw/vvv07NnTwAGDRrE8uXLmTp1KkuWLGHw4MFXXacnKbfZWbrtFG+uTyarqJxR3Zvx5IiOtI0JM7s04SgHPoJV90HzXsa928ENv2LEkbwzKE3Wu3dvevTowbJly3jqqaeYPn06L7/8MrfeeplTApdo3rw5L774IgMHDqRRo0Y/6jK/+eabzJw5k9dff/3iYI4301rz2d7zvP7lEU5nFzOgbRPevaUzvVq51y+RaKDCdPjoQYjrC3evhKAIsyv6CZlmzaLc/Xu2NzWXlz49SNKpHDo3j+DpkR25oUOMdLG9VcoWaN4TAs3rJcg0a8JjpOeX8tqXR/hgZyrRYQG8ent37kxsha+PBKTXSZpvzEze8y6Iv87samolQSncgt2uWfzdKV774jAVlZoHbmjLw8PaOWwVPeFmts2Ftb+GjqOgx0Rw856CBKUwXXJ6AU+v2sfOUzkMbh/NH8Z1Iz5a5mz0Wv95E776HXQaDRPmu31IghcFpdZazl/Vk9nnpW2Vds7llnIyq4jtJ7N4Z9NJQgJ9+dvEntzWu6X8HL3Z5r/CNy9Bl/Fwx7/A1zN6DF4RlEFBQWRlZREVFSW/ZHXQWpOVlUVQkOvmRywqs7H5WCZfH0rj+1M5nMkp/tEi96N7NOfFsV2JDgus5V2EV6goge4TYfz/OX35BkfynEprERcXR2pqKhkZGWaX4hGCgoKIi4tz+uesP5zG4q2n2HI8i3KbnYggPwa0jeJn3ZqREBVKm6gQEmJCiQ2XSW29mtZQcN64X3vYs8a2j2fdW+8VQenv709CQoLZZYhqPtl9lsdX7KZ5ZDBTB7RheOdY+sY3kcknrEZr43zk7qXw880QGecR5yQv5RVBKdzLF/vP88TKPfSNb8KCmf0IDvA1uyRhBq1h7VOwfS70ewAiWppd0VWToBQO9c2hNB5ZtoterRoxb0ZfCUmrstvh88dh5wIY+DCMeNkjW5I/kKAUDrPpaAYPvvc9nZtHMH9mX5n81sq2zzVC8vonYPjzHh2SIEEpHOTboxncvyiJa2LDWDSrHxFyobi1XTvDmNiix10eH5IAcmZdNNiGI+ncvyiJdjFhLL2vvyy6ZVWVFfDNH6AkF/yDoOckrwhJkKAUDaC15tM953hg0U7ax4ax9P7+NJblWa3JVmbMJbn5L3DsK7OrcTjpeosrprVmS3IWf/nqCLvP5NIjLpJFs/pJS9KqKkph5VQjIG953bh328tIUIorcjyjkN9+uI9tJ7NpHhnEK7d3Z8K1cXJ9pFWVF8Pyu+HERhj9d0icaXZFTiFBKertwLk8pr67Ha01L47pwqR+rQnyl8t/LK00F3JOwrg5xppTXkqCUtTL96dzmDFvO2GBfiy5fwAJMruPtZUXgV+wcVviQ9uMwRsvJv0lUaf/JGcy5V/baBIawMqfD5SQtLqSXFg4Ftb8ytj28pAECUpRh/WH05ixYAdxjYNZ+cBA4hqHmF2SMFNxNiwaB+f3QLubzK7GZaTrLWr0ftIZfvPhPjo3j2DRrH5y6Y/VFWUaIZl5DCYthQ4jzK7IZSQoxU9orfnbuqO8uT6Z69tF89aUPnKnjdXZ7bBkAmQlw+Rl0G642RW5lASl+Ik/rTnEO5tPcldiK16+rZtc+iOM+SNvfM6YkTxhiNnVuJwEpfiRhf9J4Z3NJ5k2sA2/H9tVZoy3utwzcGYbdJ9guVZkdRKU4qJvDqXx+08PcFPnprwwRkLS8nJSYOEYKM03QjK4sdkVmUaCUgBwLreEJ1buoUuLCN6Y3EvW0ba6rONGSFYUw7SPLR2S4OTLg5RSI5VSR5RSyUqpZy7zemul1Aal1C6l1F6l1Chn1iMur9KueXzFbioq7fxjch9CAuT/T0vLOALzR4GtFKZ/Ci16m12R6ZwWlEopX2AOcAvQBZislOpyyW6/A1ZqrXsDk4C3nFWPqNnb3x5n28lsXpL1tAXA8fWg7TDjc2jW3exq3IIzmw79gGSt9QkApdRyYBxwsNo+GoioehwJnHNiPeIyvj+dw9/WHWVMzxbc0cdz1zQRDlBZYYxqD3jQmHA3pInZFbkNZ3a9WwJnqm2nVj1X3YvAFKVUKrAGeMSJ9YhL5BVX8OiyXTSLCOLl8d1k8MbKzn4P/0iEc7uNbQnJHzH7ArnJwAKtdRwwClislPpJTUqp2UqpJKVUkqzd7Rh2u+bxlbtJyy/lzbt7ExksF5Rb1pntxh03Wlt+0KYmzgzKs0CrattxVc9Vdy+wEkBrvRUIAqIvfSOt9VytdaLWOjEmJsZJ5VrLnA3JrD+cznOju9CntfxyWNap/8Di2yA0GmaugcZtzK7ILTnzHOUOoL1SKgEjICcBd1+yz2lgOLBAKdUZIyilyegkB87lsfFIBjtSsvn2aAbje7Vg6gD5xbCs83vgvTsgMg6mrYaI5mZX5LacFpRaa5tS6mHgS8AXmKe1PqCUeglI0lqvBn4FvKOUehxjYGeG1lo7qyYrS0rJZuI/t2LX0D42jOkD43lqZEc5L2llMZ2gz3QY/ASExZpdjVtTnpZLiYmJOikpyewyPEpxuY1b/nczlXbNRw9dR0x4oNklCTMd3wDNekBolNmVuBWl1E6tdeLlXjN7MEe4wJ/XHuZUVjF/ubOnhKTVHfoUltwJ6543uxKPIkHp5bYkZ7Jw6ylmXZfAgLbSgrC0/atg5XTjTpuRfzK7Go8iQenFCkoreOqDvbSNDuWpkR3NLkeYac8KWHUftOoPUz+EoEizK/IoclOvl8orruDn7+3kfF4JHzw4SFZLtDJbGWx6HdpcB3evgAC5TfVKSVB6odNZxcxcsJ3T2cZ5SblO0sK0Br9AY3KLoEgIkDWProYEpZc5dD6fKf/aRkWlnUWz+jPwGjkvaVnfvQ3nvofx/yfXSDaQnKP0Mq+sPQzAhw9dJyFpZVvegC+eNtbftleaXY3Hk6D0IsfSCth0NIMZg+JpFxtmdjnCLJteh3XPQdfb4c4F4CerZzaUBKUXmbflJIF+PtwjtyVa16a/wPqXjWnSbn/HmDZNNJgEpZfIKizjw+/PcnufOJrI+tvW1XoA9L3POC/pK0MQjiLfSS8xb8tJymx27r0+3uxShKtpDae/gzYDIf56449wKGlReoHTWcW8s/kk43u1oF1suNnlCFey22HNr2H+SCMshVNIi9LDaa15+fOD+CrFM7d0Nrsc4Up2O3z2GHy/CAY9Ytx1I5xCWpQebvmOM3x1MI1HhrejWWSQ2eUIV7FXwie/MEJy8JNw8x9ApsxzGmlRerD1h9N4/pP9DG4fzQNDrjG7HOFKx9fDnqUw7Fm44Smzq/F6EpQeqKS8kn9uOs5bG47TqVkE/7i7D74+0pqwlPY3w33fQNxlp08UDiZdbw+zIyWbG/+6kb9/fYybuzZl8b39ZGEwq7CVwar74cwOY1tC0mWkRelBdp3OYca87cRGBLHygYH0S5AlRS2jogRWTIXkddBmELTqa3ZFliJB6SHsds3Tq/bSKCSA5bMH0DRCBm4so7wYlk+GE9/CmDfg2ulmV2Q50vX2EBuOpHM0rZBfjeggIWkl5UXG0g0nN8H4tyQkTSItSg/x9rfHadkomDE9W5hdinAl3wBjhcTb34HuE8yuxrIkKD3Ap3vOsSMlh+dHd8HfVzoBllCSA7ZyCG8KE+bJNZImk6B0c/vP5vH0qr0ktmnM1IEyK5AlFGfDonHG49kbwUeW8TCbBKUbS04vZNq87TQOCWDOPX2kNWkFhRlGSGYlw6SlEpJuQoLSTZ3NLWHau9vwUfDeff1lAMcKCi7AwrGQexruWQlth5pdkagiQemGSsormfruNgrKbCyfPYCEaFk1zxI+ewLyUmHKBzJVmpuRoHRD3xxO40RGEf+alkjXFrL+smWM/psRlHLHjduRk15u6LM954kJD2RYp1izSxHOln0S1jwFlTYIbyYh6aYkKN1MYZmNDUfSGdWtmUx04e2yjsP8UbBvJeSeMrsaUQvperuZrw+mUWazM1ouLPduGUdg4RhjXsnpn0GUTJPnziQo3UiZrZIVO87QLCKIa1s3Nrsc4SxpB4zRbeUDMz6H2E5mVyTqIF1vN5GUks2o/93M1hNZ3Ht9Aj7S7fZe5UUQ0gRmrpGQ9BDSojSZ1ppXvzjM3E0naBEZzMJZ/bihQ4zZZQlnKEw37ttu1Q8e+k4uJvcg0qI02f98fYx/fnuCSX1b8dXjQyQkvdXpbfDmtcYaNyAh6WGkRWmi1XvO8cY3x5iYGMefbuuOkokPvFPKFmOqtPBmcM1ws6sRV0FalCb6+mAazSODJCS92YmN8N4dENnSOCcZ2dLsisRVkBalSc5kF7PrTA4J0aH4yWQX3in/HCydBE0SYNonxvlJ4ZEkKE3wye6zPLNqHz4KXhjd1exyhLNEtICxb8I1N0JolNnViAaQoDTB618eISE6lHemJ9KyUbDZ5QhHO7jauPwn/nrocafZ1QgHkD6fixWX20jNKWFkt2YSkt5o3wfw/gzY/FfQ2uxqhIM4NSiVUiOVUkeUUslKqWdq2GeiUuqgUuqAUmqpM+txB8nphQB0aBpmciXC4XYvgw/vh9YDYOIiWb7Bizit662U8gXmADcDqcAOpdRqrfXBavu0B34DXKe1zlFKef3Z7mNpRlC2bxpuciXCob5fBKsfhYTBMHk5BMgcot7EmS3KfkCy1vqE1rocWA6Mu2Sf+4E5WuscAK11uhPrMV25zc7Hu88S6OdDmyYhZpcjHEVrSPk3tBsOd6+UkPRCzhzMaQmcqbadCvS/ZJ8OAEqpLYAv8KLW+gsn1mSacpudh5Z8z+ZjmfxhfDe5JMhblBcZwTjuLdCV4BdodkXCCcz+bfUD2gNDgcnAO0qpRpfupJSarZRKUkolZWRkuLjEhiuzVfLQkp18fSiNP4zrytQBspqiV/j33+HtwVCUCb5+EpJezJlBeRZoVW07ruq56lKB1VrrCq31SeAoRnD+iNZ6rtY6UWudGBPjWfdCa615fMVuvj6UzsvjuzF1YLzZJQlH+PY1+PoFaNELgmS5Dm/nzKDcAbRXSiUopQKAScDqS/b5GKM1iVIqGqMrfsKJNbncyqQzrNl3gWdu6cQUaUl6Pq1h/cuw4Y/QYxLc/g74+ptdlXAypwWl1toGPAx8CRwCVmqtDyilXlJKja3a7UsgSyl1ENgA/FprneWsmlztVFYRv//0IAPbRjF7cFuzyxGOsO2fsOl16D0Vxr8lswBZhFPvzNFarwHWXPLc89Uea+CJqj9eRWvNk+/vwddH8ZeJPWUiXm/RfQKUF8L1T4CP2af4havIT9pJDp7PZ0dKDr+6uYPcgePp7HZImge2cgiNhiFPSkhajNzr7SRf7r+Aj0IWCfN09kr49DHYtRj8Q6HnXWZXJEwgQekka/dfoG98E6LD5JIRj2WvhI8fgr3L4YanocdEsysSJpH+g4PlFpfzt3VHOZZeyC3dmpldjrhalRXGfdt7l8Ow38Gw38q92xYmLUoHm714J9tPZjOsYwy3XxtndjniauWcguSv4eaX4LrHzK5GmEyC0oGS0wvYfjKbp0d24sGhsqC9R7JXGpf8RLeDh3dCmGfd4CCcQ7reDqK1Zt6WFPx8FHcmSkvSI1WUGIuA/ft/jG0JSVFFgtIBbJV2Xlh9gKXbTnNX31YygOOJyotg6UQ4vh5Cos2uRrgZ6Xo7wKPLd7Fm3wUeGNKWp0d2MrsccaXKCmDJRDjzHdz2NvScZHZFws1IUDbQ+bwS1uy7wM9vuIZnbpGQ9Dj2SqO7fWY73PEv6HaH2RUJNyRB2UDfHDLmGh7Ts7nJlYir4uMLve6GAQ9Bl7F17y8sSYKyAU5mFvHntYfp2aoRnZtFmF2OuBJFWZB5BNoMgj7TzK5GuLkrHsxRSvkope5xRjGe5pcrduPnq3jrnj4y6YUnKcyAhWNg2WQozTe7GuEBagxKpVSEUuo3Sql/KKVGKMMjGPNFWv5eLlulnb2puUwZ0EYmvfAkBRdgwa2QfQImLoQg6QmIutXW9V4M5ABbgfuA3wIKGK+13u2C2txaVlE5WkPTiCCzSxH1lXfWaEkWXIApH0D89WZXJDxEbUHZVmvdHUAp9S/gPNBaa13qksrcXHp+GQAx4XLNpMdImgdFGTD1I2h96Tp3QtSstqCs+OGB1rpSKZUqIflf6QXGtyJWgtL9aW1MaDHst8YId5TcXiquTG2DOT2VUvlKqQKlVAHQo9q25c+An8wsAiBWut7uLfMYzL8Fcs8YlwJJSIqrUGOLUmsti4HU4D/HM3ntyyP0jIukmQSl+0o/bJyT1Hbj7hshrlKNQamUCgJ+DrQD9gLzqhYMs7zXvjhCs4ggFszsh69cFuSeLuyHReOMVuSMNRDT0eyKhAerreu9EEgE9gGjgL+6pCI3l1FQxp7UXO7oE0fj0ACzyxGXk3YAFo4G3wAJSeEQtQ3mdKk26v0usN01Jbm3DYfT0Rpu6hJrdimiJhEtoM11MOJlaJJgdjXCC9R31NumZBp8AL4+lEaLyCC6NJcLld3O+b0Q3QGCG8OkJWZXI7xIbV3vXlWj3Pky6m0orahk87FMbuwci/zH4WZOboZ5I+GrZ82uRHih2lqUe7TWvV1WiQfYejyLkopKhnduanYporrjG4z7thu3gSG/Nrsa4YVqa1Fql1XhIT7adZaIID8Gto0yuxTxg6NfwdK7oElbmP4ZhMvKl8LxamtRxiqlnqjpRa3135xQj9vKKizjiwMXmNy3FUH+compW6gogdWPQGwnmPoxhDQxuyLhpWoLSl8gDGMiDMv767qjVNo1UwfGm12K+IF/sHHfdkQLCG5kdjXCi9UWlOe11i+5rBI3duBcHsu2n2bGoHjaxYaZXY7Y9wFkHYehT0PTLmZXIyygtnOU0pLEWIb296sP0jgkgF/e1MHscsTupfDh/XByE9jKza5GWERtQTncZVW4sS8PXGB7SjZPjuhIZLC/2eVY286F8PFDkDAE7nkf/OTOKOEaNQal1jrblYW4I601czYcJz4qhItFnDIAABS/SURBVLv6tjK7HGvb/g58+ii0uwkmr4CAELMrEhZyxWvmWMl/jmex72wes4dcI5NfmC0wAjqNNu648ZcZm4RrySqMtZi76QQx4YHc3qel2aVYV/YJ4xrJnndBj4nGBLxCuJi0KGuxJzWXEV2aynWTZtAaNv4Z5vSH83uM5yQkhUkkKGtQZqskt7hCJuY1g9aw/mXY+CfoNgGadjO7ImFx0vWuQUaBLB5mCq1h3XPwnzehz3QY/Xfwkf/PhbnkX2AN0quCMjZCgtKlDq02QrLv/RKSwm1Ii7IGP7QoY8Ol6+1SncbAnQuhyzg5Jynchvx3XYPTWcUAtGgUbHIlFmCvhHXPGyPcPj7QdbyEpHArEpQ1OHQhn6YRgTSRdXGcq9IGHz8IW/4Xjqw1uxohLsupQamUGqmUOqKUSlZKPVPLfncopbRSKtGZ9VyJw+cL6NhMlntwqsoK+PA+2LsCbnwOBv7C7IqEuCynBaVSyheYA9wCdAEmK6V+MtWLUioceAzY5qxarlRFpZ3k9EI6Nws3uxTvZSuH92fAgY+MRcCGPGl2RULUyJktyn5Astb6hNa6HFgOjLvMfn8A/gyUOrGWK5KSWUR5pZ1OzSUonaayDArT4JbXYNAjZlcjRK2cOerdEjhTbTsV6F99B6VUH6CV1vpzpZTbLHZy6EIBAB2bStfb4SpKQNshMBxmfgG+cuGFcH+mDeYopXyAvwG/qse+s5VSSUqppIyMDKfXdvh8Pn4+imtiQ53+WZZSXgRL7oTl9xgXlktICg/hzKA8C1Sfmyyu6rkfhAPdgI1KqRRgALD6cgM6Wuu5WutErXViTEyME0s2HL5QwDUxYQT6yT3eDlNWAO9NgFNboOdkufxHeBRnBuUOoL1SKkEpFQBMAlb/8KLWOk9rHa21jtdaxwPfAWO11klOrKlOdrtmb2ouXVpIt9thSvNg8W1wZhvc8a4xE5AQHsRpQam1tgEPA18Ch4CVWusDSqmXlFJjnfW5DbXrTC6ZheUM7ej8lqtlfDgbzu2GiQuh2+1mVyPEFXPqSSKt9RpgzSXPPV/DvkOdWUt9fXXwAn4+iqEdY80uxXsMfwES74UOI8yuRIirInfmXOLrg2kMaBsl6+M0VGE6bJ1jDNo07SIhKTyaBGU1p7OKOZ5RxI2dpDXZIPnnYcGtxpySOSfNrkaIBpPrM6rZcCQdQIKyIfJSYeEYo0U5ZZWxjIMQHk6Cspr1h9OJjwohPlqun7wqOSlGSJbkwtSPoVVfsysSwiGk612lpLySrSeyGCatyauXdgDKi2HaJxKSwqtIi7LK1hOZlNvs0u2+GhWlxhKynW6FhCHG7YlCeBFpUVZZfzidYH9f+iU0MbsUz5J+CN7sA0e+MLYlJIUXkqAEtNZsOJzBde2i5bbFK3FhnzG6ba+UQRvh1SQogQv5pZzNLWFw+2izS/Ec53bBgtHgFwQz10BMB7MrEsJp5BwlcDKjCIB2sWEmV+Ihck/DwnEQFAkzPoXG8WZXJIRTSVACJzKNoEyQy4LqJ7IVXPco9LgLGrWqe38hPJwEJcaM5oF+PjSLkKVpa5XybwiNNbrZsnSDsBA5RwmkZBURHxWKj4/MkVij4+uN+STXPmV2JUK4nAQlcDyjSLrdtTn6FSydBFHXwB3/MrsaIVzO8kFZUFrBycwimai3Joc+g+V3Q2wnmP4phMqVAcJ6LB+Uh84bC4l1aylB+RNaw/a50LwnTFsNIXIxvrAmyw/mHDiXB0DXFpEmV+Jm7Hbw8YFJS4zADJL/SIR1Wb5FuedMLjHhgcSGB5pdivvYtcSYBais0LglUUJSWJzlgzLpVA594xujZFVAQ9J8+OQh8PUHZfl/HkIAFg/K83klpOaUcG0bOfcGwLa58Nkvof3PYPJyCAgxuyIh3IKlz1HuSzXOT/Zu3cjkStxA0jxY+2voNBomzAe/ALMrEsJtWLpFmVFYBkCLyGCTK3ED8UOg731w5wIJSSEuYemgzCosB6BJqEWDQWs4ts74O7od3PpX49ykEOJHLB6UZUQE+RHgZ8Fvg9bwzUuwZAIc+NDsaoRwa5Y+R5lZVE50mAUvC9IavvodbP0HXDsDutxmdkVCuDVLB2VWYRlRYRbrdtvt8MXTxh03/WbDLa+BXBolRK0s2Of8r4yCMqJCLdaiTNtvjHAPfFhCUoh6snSLMi2/jMHtY8wuw7Wa94AHNkNsZwlJIerJsi3KwjIbhWU2mkVaYLLeSht89HPYv8rYbtpFQlKIK2DZoLyQVwrg/bOaV1bAqnthzzJjrRshxBWzbNc7Ld8IyqbeHJS2Mnh/Jhz5HEb8EQY9bHZFQngkywblxRalt3a9KytgxRQ49hXc8jr0n212RUJ4LOsGZb6Xd719/KBpV+g4ChJnml2NEB7NskGZWVhGeKAfwQG+ZpfiWGWFUHAeotvDTS+aXY0QXsGygzl5JRVEBHvZfc2l+fDeHbBgNJQXmV2NEF7Dsi3K/BIbkd4UlCW5Rkie322slBggq0oK4SgWDsoKIoK95PCLs2HxbZB2ACYugk63ml2REF7Fsl3v8/klxIR7yUDOxlch/RBMWiohKYQTWDIoS8orSc0poX1smNmlOMZNL8KMz6DDCLMrEcIrWTIoj2cUojW08+SgzD8Hq+6H0jxjbZtW/cyuSAiv5dSgVEqNVEodUUolK6WeuczrTyilDiql9iqlvlFKtXFmPT84nlEIeHBQ5p6B+aPgyBrIOm52NUJ4PacFpVLKF5gD3AJ0ASYrpbpcstsuIFFr3QP4AHjNWfVUdyytEF8fRXyUB44M56TAglHGAM7Uj6FlH7MrEsLrObNF2Q9I1lqf0FqXA8uBcdV30Fpv0FoXV21+B8Q5sZ6LktMLaRMV4nlLQGQdN1qSpfkw/RNo1dfsioSwBGcmRUvgTLXt1KrnanIvsNaJ9VyUnFFIuxgP7Hb7+EJotDFw06K32dUIYRlucSGhUmoKkAjcUMPrs4HZAK1bt27QZ5Xb7KRkFjGiS9MGvY9L5aVCeAtoHA+zv5W5JIVwMWe2KM8Craptx1U99yNKqZuAZ4GxWuuyy72R1nqu1jpRa50YE9OwGclPZBZis2s6Ngtv0Pu4zPm98PZg2PBHY1tCUgiXc2ZQ7gDaK6USlFIBwCRgdfUdlFK9gX9ihGS6E2u56MiFAgA6N49wxcc1zNnvYeEY8A+BXnebXY0QluW0oNRa24CHgS+BQ8BKrfUBpdRLSqmxVbu9DoQB7yuldiulVtfwdg5z6HwB/r6KhGg3H/E+sx0WjYOgCJi5BqKuMbsiISzLqecotdZrgDWXPPd8tcc3OfPzL+fwhXzaxYbj7+vGI95lhbBskjFwM/1TiHTJxQBCiBq4xWCOKx0+X8Cga6LMLqN2gWFwx7sQ0wkimptdjRCWZ6mgzC0u50J+qfsO5CR/DUWZ0HMSXDPM7GqEEFXcuP/peIerBnI6ueNAzpEvYNlk+O7/jOVlhRBuw1pBeT4fgM7u1qI89KmxEFjTrjD1I/C1VENfCLdnqaA8dL6AJqEBxIQHml3Kf+1fBSunG3faTPsEQpqYXZEQ4hKWarrsP5dH1xYRKHe6aDvrOLTqD/eshEA3a+kKIQALtSjLbJUcTSuga4tIs0sxlOQafw/5tdGSlJAUwm1ZJiiPpRVSUanp1tINBnJ2vAtv9IbMY8YtiX4BZlckhKiFZYLybG4JAG2amHxHzndvw+dPGDOSR7aqe38hhOksE5Q5ReUANAkzsfW25Q344mnoNBomLgZ/L1ncTAgvZ52gLK4AoEmISUG5/0NY9xx0vR3uXCDdbSE8iGVGvXOKywny9yE4wNecAjqOghEvQ/8H5TpJITyMZVqU2UXlNHZ1a1Jr45xkcbbRzR70iISkEB7IMkGZ4+qg1Bq+fNY4J7lrses+VwjhcJZp3mQXl9M41N81H2a3w9qnYMc7Rld70KOu+VwhhFNYpkWZX1JBZLALgtJuh89+aYTkoEdg5CuyfIMQHs4yLcq8EptrgrIkG05shMFPwo2/k5AUwgtYJijzSyuIcGZQVtqMUAyNhgc2QXAj532WEMKlLNH1LrNVUm6zExHkpKCsrIAPZsLqR4xBHAlJIbyKJYKyuKwSgFBnXENpK4OV0+DQamM+SelqC+F1LNH1LiwzZgwPCXTw4VaUwIqpkLwORv0F+t3v2PcXQrgFSwRlcbnRogxzdFB+cK+xzs2Y/4VrZzj2vYUQbsMSQVlUXtWidHTXu/9s6DwGek127PsKIdyKJYIyr2pCDIeMepfmw8lvjYBsO7Th7yeEcHuWGMw5k1MMQFyj4Ia9UUkOLB4PH8yC3DMOqEwI4Qks0aI8n1eKv68iOqwBi4oVZ8OicZB+CCYuhEYy6a4QVmGJoPxhQgwfn6u8dKcwwwjJrGSYvAza3+zYAoUQbs0SQZldVE6T0AbMHHR0LWSfgLtXwDXDHFeYEMIjWCIoMwvLri4otTYuIO8zDdoOk+62EBZlkcGcElpe6UBO7mmYewOc/d7YlpAUwrK8vkVZWlFJRkEZrZqE1P+Lsk/CwjHGpUDa7rzihBAeweuDMrXq0qBWTerZosxMNkLSVgLTV0OLXk6sTgjhCbw+KM/mlgLQslE9WpQ5p2DBKLBXwvTPoFk3J1cnhPAEXh+U6flGUDaNqMc1lOHNocPPYMAvILaTkysTQngK7w/KgjIAYsJrCcoL+yCsGYTFwNg3XVSZEMJTeP2o97G0AppHBhESUMP/Cak7YcGtsPph1xYmhPAYXh+UZ3JKSIgOvfyLp7cZd9wENYJRr7u2MCGEx/D6oCwstV1+CYiULbD4NgiLhZlroVFr1xcnhPAIXn+OsrDMRljQJYdpt8OXv4HIOOMSoPBm5hQnhPAIXh+UBaUVP53Z3McHJi8HH39jAEcIIWrh9V3vovJKQgOrZjY/shY+etC4TjKihYSkEKJenBqUSqmRSqkjSqlkpdQzl3k9UCm1our1bUqpeEfXYNcaX6Xg4CewYgpkHIbyIkd/jBDCizktKJVSvsAc4BagCzBZKdXlkt3uBXK01u2A/wH+7IxaOmZ+Be/PhJbXwrSPISjCGR8jhPBSzmxR9gOStdYntNblwHJg3CX7jAMWVj3+ABiulGMXxh7n829uOfo8tB4AU1ZBUKQj314IYQHODMqWQPWFZVKrnrvsPlprG5AHRF36Rkqp2UqpJKVUUkZGxhUVcU5Hc6Lx9XDP+xAYfkVfK4QQ4CGDOVrruVrrRK11YkzMlQ3APP+L+wif+T4E1HDRuRBC1MGZlwedBarPdhtX9dzl9klVSvkBkUCWI4vo1lK62kKIhnFmi3IH0F4plaCUCgAmAasv2Wc1ML3q8QRgvdZaO7EmIYS4Yk5rUWqtbUqph4EvAV9gntb6gFLqJSBJa70aeBdYrJRKBrIxwlQIIdyKU+/M0VqvAdZc8tzz1R6XAnc6swYhhGgojxjMEUIIM0lQCiFEHSQohRCiDhKUQghRBwlKIYSogwSlEELUQYJSCCHqoDztRhilVAZw6gq/LBrIdEI5ruYtxwFyLO7KW47lao6jjdb6spNJeFxQXg2lVJLWOtHsOhrKW44D5Fjclbcci6OPQ7reQghRBwlKIYSog1WCcq7ZBTiItxwHyLG4K285FocehyXOUQohRENYpUUphBBXzauC0h2Wx3WEehzHE0qpg0qpvUqpb5RSbcyosz7qOpZq+92hlNJKKbcdca3PsSilJlb9bA4opZa6usb6qMe/r9ZKqQ1KqV1V/8ZGmVFnfSil5iml0pVS+2t4XSml3qg61r1KqT5X9UFaa6/4gzE58HGgLRAA7AG6XLLPQ8DbVY8nASvMrvsqj2MYEFL1+EF3PI76HkvVfuHAJuA7INHsuhvwc2kP7AIaV23Hml33VR7HXODBqsddgBSz667leIYAfYD9Nbw+ClgLKGAAsO1qPsebWpRusTyuA9R5HFrrDVrr4qrN7zDWI3JH9fmZAPwBY033UlcWd4Xqcyz3A3O01jkAWut0F9dYH/U5Dg1EVD2OBM65sL4rorXehLE6Qk3GAYu04TugkVKq+ZV+jjcFpcOWxzVZfY6junsx/sd0R3UeS1VXqJXW+nNXFnYV6vNz6QB0UEptUUp9p5Qa6bLq6q8+x/EiMEUplYqxQsEjrinNKa709+mynLoUhHAupdQUIBG4wexaroZSygf4GzDD5FIcxQ+j+z0Uo5W/SSnVXWuda2pVV24ysEBr/Vel1ECMda26aa3tZhdmFm9qUV7J8rg4a3lcB6jPcaCUugl4FhirtS5zUW1Xqq5jCQe6ARuVUikY55BWu+mATn1+LqnAaq11hdb6JHAUIzjdSX2O415gJYDWeisQhHHvtCeq1+9TXbwpKL1ledw6j0Mp1Rv4J0ZIuuN5sB/Ueixa6zytdbTWOl5rHY9xvnWs1jrJnHJrVZ9/Xx9jtCZRSkVjdMVPuLLIeqjPcZwGhgMopTpjBGWGS6t0nNXAtKrR7wFAntb6/BW/i9mjVg4eARuF8b/4ceDZqudewvjlA+MH/j6QDGwH2ppd81Uex9dAGrC76s9qs2u+2mO5ZN+NuOmodz1/LgrjVMJBYB8wyeyar/I4ugBbMEbEdwMjzK65lmNZBpwHKjBa9PcCPwd+Xu1nMqfqWPdd7b8vuTNHCCHq4E1dbyGEcAoJSiGEqIMEpRBC1EGCUggh6iBBKYQQdZCgFB5LKVWplNpd7U+8UmqoUiqvavuQUuqFqn2rP39YKfUXs+sXnkNuYRSerERr3av6E1VT523WWo9WSoUCu5VSn1a9/MPzwcAupdRHWustri1ZeCJpUQqvpbUuAnYC7S55vgTjQuornhxBWJMEpfBkwdW63R9d+qJSKgrj/vEDlzzfGOMe7E2uKVN4Oul6C0/2k653lcFKqV2AHXhVa31AKTW06vk9GCH5d631BRfWKjyYBKXwRpu11qNrel4plQB8p5RaqbXe7erihOeRrrewHG1MgfYq8LTZtQjPIEEprOptYIi7LjAn3IvMHiSEEHWQFqUQQtRBglIIIeogQSmEEHWQoBRCiDpIUAohRB0kKIUQog4SlEIIUQcJSiGEqMP/A/dOu3/aeRgnAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import roc_curve"
      ],
      "metadata": {
        "id": "mWndm6pzqxKU"
      },
      "execution_count": 147,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "fpr, tpr, thresholds = roc_curve(y_val, y_pred)"
      ],
      "metadata": {
        "id": "EVwFA45AqzA0"
      },
      "execution_count": 148,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(5, 5))\n",
        "\n",
        "plt.plot(fpr, tpr, label='Model')\n",
        "plt.plot([0, 1], [0, 1], label='Random', linestyle='--')\n",
        "\n",
        "plt.xlabel('FPR')\n",
        "plt.ylabel('TPR')\n",
        "\n",
        "plt.legend()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 352
        },
        "id": "BCGrhtlWq02K",
        "outputId": "97736d3b-d8d7-4296-9e70-a11fcfea9a06"
      },
      "execution_count": 149,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f0c10183e90>"
            ]
          },
          "metadata": {},
          "execution_count": 149
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 360x360 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUoAAAE9CAYAAABtDit8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhV1b3/8feXMASQQRkUAQUVRZBJowhVC6JWkcEBERQI4NDqFW29Vnuda+1w9ba3V4u1aJmU0TkqTlWpSBENgihSFBAhiBLCpBCGkPX7Yyf8DiHJScjZZ5+z9+f1PHnMOWfnnO8m+GGtvfZay5xziIhIxWoFXYCISKpTUIqIxKGgFBGJQ0EpIhKHglJEJA4FpYhIHLWDLqC6mjdv7tq1axd0GSISMosWLdrknGtR3mtpF5Tt2rUjNzc36DJEJGTM7OuKXlPXW0QkDgWliEgcCkoRkTjS7hplefbu3UteXh67du0KupS0kJmZSZs2bahTp07QpYikhVAEZV5eHo0aNaJdu3aYWdDlpDTnHAUFBeTl5dG+ffugyxFJC6Hoeu/atYtmzZopJKvAzGjWrJla3yLVEIqgBBSS1aA/K5Hq8S0ozWyimW00s88qeN3M7BEzW2lmS83sVL9qSQYzY8SIEfsfFxUV0aJFCwYMGFCt92nXrh2bNm2q8TEikjh+tignAxdW8vpFQIeSr+uBv/pYi+8aNmzIZ599RmFhIQBvvfUWrVu3DrgqEUkE3wZznHPvmVm7Sg4ZDEx13hLrH5hZUzNr5Zzb4FdNfuvfvz+vvvoqQ4YMYcaMGQwfPpx58+YBsHnzZsaOHcvq1atp0KABEyZMoGvXrhQUFDB8+HDWr19Pr169iF1x/umnn+aRRx5hz5499OzZk8cee4yMjIygTk+kxqYvXMtLS9b7+hl13G46HN2c+wadkrD3DPIaZWtgXczjvJLnDmJm15tZrpnl5ufnJ6W4QzFs2DBmzpzJrl27WLp0KT179tz/2n333UePHj1YunQpv/vd7xg1ahQAv/71rznrrLNYtmwZl156KWvXrgVg+fLlzJo1i/nz57NkyRIyMjKYNm1aIOclUh3TF67lyr8tKPfrzhc+ZeFXm3377PrFO7i34A76ffN4Qt83LW4Pcs5NACYAZGVlVbrJz69fXsbn32xP6Od3Orox9w3sHPe4rl27smbNGmbMmEH//v0PeO3999/nueeeA+Dcc8+loKCA7du389577/H8888DcPHFF3P44YcD8Pbbb7No0SJOP/10AAoLC2nZsmUiT0vSVDJaZTVRGoQ92x9x0Gs92x/B4O6tuarnMYn/4J2b4enLYN8qTvzx3Ql96yCDcj3QNuZxm5Ln0tqgQYO47bbbmDt3LgUFBYf8Ps45srOz+f3vf5/A6iRdxYZjZUGUCnwNw4rs2QFTB0H+CrjyaTipsuGR6gsyKHOAm8xsJtAT2JaI65NVafn5aezYsTRt2pQuXbowd+7c/c+fffbZTJs2jXvuuYe5c+fSvHlzGjduzDnnnMP06dO5++67ee2119iyZQsA/fr1Y/DgwfziF7+gZcuWbN68me+//55jjz02oDOTZCrbaowNx0CCKNXVaQAdLoDz7ocTzkv42/sWlGY2A+gDNDezPOA+oA6Ac+5xYA7QH1gJ7ATG+FVLMrVp04abb775oOfvv/9+xo4dS9euXWnQoAFTpkwBvGuXw4cPp3PnzvTu3ZtjjvH+8nfq1IkHH3yQCy64gOLiYurUqcP48eMVlCFQla5z2VajwrEC2zfA7u3Q4iTod69vH2Pptq93VlaWK7se5fLlyzn55JMDqig96c8suQ6l66xgjGNbHkwZCBj8x4eQUbN2n5ktcs5llfdaWgzmiKS7l5as5/MN2+nUqrFah4mw5WuYMgAKt8KI52ockvEoKEV8VNqSLA3JWT/tFXRJ6a9gFUwZBHt+gFEvQWv/J/UpKEV8UBqQsd3swd01Uysh5v4Bigoh+2Vo1TUpH6mgFKmGqt7DWDYg1c1OoIF/hu3fQPMOSftIBaVIOSoKxKoOxCggE+zbT+Gd38LlT0C9RkkNSVBQihxk+sK13PnCp8DBgagADMA3i2HqJVC3IezY5AVlkikoEyQjI4MuXbpQVFRE+/bteeqpp2jatGmN33fy5Mnk5ubyl7/8JQFVSnXuYfzdpV0UiEFb9xE8fTnUb+Jdkzy8XSBlKCgTpH79+ixZsgSA7Oxsxo8fz1133RVwVdFVk66zWo0pYu1Cb+52wxZeSDZtG/9nfKKg9EGvXr1YunQpAB9++CG33HILu3bton79+kyaNImTTjqJyZMnk5OTw86dO1m1ahWXXnopDz30EACTJk3i97//PU2bNqVbt27Uq1cPgDVr1jB27Fg2bdpEixYtmDRpEscccwyjR4+mfv36LF68mI0bNzJx4kSmTp3KggUL6NmzJ5MnTw7qj8J31Q1EhWAaOawFtD4NLn0cGh8daCkKygTbt28fb7/9Ntdccw0AHTt2ZN68edSuXZt//OMf3HnnnftXEVqyZAmLFy+mXr16nHTSSYwbN47atWtz3333sWjRIpo0aULfvn3p0aMHAOPGjSM7O5vs7GwmTpzIzTffzIsvvgjAli1bWLBgATk5OQwaNIj58+fz5JNPcvrpp7NkyRK6d+8ezB+IT8q7/SaWAjGNbVwOLTrCEcdBdk7Q1QBhDcpJFx/8XOdL4IzrYM9OmHbFwa93vwp6XA07CmD2qANfG/Nq3I8sLCyke/furF+/npNPPpnzzz8fgG3btpGdnc2XX36JmbF37979P9OvXz+aNGkCeHO7v/76azZt2kSfPn1o0aIFAFdeeSVffPEFAAsWLNi/JNvIkSO5/fbb97/XwIEDMTO6dOnCkUceSZcuXbzT7tyZNWvWhC4oS2/iViCGzBdvwqwR0PdOOOvnQVezXziDMgCl1yh37tzJT37yE8aPH8/NN9/MPffcQ9++fXnhhRdYs2YNffr02f8zpV1q8AaDioqKDvnzS9+rVq1aB7xvrVq1avS+qaJsF1szXUJo+SvwzGg4sjOcOiru4ckUzqCsrAVYt0HlrzdsVqUWZEUaNGjAI488wiWXXMKNN97Itm3b9u+dU5VrhT179uSWW26hoKCAxo0b88wzz9CtWzcAevfuzcyZMxk5ciTTpk3j7LPPPuQ600l5t+t0atVYM13CZNkL8Ny10Kq7N3e7fs3vGEmkcAZlwHr06EHXrl2ZMWMGt99+O9nZ2Tz44INcfHE5lwTKaNWqFffffz+9evWiadOmB3SZH330UcaMGcPDDz+8fzAn7GJDUrfrhNQPG+GFG6DN6XDVbMhsHHRFB9EyaxGVDn9mCskIWTMfWnWDeocFVoKWWZO0Ud66jQrJkMqd5K1M3u1KaPejoKupVJC7MIocoLQFGXvLj0IypBZOgFd+Dp+/CGnQq1WLUlKCutkR8q9H4c27oeMAGDIJzIKuKK7QBKVzDkuDP/BUEOR16XgzaRSSITfvj/D2A9DpErj8ScioE3RFVRKKoMzMzKSgoIBmzZopLONwzlFQUEBmZmZSP1czaQSAvYXQZShc8lfft29IpPSptBJt2rQhLy+P/Pz8oEtJC5mZmbRp08b3z6loQy0FYsQ4B99v8OZr973Le1wrvYZHQhGUderUoX379kGXITHK3iSugIwo57zrkUumw8/mQZM2aXFNsqxQBKWkFg3MCOCF5Gu3w4cT4IyfQuP0nUmloJSEUkgKAMXF8OovYNFk6HUTXPBgWrYkSykoJWEUkrLfhxO8kDzrVuh3b1qHJCgoJUEUknKA00Z7C1t0vTLtQxI0M0cSpHR0WyEZYfv2wtu/gcKtUCcTug0LRUiCWpRSA7G3/5QuoquQjKii3fDsWPj3K9DiJOg6NOiKEkpBKdVW3s3jWh8ywvbugtkj4cs34aKHQxeSoKCUaip7f6TujYy4PTth5lWwei4M+DNkjQm6Il8oKKXKNGAjB9m1FbZ8BYPHe3tOhZSCUqpMAzay354dULu+Ny3xxoXe4E2IKSglrtJrkhqwEcAb1X76cmjVFQb8b+hDEhSUEkd51yQlwnZuhqcuhe+Wwdm3Bl1N0igopVxlR7bV3RZ2bIKpg2HTlzBsOpx4QdAVJY2CUg6ikW05SHExTBsCBSth+Aw4oV/QFSWVglIOokEbOUitWnDuPd6K5O3PCbqapFNQSrk0aCMAbF0H6xZClyGRa0XGUlDKfrGj251apd4m9JJkW9bAlIGwa7sXkvUPD7qiwCgoBdDotpRRsMoLyb07YdSLkQ5J8DkozexC4P+ADOBJ59wfyrx+DDAFaFpyzK+cc3P8rEnKp+uSsl/+CpgyCIr3QvbLcFSXoCsKnG/LrJlZBjAeuAjoBAw3s05lDrsbmO2c6wEMAx7zqx6p2PSFa1n41WZdlxTPqnfAFcPoVxWSJfxsUZ4BrHTOrQYws5nAYODzmGMcUHoxrAnwjY/1SAVKW5Pqbkfcvr3eqPaZN3gL7jY4Iv7PRISfC/e2BtbFPM4reS7W/cAIM8sD5gDjfKxHyqHWpACw/mP4SxZ8s8R7rJA8QNArnA8HJjvn2gD9gafM7KCazOx6M8s1s1zt3Z04sQM4ak1G2LoPvRk3zkV+0KYifna91wNtYx63KXku1jXAhQDOuQVmlgk0BzbGHuScmwBMAMjKynJ+FRwVmp4o+339L5h2BRzW0hu4adIm6IpSkp9B+RHQwcza4wXkMOCqMsesBfoBk83sZCATUJPRJ+WtTK7piRG24RNvFaAmbWBUDjRuFXRFKcu3oHTOFZnZTcAbeLf+THTOLTOzB4Bc51wO8J/AE2b2C7yBndHOObUYfaD523KQFh3h1GxvFaDDWgZdTUqzdMulrKwsl5ubG3QZaUUrk8sBVr0LR3WFhs2CriSlmNki51xWea8FPZgjPlNIygGWv+xdk3zr3qArSSsKyhBTSMoBPnsOZmfD0T3gwt8FXU1aUVCGmKYlyn6fzILnroW2PWHk85DZJOiK0ooWxQgh7XEjByjaDe89DMf+CK6aBXUbBl1R2lFQhlDsUmm6kTzinIPa9bx7JDObQN0GQVeUlhSUIRM7JXHWT3sFXY4E6YPH4ZuP4ZK/6h7JGtI1ypDRAhcCwPxH4PU7vP23i/cFXU3aU4syRLTAhQDe9ch3HoTOl8FlE7wVgaRG1KIMCS1wIQC89z9eSHa9Ei57QiGZIArKkNCtQALAMWfC6dd61yUz1GFMFP1Jhoi63BHlHKz9AI7tBe3O8r4kodSiFElnxcUw55cw6UIvLMUXCsoQKB3EkYgpLoZXboGPnoDe47xZN+ILdb3TWNn1JTWIEyHF++Clm+CT6XD2bXDu3WAWdFWhpaBMY7HTFLW+ZMSsescLyb53wY9vD7qa0FNQpqHYudydWjXWDJwo6nA+XPs2tCl3+URJMF2jTDOl90su/Gqz5nJHTdFueO46WPeR91ghmTRqUaYRrS8ZYXsLYdZIWPkWHNsb2p4edEWRoqBMI7qpPKL27ISZw2H1P2HgI3BadtAVRY6CMk1oHndE7dkB04bC2n/BJY9B97IbmUoyKCjThFYFiqiMut4OiZc9AV2GBF1NZCko04hakxFSuAWK9kCjI2HIRN0jGTAFZYoreyuQRMDOzTB1sPf99XOhVkaQ1QgKypSnbR0i5od8LyQLVsKw6QrJFKGgTGHa1iFivv8WpgyCrWvh6tlwXJ+gK5ISCsoUpYV4I+iVW2FbHox4VkulpRgFZQrSjeURNeBPXlBqxk3K0RTGFKQbyyNk81cw53bYVwSNjlJIpigFZYrRjeURUrAKJvWHT2fD1q+DrkYqoa53itGN5RGRvwKmDPTWlcx+BZodH3RFUgkFZQpRazIivlvmjW5bLRj9KrTsGHRFEoe63ilCo9wRsmcHNDgCxsxRSKYJtSgDVnY7Bw3ghNgPG715223PgBs/0M3kaUQtyoDFbuegkAyxtQvh0dPg46neY4VkWlGLMkCaeRMRa+bDtCu823+O7xd0NXII1KIMkEa4I2D1XHj6cmjS2rsm2US/63SkFmVANMIdAdu/genD4Ij2MOol7/qkpCUFZQA0wh0RjY+GQY/C8edCw2ZBVyM1oKAMgKYohtznOd7tP+3Ogq5XBF2NJICuUQZEXe6Q+vRZeGY0zPsjOBd0NZIgvgalmV1oZivMbKWZ/aqCY4aa2edmtszMpvtZTyoovTYpIbRkBjx/HRxzJgydqu0bQsS3rreZZQDjgfOBPOAjM8txzn0ec0wH4L+AHznntphZ6K92a6Q7pD6eCjk3Q/uzYfhMqNsw6IokgfxsUZ4BrHTOrXbO7QFmAoPLHHMdMN45twXAObfRx3oCp5HukHIO1rwPJ/SDq2YrJEPIz8Gc1sC6mMd5QM8yx5wIYGbzgQzgfufc6z7WFIiy0xTVmgyRPTu8YBz8GLh9ULte0BWJD4IezKkNdAD6AMOBJ8ysadmDzOx6M8s1s9z8/Pwkl1hzmqYYUu//GR4/G3ZsgozaCskQ87NFuR5oG/O4TclzsfKAhc65vcBXZvYFXnB+FHuQc24CMAEgKysrLYcSO7VqrGmKYfLPh+Dd38Ipl0Nmk6CrEZ/52aL8COhgZu3NrC4wDMgpc8yLeK1JzKw5Xld8tY81JdX0hWu58m8L+HzD9qBLkURxDt550AvJrsPgsicgo07QVYnPfAtK51wRcBPwBrAcmO2cW2ZmD5jZoJLD3gAKzOxz4F3gl865Ar9qSjbtyR1CC/8G7z0MPUbCJY9pFaCI8HVmjnNuDjCnzHP3xnzvgFtLvkJJXe6Q6TIE9vwAZ90KtYK+xC/Jot+0T3RjeYgUF0PuRCjaAw2bwzm3KSQjRnO9faIby0OieB+8fAssfgrqNIRuVwZdkQRAQekj3Vie5or3wYs3wtKZ8OM7oOvQoCuSgKj/4AN1u0Ng315v3vbSmdD3buh7p+ZuR5halAmmtSZDYsvXsPIfcP4D8KNbgq5GAqagTKDYkNQMnDRVvM+75af5CXDTIjisRdAVSQpQ1ztBFJIhsLfQ2wTs/f/1HiskpYSCMgEUkiGwZwdMHwqr3oEGzYOuRlKMut4JoK0d0tzu72HaUFj3AVz6OHQbFnRFkmIUlDWkNSbTXPE+r7u97kO4/ElvkQuRMhSUNaQby9NcrQzofhWceSN0GhT/eIkkBWUNqDWZxnYUwKYVcGxvOHVU0NVIiqv2YI6Z1TKzq/0oJt2oNZmmfsiHKQNhxnDYpSXwJL4Kg9LMGpvZf5nZX8zsAvOMw1svUnO5Sqg1mWa+/xYmXwybV8PQKZDZOOiKJA1U1vV+CtgCLACuBe4EDLjEObckCbWJJNa29V5L8vtvYcSz0O6soCuSNFFZUB7nnOsCYGZPAhuAY5xzu5JSmUii5U6EHfkw8gU4puw+dyIVq+wa5d7Sb5xz+4A8heT/p4Uv0ogr2Wap751w/VyFpFRbZUHZzcy2m9n3ZvY90DXmceSvgGsgJ01s+hImXQRb13m3AjU7PuiKJA1V2PV2zmkzkHKU7tFduv2sBnJS2MZ/e9ckXbE3+0bkEFUYlGaWCfwMOAFYCkws2TAs0rRhWJr49jOYOthrRY6eAy1OCroiSWOVDeZMwbtOOQ/oD3QGtDAf2jAs5X23DKYMgNr1Iftlb8k0kRqoLCg7xYx6/x34MDklidRQ46Ph2B/BBQ/CEe2DrkZCoLKgjB31LjItgy+pbsNSaH4i1D8chk0LuhoJkcpGvbuXjHJv16i3pLyv5sHEC+HNu4KuREKosqD8xDnXuOSrkXOudsz3kZz3pXsnU9Sqd72l0pq2hXN+GXQ1EkKVBaVLWhVpQvdOpqAv3oTpV8IRx0H2K9DoqKArkhCq7BplSzO7taIXnXN/8qGelKUl1VLQ3kLIGQctO8LIF6HBEUFXJCFVWVBmAIfhLYQRadqCNkXVqe/N2258NNRvGnQ1EmKVBeUG59wDSaskhWlPnBTz6bNQsAr63AFHdgq6GomAyq5RRr4lGUtd7hSxZDo8fx189R4U7Qm6GomIylqU/ZJWRYqKndfdqVUkB/pTy6Ip8PItcNyPYdgMqF036IokIipsUTrnIn8fjOZ1p5APn4CXb4YTzoPhs6Bug6ArkgjR5mJxaF53iqjXGDoOgCEToXa9oKuRiFFQSmrbvNq7R7LbldB1KGgqrQSg2rswiiSFczD3v2F8T9jwifecQlICoqCU1OMcvPMgzP0dnDIEjjwl6Iok4tT1ltTiHLx1D/zrUTg1Gwb8GWrp33MJlv4GVkALYARkeY4Xkqdfp5CUlKEWZQW0AEZAOg6EK6ZAp8G6JikpQ/9cV0KzcZKkeB+8da83wl2rFnS+RCEpKUVBKcHaVwQv3gDz/w9WvBZ0NSLl8jUozexCM1thZivN7FeVHHe5mTkzy/KzHkkx+/bC89fC0llw7j3Q6z+CrkikXL4FpZllAOOBi4BOwHAzO2ipFzNrhLe740K/apEUVLQHnhkNy17wNgE757agKxKpkJ8tyjOAlc651c65PcBMYHA5x/0G+G9gl4+1SKrZtxt++A4uegh6jwu6GpFK+Tnq3RpYF/M4D+gZe4CZnQq0dc69amba7CQK9haCK4Z6jWDM65ChGy8k9QU2mGNmtYA/Af9ZhWOvN7NcM8vNz8/3vTbdQ+mTPTu8TcBmXu3dWK6QlDThZ1CuB9rGPG5T8lypRsApwFwzWwOcCeSUN6DjnJvgnMtyzmW1aNHCx5K17YNvdn8PTw+Br+dDt+G6/UfSip//pH8EdDCz9ngBOQy4qvRF59w2oHnpYzObC9zmnMv1saa4tO2DD3Ztg6cvh/Ufw+V/h1MuC7oikWrxrUXpnCsCbgLeAJYDs51zy8zsATMb5NfnJoJuNE+w56+Hb5bA0CkKSUlLvl4kcs7NAeaUee7eCo7t42ctEqB+90HWNXDiBUFXInJINDNH/PHDRlgw3hu0ObKTQlLSmoYdJfG2b4Cpg2BbHpx0kbdCuUgaU4syhm4LSoBteTC5P2z/BkY8p5CUUFCLMoaWVquhLWtgykAo3AojX4S2pwddkUhCKChLlLYmNeJdA98tgz07YdRL0PrUoKsRSRgFZQm1Jmtg7y6okwkdL4b253jTE0VCRNcoY6g1eQg2LodHT4UVr3uPFZISQgpKOXTffgqTL/ZWKNegjYSYghKNdh+SbxbD5AFQOxPGzIEWJwZdkYhvdI0SXZ+stq1rYcpgyGwCo1+Gw9sFXZGIrxSUJXR9shqatIUf3Qxdr4SmbeMfL5LmFJRSdWveh4YtvW62tm6QCNE1SqmaVe9460m+dnvQlYgknYJS4vviTZg+DJodD5c/GXQ1IkkX+aDUiHccy1+BmVdBy46Q/TI0bB7/Z0RCJvJBqRHvSjgHH06AVt1gVA40OCLoikQCocEcNOJdruJiqFULhk3zAjOzcdAViQQm8i1KKcfiad4qQLt/8KYkKiQl4hSUcqDcSfDSjZBRB0x/PUQg4kGpgZwyFk6AV34OHX4Cw2dC3QZBVySSEiJ9jVIDOTFyJ8Jrv4SOA2DIJKhdN+iKRFJGpFuUoIGc/dqdA6dfC1dMVkiKlBH5oIw05+DLt7z/Nj8BLv6jd21SRA6goIwq5+DtB2DaEFj2fNDViKS0SF+jjCzn4M27YcFf4LTR0OnSoCsSSWkKyqgpLobX7/Bm3JxxPVz0EJgFXZVISlPXO2q++8wb4e51k0JSpIrUooyaVl3hp/Og5ckKSZEqUosyCvYVwQs/g8+e8x4f2UkhKVINCsqw27cXnrsGPpnh7XUjItWmrneYFe2GZ8bAilfhgt9C75uCrkgkLUW2RRn6ed779sKsEV5IXvSwQlKkBiLbogz9PO9ateHIznBSf8gaE3Q1ImktskEJIZ3nvfsH+H4DNO8A590fdDUioRDZrnco7doOT18OkwfAnh1BVyMSGpFuUYZK4VYvJDcs8XZKrNsw6IpEQkNBGQY7N8NTl8J3y2DoVOh4cdAViYSKgjIM5v4BNi6HYdPhxAuCrkYkdBSUYXDe/dBlCLQ9I+hKREJJgznpavs38Nx1sGubt7eNQlLEN74GpZldaGYrzGylmf2qnNdvNbPPzWypmb1tZsf6WU+ptL/ZfOs6mNQfVsyBglVBVyMSer4FpZllAOOBi4BOwHAz61TmsMVAlnOuK/As8JBf9cRK65vNt6yByf29AZyRL0LrU4OuSCT0/GxRngGsdM6tds7tAWYCg2MPcM6965zbWfLwA6CNj/UcIC1vNi9Y5bUkd22H7Jeg7elBVyQSCX4GZWtgXczjvJLnKnIN8JqP9aS/WhnQsDmMfgWO7hF0NSKRkRKDOWY2AsgCHq7g9evNLNfMcvPz82v0WWl5fXJbnreFw+Ht4Pp/wlFdgq5IJFL8DMr1QNuYx21KnjuAmZ0H3AUMcs7tLu+NnHMTnHNZzrmsFi1a1KiotLs+uWEpPH42vPtb77EW3BVJOj+D8iOgg5m1N7O6wDAgJ/YAM+sB/A0vJDf6WMsB0ub65PqPYcpAqNMAul8VdDUikeVbUDrnioCbgDeA5cBs59wyM3vAzAaVHPYwcBjwjJktMbOcCt4uetZ9CFMHQ2ZjGDMHmh0fdEUikeXrzBzn3BxgTpnn7o35/jw/P7+s0uuTPdsfkcyPrb7dP8CMYd7ATfbL0CRpNwOISDkiNYUxba5P1jsMLv87tOgIjVsFXY1I5EUqKCHFr0+u/Afs2ATdhsHxfYOuRkRKpMTtQQKseB1mDIcP/uptLysiKUNBmQqWv+xtBHZkZxj5AmRErqEvktIiE5Qpe6P5Z8/B7Gxvps2ol6BBig80iURQZJouKTuQU7AK2vaEq2dDvUZBVyMi5YhMixJSbCCncKv333N+6bUkFZIiKStSQZkyPvo7PNIDNn3pTUmsXTfoikSkEgrKZPvgcXj1Vm9F8iZt4x8vIoFTUCbT/Efg9Tug4wAY+hTUyQy6IhGpAgVlsnz2PLx1D3S+DK6YrO62SBqJzKh34E7qDxc8CD1v0H2SImlGLUo/Oeddk9y52etm9x6nkBRJQwpKvzgHb9zlXZNc/FTQ1YhIDah544fiYnjtdvjoCa+r3fvmoCsSkRpQUG/zcQIAAAgaSURBVCZacTG88nP4eIrX1T7/N9q+QSTNKSgTrXAzrJ4LZ98G596tkBQJAQVlouwr8kKxYXP46XtQv2nQFYlIgmgwJxH27YVnx0DOOG8QRyEpEioKypoq2g2zR8HyHG89SXW1RUJHXe+a2FsIs0bCyreg///AGdcFXZGI+EBBWRPPXuPtczPw/+C00UFXIyI+iUTX27fVzXteD5f8VSEpEnKRaFEmdHXzXdvhq3/CyQPhuD41fz8RSXmRaFFCglY3L9wCT10Cz46FresSU5iIpLxItCgTYudmmDoYNi6HoVOgqRbdFYkKBWVV/JDvhWTBShg+AzqcH3RFIpJECsqq+OI12LwarpoFx/cNuhoRSTIFZWWc824gP3UUHNdX3W2RiIrMYE61bV0LE34M6z/2HiskRSJLLcrybP4Kpgz0bgVyxUFXIyIBU1CWtWmlF5JFhZCdA0d3D7oiEQlY6Lve1ZqVs+VrmNwf9u2B7FcUkiICRKBFWa1ZOY1awYk/gTP/A1p29LkyEUkXoQ9KqMKsnG8/hcOOgsNawKBHk1eYiKSF0He948pbBJMvhpybgq5ERFJUtINy7UJvxk1mU+j/cNDViEiKim5QrpkPT10Kh7WEMa9B0xoumCEioRWJa5QHKS6GN/4LmrTxbgFqdFTQFYlICotmUNaqBcNnQq063gCOiEglotX1XvEavHADFO+DxkcrJEWkSnwNSjO70MxWmNlKM/tVOa/XM7NZJa8vNLN2vhXz+UswawTk/xv27PDtY0QkfHwLSjPLAMYDFwGdgOFm1qnMYdcAW5xzJwD/C/y3H7X0LpwLz4yB1qfBqBchs7EfHyMiIeVni/IMYKVzbrVzbg8wExhc5pjBwJSS758F+pkldmPsswrfYdzWh+CYM2HEc5DZJJFvLyIR4GdQtgZiN5bJK3mu3GOcc0XANqBZ2Tcys+vNLNfMcvPz86tVxGEt2/FFk95w9TNQr1G1flZEBNJk1Ns5NwGYAJCVleWq87NjrroauNqPskQkIvxsUa4HYle7bVPyXLnHmFltoAlQ4GNNIiLV5mdQfgR0MLP2ZlYXGAbklDkmB8gu+X4I8I5zrlotRhERv/nW9XbOFZnZTcAbQAYw0Tm3zMweAHKdcznA34GnzGwlsBkvTEVEUoqv1yidc3OAOWWeuzfm+13AFX7WICJSU9GamSMicggUlCIicSgoRUTiUFCKiMShoBQRiUNBKSISh4JSRCQOS7eJMGaWD3xdzR9rDmzyoZxkC8t5gM4lVYXlXA7lPI51zpW7mnfaBeWhMLNc51xW0HXUVFjOA3QuqSos55Lo81DXW0QkDgWliEgcUQnKCUEXkCBhOQ/QuaSqsJxLQs8jEtcoRURqIiotShGRQxaqoEyp7XFroArncauZfW5mS83sbTM7Nog6qyLeucQcd7mZOTNL2RHXqpyLmQ0t+d0sM7Ppya6xKqrw9+sYM3vXzBaX/B3rH0SdVWFmE81so5l9VsHrZmaPlJzrUjM79ZA+yDkXii+8xYFXAccBdYFPgE5ljrkReLzk+2HArKDrPsTz6As0KPn+hlQ8j6qeS8lxjYD3gA+ArKDrrsHvpQOwGDi85HHLoOs+xPOYANxQ8n0nYE3QdVdyPucApwKfVfB6f+A1wIAzgYWH8jlhalGmxPa4CRD3PJxz7zrndpY8/ABvP6JUVJXfCcBv8PZ035XM4qqpKudyHTDeObcFwDm3Mck1VkVVzsMBjUu+bwJ8k8T6qsU59x7e7ggVGQxMdZ4PgKZm1qq6nxOmoEzY9rgBq8p5xLoG71/MVBT3XEq6Qm2dc68ms7BDUJXfy4nAiWY238w+MLMLk1Zd1VXlPO4HRphZHt4OBeOSU5ovqvv/U7nSYrtaKZ+ZjQCygB8HXcuhMLNawJ+A0QGXkii18brfffBa+e+ZWRfn3NZAq6q+4cBk59wfzawX3r5WpzjnioMuLChhalGGZXvcqpwHZnYecBcwyDm3O0m1VVe8c2kEnALMNbM1eNeQclJ0QKcqv5c8IMc5t9c59xXwBV5wppKqnMc1wGwA59wCIBNv7nQ6qtL/T/GEKSjDsj1u3PMwsx7A3/BCMhWvg5Wq9Fycc9ucc82dc+2cc+3wrrcOcs7lBlNupary9+tFvNYkZtYcryu+OplFVkFVzmMt0A/AzE7GC8r8pFaZODnAqJLR7zOBbc65DdV+l6BHrRI8AtYf71/xVcBdJc89gPc/H3i/8GeAlcCHwHFB13yI5/EP4DtgSclXTtA1H+q5lDl2Lik66l3F34vhXUr4HPgUGBZ0zYd4Hp2A+Xgj4kuAC4KuuZJzmQFsAPbiteivAX4G/CzmdzK+5Fw/PdS/X5qZIyISR5i63iIivlBQiojEoaAUEYlDQSkiEoeCUkQkDgWlpC0z22dmS2K+2plZHzPbVvJ4uZndV3Js7PP/NrP/Cbp+SR+awijprNA51z32iZKl8+Y55waYWUNgiZm9XPJy6fP1gcVm9oJzbn5yS5Z0pBalhJZzbgewCDihzPOFeDdSV3txBIkmBaWks/ox3e4Xyr5oZs3w5o8vK/P84XhzsN9LTpmS7tT1lnR2UNe7xNlmthgoBv7gnFtmZn1Knv8ELyT/7Jz7Nom1ShpTUEoYzXPODajoeTNrD3xgZrOdc0uSXZykH3W9JXKctwTaH4A7gq5F0oOCUqLqceCcVN1gTlKLVg8SEYlDLUoRkTgUlCIicSgoRUTiUFCKiMShoBQRiUNBKSISh4JSRCQOBaWISBz/D4sBUe5AxoJSAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**4.6 ROC AUC**\n",
        "\n",
        "- Area under the ROC curve - useful metric\n",
        "- Interpretation of AUC"
      ],
      "metadata": {
        "id": "HeZbLhB3q5Gk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import auc\n"
      ],
      "metadata": {
        "id": "NmVj1OV8q2gf"
      },
      "execution_count": 150,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "auc(fpr, tpr)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W-sWCLMYrAaJ",
        "outputId": "7b33bac0-eb2d-402b-c160-11e3564e5e49"
      },
      "execution_count": 151,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8438530381535563"
            ]
          },
          "metadata": {},
          "execution_count": 151
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "auc(df_scores.fpr, df_scores.tpr)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CD4nvk8crA8z",
        "outputId": "7263a0e3-8f2d-4297-fd75-fe2271aeaf26"
      },
      "execution_count": 152,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8438732975754537"
            ]
          },
          "metadata": {},
          "execution_count": 152
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "auc(df_ideal.fpr, df_ideal.tpr)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DmiGWtPwrDie",
        "outputId": "11e92652-0094-47fc-a48e-dfbb6762f592"
      },
      "execution_count": 153,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9999430203759136"
            ]
          },
          "metadata": {},
          "execution_count": 153
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "fpr, tpr, thresholds = roc_curve(y_val, y_pred)\n",
        "auc(fpr, tpr)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cSHom7aRrL8t",
        "outputId": "fce6f257-d031-4cdd-cd41-6e35a5ed0f9e"
      },
      "execution_count": 154,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8438530381535563"
            ]
          },
          "metadata": {},
          "execution_count": 154
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import roc_auc_score"
      ],
      "metadata": {
        "id": "CXrhWr9arObI"
      },
      "execution_count": 155,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "roc_auc_score(y_val, y_pred)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e-9tY3pkrSHK",
        "outputId": "7f0434eb-d709-4a4d-d8b7-149b613d28d8"
      },
      "execution_count": 156,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8438530381535563"
            ]
          },
          "metadata": {},
          "execution_count": 156
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "neg = y_pred[y_val == 0]\n",
        "pos = y_pred[y_val == 1]"
      ],
      "metadata": {
        "id": "Y_bSsJi7rU-U"
      },
      "execution_count": 157,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import random"
      ],
      "metadata": {
        "id": "-a_H551drXQm"
      },
      "execution_count": 158,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "n = 100000\n",
        "success = 0 \n",
        "\n",
        "for i in range(n):\n",
        "    pos_ind = random.randint(0, len(pos) - 1)\n",
        "    neg_ind = random.randint(0, len(neg) - 1)\n",
        "\n",
        "    if pos[pos_ind] > neg[neg_ind]:\n",
        "        success = success + 1\n",
        "\n",
        "success / n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bUhvBZJArZVx",
        "outputId": "f3b55e26-0fe6-4f54-9719-3f19112f53b6"
      },
      "execution_count": 159,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.84394"
            ]
          },
          "metadata": {},
          "execution_count": 159
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "n = 50000\n",
        "\n",
        "np.random.seed(1)\n",
        "pos_ind = np.random.randint(0, len(pos), size=n)\n",
        "neg_ind = np.random.randint(0, len(neg), size=n)\n",
        "\n",
        "(pos[pos_ind] > neg[neg_ind]).mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L55kdbx-raV4",
        "outputId": "4f6bb728-d236-4496-daf2-56843cf6fd06"
      },
      "execution_count": 160,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.84646"
            ]
          },
          "metadata": {},
          "execution_count": 160
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**4.7 Cross-Validation**\n",
        "\n",
        "- Evaluating the same model on different subsets of data\n",
        "- Getting the average prediction and the spread within predictions\n"
      ],
      "metadata": {
        "id": "iEba7u0vrgyv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def train(df_train, y_train, C=1.0):\n",
        "    dicts = df_train[categorical + numerical].to_dict(orient='records')\n",
        "\n",
        "    dv = DictVectorizer(sparse=False)\n",
        "    X_train = dv.fit_transform(dicts)\n",
        "\n",
        "    model = LogisticRegression(C=C, max_iter=1000)\n",
        "    model.fit(X_train, y_train)\n",
        "    \n",
        "    return dv, model"
      ],
      "metadata": {
        "id": "Uk5iYXinrciw"
      },
      "execution_count": 161,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dv, model = train(df_train, y_train, C=0.001)"
      ],
      "metadata": {
        "id": "OgSyE8XJroPD"
      },
      "execution_count": 162,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def predict(df, dv, model):\n",
        "    dicts = df[categorical + numerical].to_dict(orient='records')\n",
        "\n",
        "    X = dv.transform(dicts)\n",
        "    y_pred = model.predict_proba(X)[:, 1]\n",
        "\n",
        "    return y_pred"
      ],
      "metadata": {
        "id": "yly1gucUrqq2"
      },
      "execution_count": 163,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y_pred = predict(df_val, dv, model)"
      ],
      "metadata": {
        "id": "s5MU8rGersm_"
      },
      "execution_count": 164,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import KFold"
      ],
      "metadata": {
        "id": "9750yS1Gruj1"
      },
      "execution_count": 165,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install tqdm"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yBGOsZm1ryqV",
        "outputId": "81db051e-8a2f-4ee9-b9dc-102fa3e60d0c"
      },
      "execution_count": 166,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (4.63.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm.auto import tqdm"
      ],
      "metadata": {
        "id": "0PD9grr8r075"
      },
      "execution_count": 167,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "n_splits = 5\n",
        "\n",
        "for C in tqdm([0.001, 0.01, 0.1, 0.5, 1, 5, 10]):\n",
        "    kfold = KFold(n_splits=n_splits, shuffle=True, random_state=1)\n",
        "\n",
        "    scores = []\n",
        "\n",
        "    for train_idx, val_idx in kfold.split(df_full_train):\n",
        "        df_train = df_full_train.iloc[train_idx]\n",
        "        df_val = df_full_train.iloc[val_idx]\n",
        "\n",
        "        y_train = df_train.churn.values\n",
        "        y_val = df_val.churn.values\n",
        "\n",
        "        dv, model = train(df_train, y_train, C=C)\n",
        "        y_pred = predict(df_val, dv, model)\n",
        "\n",
        "        auc = roc_auc_score(y_val, y_pred)\n",
        "        scores.append(auc)\n",
        "\n",
        "    print('C=%s %.3f +- %.3f' % (C, np.mean(scores), np.std(scores)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 174,
          "referenced_widgets": [
            "fb91d98cebab4bbe90d5a8123d60a9ce",
            "eb355b827ff1440296adce6e383fcccc",
            "7eed98b55f3c47e7a211ba33a966fe5f",
            "212e5c3bd5f44ce8bf1ab8bee51e16ff",
            "d6f9a289b07d447d8d329b70de7e1784",
            "dfcbe3fa676f4c9ba40aaa52cdf618da",
            "27f0cc1c712c4c64a5f8519befe85f05",
            "14b90626139143259f0d47b34b165b55",
            "0aa548606dc9469bbfeabfafdbfdced2",
            "b0abb94bb47e43038028fe925de666b7",
            "4127591b58c34d4f8e069fd643998664"
          ]
        },
        "id": "Nic7B-MBr4ax",
        "outputId": "247bb50c-d636-422b-d214-40b8e6d1a5e2"
      },
      "execution_count": 168,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/7 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fb91d98cebab4bbe90d5a8123d60a9ce"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "C=0.001 0.825 +- 0.009\n",
            "C=0.01 0.840 +- 0.009\n",
            "C=0.1 0.841 +- 0.007\n",
            "C=0.5 0.840 +- 0.006\n",
            "C=1 0.840 +- 0.008\n",
            "C=5 0.841 +- 0.008\n",
            "C=10 0.842 +- 0.008\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "scores"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nBKPD79jr6xg",
        "outputId": "e7e07ec0-7ce9-4e2f-e49c-b79c43ed85b0"
      },
      "execution_count": 169,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.8437252239169172,\n",
              " 0.8458461627598993,\n",
              " 0.8325265904073851,\n",
              " 0.8329702788258618,\n",
              " 0.8525462018763139]"
            ]
          },
          "metadata": {},
          "execution_count": 169
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dv, model = train(df_full_train, df_full_train.churn.values, C=1.0)\n",
        "y_pred = predict(df_test, dv, model)\n",
        "\n",
        "auc = roc_auc_score(y_test, y_pred)\n",
        "auc"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "irshDZomsFlj",
        "outputId": "a65c5764-ed9e-4a7f-cdd3-4eb917dd41bb"
      },
      "execution_count": 170,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8572386167896259"
            ]
          },
          "metadata": {},
          "execution_count": 170
        }
      ]
    }
  ]
}