11_Regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "beautiful-tenant",
   "metadata": {},
   "source": [
    "# Flight Delay Prediction\n",
    "## Regression Engine to predict Arrival Delay\n",
    "\n",
    "This notebook contains the code to train a regression engine which predicts the Arrival delay period (in minutes) for delayed flights."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "dress-needle",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pre-requisites\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Store the classifier models to save time\n",
    "import joblib\n",
    "\n",
    "# Preprocessing\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Classifiers from scikit-learn\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import ExtraTreesRegressor\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "# Performance metrics\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import r2_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "southeast-montreal",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/admin/.local/lib/python3.8/site-packages/numpy/lib/arraysetops.py:580: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  mask |= (ar1 == a)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Shape: (1851436, 33)\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1851436 entries, 0 to 1851435\n",
      "Data columns (total 33 columns):\n",
      " #   Column           Dtype  \n",
      "---  ------           -----  \n",
      " 0   Year             int64  \n",
      " 1   Quarter          int64  \n",
      " 2   Month            int64  \n",
      " 3   DayofMonth       int64  \n",
      " 4   FlightDate       object \n",
      " 5   OriginAirportID  int64  \n",
      " 6   Origin           object \n",
      " 7   DestAirportID    int64  \n",
      " 8   Dest             object \n",
      " 9   CRSDepTime       int64  \n",
      " 10  DepTime          float64\n",
      " 11  DepDelayMinutes  float64\n",
      " 12  DepDel15         float64\n",
      " 13  CRSArrTime       int64  \n",
      " 14  ArrTime          float64\n",
      " 15  ArrDelayMinutes  float64\n",
      " 16  ArrDel15         float64\n",
      " 17  Time_new         int64  \n",
      " 18  windspeedKmph    int64  \n",
      " 19  winddirDegree    int64  \n",
      " 20  weatherCode      int64  \n",
      " 21  precipMM         float64\n",
      " 22  visibility       int64  \n",
      " 23  pressure         int64  \n",
      " 24  cloudcover       int64  \n",
      " 25  DewPointF        int64  \n",
      " 26  WindGustKmph     int64  \n",
      " 27  tempF            int64  \n",
      " 28  WindChillF       int64  \n",
      " 29  humidity         int64  \n",
      " 30  date             object \n",
      " 31  time             int64  \n",
      " 32  airport          object \n",
      "dtypes: float64(7), int64(21), object(5)\n",
      "memory usage: 480.3+ MB\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"Data/flight_and_weather.csv\", index_col=0)\n",
    "print(f\"\\nShape: {df.shape}\", end=\"\\n\\n\")\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "lonely-citation",
   "metadata": {},
   "source": [
    "### Eliminating Redundancy\n",
    "|Column|Reason for elimination|\n",
    "|:-|:-|\n",
    "|FlightDate| The columns Year, Month and DayofMonth give this information in separate columns|\n",
    "|OriginAirportID| It gives the same information as Origin|\n",
    "|DestAirportID| It gives the same information as Dest|\n",
    "|**CRSArrTime, ArrTime and ArrDel15**| They **leak information about target ArrDelayMinutes**|\n",
    "|Time_new|It is a duplicate of time|\n",
    "|date |It is a duplicate of FlightDate|\n",
    "|airport|It is a duplicate of Origin|"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "thirty-syndication",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Shape: (1851436, 24)\n",
      "\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1851436 entries, 0 to 1851435\n",
      "Data columns (total 24 columns):\n",
      " #   Column           Dtype  \n",
      "---  ------           -----  \n",
      " 0   Year             int64  \n",
      " 1   Quarter          int64  \n",
      " 2   Month            int64  \n",
      " 3   DayofMonth       int64  \n",
      " 4   Origin           object \n",
      " 5   Dest             object \n",
      " 6   CRSDepTime       int64  \n",
      " 7   DepTime          float64\n",
      " 8   DepDelayMinutes  float64\n",
      " 9   DepDel15         float64\n",
      " 10  ArrDelayMinutes  float64\n",
      " 11  windspeedKmph    int64  \n",
      " 12  winddirDegree    int64  \n",
      " 13  weatherCode      int64  \n",
      " 14  precipMM         float64\n",
      " 15  visibility       int64  \n",
      " 16  pressure         int64  \n",
      " 17  cloudcover       int64  \n",
      " 18  DewPointF        int64  \n",
      " 19  WindGustKmph     int64  \n",
      " 20  tempF            int64  \n",
      " 21  WindChillF       int64  \n",
      " 22  humidity         int64  \n",
      " 23  time             int64  \n",
      "dtypes: float64(5), int64(17), object(2)\n",
      "memory usage: 353.1+ MB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# Dropping columns with redundant or duplicate data\n",
    "df.drop(columns=[\"FlightDate\",\n",
    "                 \"OriginAirportID\",\n",
    "                 \"DestAirportID\",\n",
    "                 \"CRSArrTime\",\n",
    "                 \"ArrTime\",\n",
    "                 \"ArrDel15\",\n",
    "                 \"Time_new\",\n",
    "                 \"date\",\n",
    "                 \"airport\"],\n",
    "        inplace=True)\n",
    "print(f\"\\nShape: {df.shape}\", end=\"\\n\\n\")\n",
    "print(df.info())\n",
    "# df.to_csv(\"./Data/flight_and_weather_without_redundant_info.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "forbidden-scroll",
   "metadata": {},
   "outputs": [],
   "source": [
    "labelEncoder = LabelEncoder()\n",
    "df[\"Origin\"] = labelEncoder.fit_transform(df[\"Origin\"])\n",
    "df[\"Dest\"] = labelEncoder.fit_transform(df[\"Dest\"])\n",
    "# Only need the observations where the flight is delayed\n",
    "df = df[df[\"ArrDelayMinutes\"] > 0]\n",
    "df.reset_index(inplace=True, drop=True)\n",
    "# print(df.columns)\n",
    "# print(df.shape)\n",
    "features = df.loc[:, df.columns != \"ArrDelayMinutes\"]\n",
    "labels = np.asarray(df[\"ArrDelayMinutes\"])\n",
    "# print(features.shape)\n",
    "# print(labels.shape)\n",
    "# df.to_csv(\"Data/flight_and_weather_encoded_regression.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "animated-admission",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Dataset shape: (700439, 24)\n",
      "features_train shape: (560351, 23) | features_test shape: (140088, 23)\n",
      "labels_train shape: (560351,) | labels_test shape: (140088,)\n"
     ]
    }
   ],
   "source": [
    "# Number of samples/observations/rows is greater than 100,000\n",
    "print(f\"\\nDataset shape: {df.shape}\")\n",
    "features_train, features_test, labels_train, labels_test = train_test_split(features, labels, test_size=0.20, random_state=42)\n",
    "print(f\"features_train shape: {features_train.shape} | features_test shape: {features_test.shape}\")\n",
    "print(f\"labels_train shape: {labels_train.shape} | labels_test shape: {labels_test.shape}\")\n",
    "# print(f\"{features_train.shape[1]} Features: {features_train.columns.to_list()}\")\n",
    "del df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "complimentary-importance",
   "metadata": {},
   "outputs": [],
   "source": [
    "perf_df = pd.DataFrame(columns=[\"Regressors\", \"MSE\", \"RMSE\", \"MAE\", \"R2\"])\n",
    "def print_metrics(labels_test, model_pred, regressor_name, perf_df):\n",
    "    \n",
    "    mse = mean_squared_error(labels_test, model_pred)\n",
    "    rmse = np.sqrt(mse)\n",
    "    mae = mean_absolute_error(labels_test, model_pred)\n",
    "    r2 = r2_score(labels_test, model_pred)\n",
    "    \n",
    "    print(f\"MSE      : {mse}\", end=\"\\n\\n\")\n",
    "    print(f\"RMSE     : {rmse}\", end=\"\\n\\n\")\n",
    "    print(f\"MAE      : {mae}\", end=\"\\n\\n\")\n",
    "    print(f\"R2 Score : {r2}\", end=\"\\n\\n\")\n",
    "    \n",
    "    perf_df = perf_df.append({\"Regressors\": regressor_name,\n",
    "                                        \"MSE\": mse, \n",
    "                                        \"RMSE\": rmse,\n",
    "                                        \"MAE\": mae,\n",
    "                                        \"R2\": r2}, ignore_index=True)\n",
    "    return perf_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "viral-christian",
   "metadata": {},
   "source": [
    "## Training Different Regression Models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "precious-genome",
   "metadata": {},
   "source": [
    "### Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "universal-parent",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE      : 246.11533169589046\n",
      "\n",
      "RMSE     : 15.688063350710006\n",
      "\n",
      "MAE      : 10.633112849932374\n",
      "\n",
      "R2 Score : 0.9312750617189874\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# model = LinearRegression(n_jobs=-1)\n",
    "# model.fit(features_train, labels_train)\n",
    "# joblib.dump(model, \"./Regressors/LogisticRegression.joblib\")\n",
    "model = joblib.load(\"./Regressors/LogisticRegression.joblib\")\n",
    "model_pred = model.predict(features_test)\n",
    "perf_df = print_metrics(labels_test, model_pred, \"LinearRegression\", perf_df)\n",
    "del model\n",
    "del model_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "continental-cattle",
   "metadata": {},
   "source": [
    "### Decision Tree Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dying-beauty",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE      : 472.70934341271203\n",
      "\n",
      "RMSE     : 21.741879942008513\n",
      "\n",
      "MAE      : 14.630232425332649\n",
      "\n",
      "R2 Score : 0.8680012324829942\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# model = DecisionTreeRegressor()\n",
    "# model.fit(features_train, labels_train)\n",
    "# joblib.dump(model, \"./Regressors/DecisionTreeRegressor.joblib\")\n",
    "model = joblib.load(\"./Regressors/DecisionTreeRegressor.joblib\")\n",
    "model_pred = model.predict(features_test)\n",
    "perf_df = print_metrics(labels_test, model_pred, \"DecisionTreeRegressor\", perf_df)\n",
    "del model\n",
    "del model_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "african-ordinary",
   "metadata": {},
   "source": [
    "### XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "statewide-filling",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE      : 234.03676085446193\n",
      "\n",
      "RMSE     : 15.298260059708161\n",
      "\n",
      "MAE      : 10.367285945784936\n",
      "\n",
      "R2 Score : 0.9346478667770067\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# model = GradientBoostingRegressor()\n",
    "# model.fit(features_train, labels_train)\n",
    "# joblib.dump(model, \"./Regressors/GradientBoostingRegressor.joblib\")\n",
    "model = joblib.load(\"./Regressors/GradientBoostingRegressor.joblib\")\n",
    "model_pred = model.predict(features_test)\n",
    "perf_df = print_metrics(labels_test, model_pred, \"GradientBoostingRegressor\", perf_df)\n",
    "del model\n",
    "del model_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "boring-harvey",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "mechanical-mixer",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE      : 228.859378607919\n",
      "\n",
      "RMSE     : 15.128098975347795\n",
      "\n",
      "MAE      : 10.419297918179321\n",
      "\n",
      "R2 Score : 0.9360935925385799\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# model = RandomForestRegressor(n_jobs=-1)\n",
    "# model.fit(features_train, labels_train)\n",
    "# joblib.dump(model, \"./Regressors/RandomForestRegressor.joblib\")\n",
    "model = joblib.load(\"./Regressors/RandomForestRegressor.joblib\")\n",
    "model_pred = model.predict(features_test)\n",
    "perf_df = print_metrics(labels_test, model_pred, \"RandomForestRegressor\", perf_df)\n",
    "del model\n",
    "del model_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "postal-antibody",
   "metadata": {},
   "source": [
    "### Extra Trees Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "driving-belief",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE      : 232.2070527990739\n",
      "\n",
      "RMSE     : 15.238341537026722\n",
      "\n",
      "MAE      : 10.500640740106219\n",
      "\n",
      "R2 Score : 0.9351587921724802\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model = ExtraTreesRegressor(n_jobs=-1)\n",
    "model.fit(features_train, labels_train)\n",
    "# joblib.dump(model, \"./Regressors/ExtraTreesRegressor.joblib\")\n",
    "# model = joblib.load(\"./Regressors/ExtraTreesRegressor.joblib\")\n",
    "model_pred = model.predict(features_test)\n",
    "perf_df = print_metrics(labels_test, model_pred, \"ExtraTreesRegressor\", perf_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "living-cotton",
   "metadata": {},
   "source": [
    "## Performance Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "minor-economics",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MSE</th>\n",
       "      <th>RMSE</th>\n",
       "      <th>MAE</th>\n",
       "      <th>R2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Regressors</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>LinearRegression</th>\n",
       "      <td>246.115332</td>\n",
       "      <td>15.688063</td>\n",
       "      <td>10.633113</td>\n",
       "      <td>0.931275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DecisionTreeRegressor</th>\n",
       "      <td>472.709343</td>\n",
       "      <td>21.741880</td>\n",
       "      <td>14.630232</td>\n",
       "      <td>0.868001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GradientBoostingRegressor</th>\n",
       "      <td>234.036761</td>\n",
       "      <td>15.298260</td>\n",
       "      <td>10.367286</td>\n",
       "      <td>0.934648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RandomForestRegressor</th>\n",
       "      <td>228.859379</td>\n",
       "      <td>15.128099</td>\n",
       "      <td>10.419298</td>\n",
       "      <td>0.936094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ExtraTreesRegressor</th>\n",
       "      <td>232.207053</td>\n",
       "      <td>15.238342</td>\n",
       "      <td>10.500641</td>\n",
       "      <td>0.935159</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  MSE       RMSE        MAE        R2\n",
       "Regressors                                                           \n",
       "LinearRegression           246.115332  15.688063  10.633113  0.931275\n",
       "DecisionTreeRegressor      472.709343  21.741880  14.630232  0.868001\n",
       "GradientBoostingRegressor  234.036761  15.298260  10.367286  0.934648\n",
       "RandomForestRegressor      228.859379  15.128099  10.419298  0.936094\n",
       "ExtraTreesRegressor        232.207053  15.238342  10.500641  0.935159"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Set name of the regressors as index labels\n",
    "perf_df.set_index(\"Regressors\", inplace=True)\n",
    "perf_df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}