2nd-attempt-mnist-accuracy-99.ipynb

{"cells":[{"source":"<a href=\"https://www.kaggle.com/code/jarredpriester/2nd-attempt-mnist-accuracy-99?scriptVersionId=100383446\" target=\"_blank\"><img align=\"left\" alt=\"Kaggle\" title=\"Open in Kaggle\" src=\"https://kaggle.com/static/images/open-in-kaggle.svg\"></a>","metadata":{},"cell_type":"markdown","outputs":[],"execution_count":0},{"cell_type":"markdown","id":"f8ced37f","metadata":{"papermill":{"duration":0.012085,"end_time":"2022-07-09T03:26:27.446257","exception":false,"start_time":"2022-07-09T03:26:27.434172","status":"completed"},"tags":[]},"source":["# My 2nd Attempt at MNIST"]},{"cell_type":"markdown","id":"d22e9fb8","metadata":{"papermill":{"duration":0.008547,"end_time":"2022-07-09T03:26:27.463779","exception":false,"start_time":"2022-07-09T03:26:27.455232","status":"completed"},"tags":[]},"source":["### Jarred Priester\n","### 7/8/22"]},{"cell_type":"markdown","id":"ba9af6f6","metadata":{"papermill":{"duration":0.008617,"end_time":"2022-07-09T03:26:27.481212","exception":false,"start_time":"2022-07-09T03:26:27.472595","status":"completed"},"tags":[]},"source":["1. Introduction\n","2. Downloading the data\n","3. Processing the images\n","4. Model\n","5. Predictions\n","6. Conclusion"]},{"cell_type":"markdown","id":"fbe4ba10","metadata":{"papermill":{"duration":0.008552,"end_time":"2022-07-09T03:26:27.498468","exception":false,"start_time":"2022-07-09T03:26:27.489916","status":"completed"},"tags":[]},"source":["# 1. Introduction"]},{"cell_type":"markdown","id":"8b0bb453","metadata":{"papermill":{"duration":0.008528,"end_time":"2022-07-09T03:26:27.515654","exception":false,"start_time":"2022-07-09T03:26:27.507126","status":"completed"},"tags":[]},"source":["In my first submission for the Kaggle Digit Recognizer competition I submitted a simple yet affective model with the accuracy score of 98%. In this notebook I will attempt to beat that score by adding more layers to the first model.\n","\n","The first submission can be found here for comparison:\n","https://www.kaggle.com/code/jarredpriester/my-first-cnn"]},{"cell_type":"markdown","id":"19a64db1","metadata":{"papermill":{"duration":0.009266,"end_time":"2022-07-09T03:26:27.533755","exception":false,"start_time":"2022-07-09T03:26:27.524489","status":"completed"},"tags":[]},"source":["# 2. Downloading the data"]},{"cell_type":"code","execution_count":1,"id":"1ea9fff7","metadata":{"execution":{"iopub.execute_input":"2022-07-09T03:26:27.553186Z","iopub.status.busy":"2022-07-09T03:26:27.552713Z","iopub.status.idle":"2022-07-09T03:26:34.834692Z","shell.execute_reply":"2022-07-09T03:26:34.833136Z"},"papermill":{"duration":7.294204,"end_time":"2022-07-09T03:26:34.836741","exception":false,"start_time":"2022-07-09T03:26:27.542537","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["/kaggle/input/digit-recognizer/sample_submission.csv\n","/kaggle/input/digit-recognizer/train.csv\n","/kaggle/input/digit-recognizer/test.csv\n"]}],"source":["#importing libaries\n","import numpy as np \n","import pandas as pd\n","import random as rd\n","\n","import matplotlib.pyplot as plt\n","%matplotlib inline\n","\n","from PIL import Image\n","\n","from sklearn.model_selection import train_test_split\n","\n","import tensorflow as tf\n","from tensorflow import keras\n","from tensorflow.keras import layers\n","from tensorflow.keras.layers.experimental import preprocessing\n","from keras.applications.vgg16 import VGG16\n","\n","#setting seed for reproducability\n","from numpy.random import seed\n","seed(10)\n","tf.random.set_seed(20)\n","\n","import os\n","for dirname, _, filenames in os.walk('/kaggle/input'):\n","    for filename in filenames:\n","        print(os.path.join(dirname, filename))\n","        "]},{"cell_type":"code","execution_count":2,"id":"3588c7e6","metadata":{"execution":{"iopub.execute_input":"2022-07-09T03:26:34.856438Z","iopub.status.busy":"2022-07-09T03:26:34.855769Z","iopub.status.idle":"2022-07-09T03:26:34.896622Z","shell.execute_reply":"2022-07-09T03:26:34.895856Z"},"papermill":{"duration":0.052197,"end_time":"2022-07-09T03:26:34.898076","exception":false,"start_time":"2022-07-09T03:26:34.845879","status":"completed"},"tags":[]},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>ImageId</th>\n","      <th>Label</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>3</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>4</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>5</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   ImageId  Label\n","0        1      0\n","1        2      0\n","2        3      0\n","3        4      0\n","4        5      0"]},"execution_count":2,"metadata":{},"output_type":"execute_result"}],"source":["#downloading the sample submission\n","sample_sub = pd.read_csv(\"../input/digit-recognizer/sample_submission.csv\")\n","sample_sub.head()"]},{"cell_type":"code","execution_count":3,"id":"795d52da","metadata":{"execution":{"iopub.execute_input":"2022-07-09T03:26:34.911925Z","iopub.status.busy":"2022-07-09T03:26:34.911529Z","iopub.status.idle":"2022-07-09T03:26:37.657353Z","shell.execute_reply":"2022-07-09T03:26:37.656359Z"},"papermill":{"duration":2.755399,"end_time":"2022-07-09T03:26:37.659567","exception":false,"start_time":"2022-07-09T03:26:34.904168","status":"completed"},"tags":[]},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>label</th>\n","      <th>pixel0</th>\n","      <th>pixel1</th>\n","      <th>pixel2</th>\n","      <th>pixel3</th>\n","      <th>pixel4</th>\n","      <th>pixel5</th>\n","      <th>pixel6</th>\n","      <th>pixel7</th>\n","      <th>pixel8</th>\n","      <th>...</th>\n","      <th>pixel774</th>\n","      <th>pixel775</th>\n","      <th>pixel776</th>\n","      <th>pixel777</th>\n","      <th>pixel778</th>\n","      <th>pixel779</th>\n","      <th>pixel780</th>\n","      <th>pixel781</th>\n","      <th>pixel782</th>\n","      <th>pixel783</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>4</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>5 rows × 785 columns</p>\n","</div>"],"text/plain":["   label  pixel0  pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  \\\n","0      1       0       0       0       0       0       0       0       0   \n","1      0       0       0       0       0       0       0       0       0   \n","2      1       0       0       0       0       0       0       0       0   \n","3      4       0       0       0       0       0       0       0       0   \n","4      0       0       0       0       0       0       0       0       0   \n","\n","   pixel8  ...  pixel774  pixel775  pixel776  pixel777  pixel778  pixel779  \\\n","0       0  ...         0         0         0         0         0         0   \n","1       0  ...         0         0         0         0         0         0   \n","2       0  ...         0         0         0         0         0         0   \n","3       0  ...         0         0         0         0         0         0   \n","4       0  ...         0         0         0         0         0         0   \n","\n","   pixel780  pixel781  pixel782  pixel783  \n","0         0         0         0         0  \n","1         0         0         0         0  \n","2         0         0         0         0  \n","3         0         0         0         0  \n","4         0         0         0         0  \n","\n","[5 rows x 785 columns]"]},"execution_count":3,"metadata":{},"output_type":"execute_result"}],"source":["#downloading the training data\n","train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\n","train.head()"]},{"cell_type":"code","execution_count":4,"id":"2333e866","metadata":{"execution":{"iopub.execute_input":"2022-07-09T03:26:37.68149Z","iopub.status.busy":"2022-07-09T03:26:37.680233Z","iopub.status.idle":"2022-07-09T03:26:39.591863Z","shell.execute_reply":"2022-07-09T03:26:39.588967Z"},"papermill":{"duration":1.926994,"end_time":"2022-07-09T03:26:39.596395","exception":false,"start_time":"2022-07-09T03:26:37.669401","status":"completed"},"tags":[]},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>pixel0</th>\n","      <th>pixel1</th>\n","      <th>pixel2</th>\n","      <th>pixel3</th>\n","      <th>pixel4</th>\n","      <th>pixel5</th>\n","      <th>pixel6</th>\n","      <th>pixel7</th>\n","      <th>pixel8</th>\n","      <th>pixel9</th>\n","      <th>...</th>\n","      <th>pixel774</th>\n","      <th>pixel775</th>\n","      <th>pixel776</th>\n","      <th>pixel777</th>\n","      <th>pixel778</th>\n","      <th>pixel779</th>\n","      <th>pixel780</th>\n","      <th>pixel781</th>\n","      <th>pixel782</th>\n","      <th>pixel783</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>5 rows × 784 columns</p>\n","</div>"],"text/plain":["   pixel0  pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  pixel8  \\\n","0       0       0       0       0       0       0       0       0       0   \n","1       0       0       0       0       0       0       0       0       0   \n","2       0       0       0       0       0       0       0       0       0   \n","3       0       0       0       0       0       0       0       0       0   \n","4       0       0       0       0       0       0       0       0       0   \n","\n","   pixel9  ...  pixel774  pixel775  pixel776  pixel777  pixel778  pixel779  \\\n","0       0  ...         0         0         0         0         0         0   \n","1       0  ...         0         0         0         0         0         0   \n","2       0  ...         0         0         0         0         0         0   \n","3       0  ...         0         0         0         0         0         0   \n","4       0  ...         0         0         0         0         0         0   \n","\n","   pixel780  pixel781  pixel782  pixel783  \n","0         0         0         0         0  \n","1         0         0         0         0  \n","2         0         0         0         0  \n","3         0         0         0         0  \n","4         0         0         0         0  \n","\n","[5 rows x 784 columns]"]},"execution_count":4,"metadata":{},"output_type":"execute_result"}],"source":["#downloading the test data\n","test = pd.read_csv(\"../input/digit-recognizer/test.csv\")\n","test.head()"]},{"cell_type":"markdown","id":"4096688a","metadata":{"papermill":{"duration":0.005999,"end_time":"2022-07-09T03:26:39.608827","exception":false,"start_time":"2022-07-09T03:26:39.602828","status":"completed"},"tags":[]},"source":["# 3. Processing the images"]},{"cell_type":"markdown","id":"cc1164f5","metadata":{"papermill":{"duration":0.00654,"end_time":"2022-07-09T03:26:39.621312","exception":false,"start_time":"2022-07-09T03:26:39.614772","status":"completed"},"tags":[]},"source":["In the training data set we can see that the first column is 'label' and the rest are pixels. We will use the label column as our Y and the rest as our X"]},{"cell_type":"code","execution_count":5,"id":"587efb88","metadata":{"execution":{"iopub.execute_input":"2022-07-09T03:26:39.636294Z","iopub.status.busy":"2022-07-09T03:26:39.635931Z","iopub.status.idle":"2022-07-09T03:26:39.695848Z","shell.execute_reply":"2022-07-09T03:26:39.694815Z"},"papermill":{"duration":0.069409,"end_time":"2022-07-09T03:26:39.697637","exception":false,"start_time":"2022-07-09T03:26:39.628228","status":"completed"},"tags":[]},"outputs":[],"source":["Y_train = train[\"label\"]\n","\n","X_train = train.drop(labels = [\"label\"],axis = 1) "]},{"cell_type":"markdown","id":"6e145ce1","metadata":{"papermill":{"duration":0.005806,"end_time":"2022-07-09T03:26:39.710126","exception":false,"start_time":"2022-07-09T03:26:39.70432","status":"completed"},"tags":[]},"source":["Next we need to scale the data from 0-255 to 0-1. This will make things easier to work with the neural network because it will allow the nn to converge faster."]},{"cell_type":"code","execution_count":6,"id":"aae861ed","metadata":{"execution":{"iopub.execute_input":"2022-07-09T03:26:39.723564Z","iopub.status.busy":"2022-07-09T03:26:39.723208Z","iopub.status.idle":"2022-07-09T03:26:39.796066Z","shell.execute_reply":"2022-07-09T03:26:39.795315Z"},"papermill":{"duration":0.082175,"end_time":"2022-07-09T03:26:39.798268","exception":false,"start_time":"2022-07-09T03:26:39.716093","status":"completed"},"tags":[]},"outputs":[],"source":["X_train = X_train / 255.0\n","test = test / 255.0"]},{"cell_type":"markdown","id":"4897d670","metadata":{"papermill":{"duration":0.009567,"end_time":"2022-07-09T03:26:39.818192","exception":false,"start_time":"2022-07-09T03:26:39.808625","status":"completed"},"tags":[]},"source":["The images will need to be reshaped in order feed into our model. Our images will be 28x28 and since we will be using grayscale the color channel will be 1."]},{"cell_type":"code","execution_count":7,"id":"d363471a","metadata":{"execution":{"iopub.execute_input":"2022-07-09T03:26:39.839353Z","iopub.status.busy":"2022-07-09T03:26:39.838971Z","iopub.status.idle":"2022-07-09T03:26:39.843361Z","shell.execute_reply":"2022-07-09T03:26:39.842738Z"},"papermill":{"duration":0.017864,"end_time":"2022-07-09T03:26:39.845818","exception":false,"start_time":"2022-07-09T03:26:39.827954","status":"completed"},"tags":[]},"outputs":[],"source":["X_train = X_train.values.reshape(-1,28,28,1)\n","test = test.values.reshape(-1,28,28,1)"]},{"cell_type":"code","execution_count":8,"id":"60add173","metadata":{"execution":{"iopub.execute_input":"2022-07-09T03:26:39.8686Z","iopub.status.busy":"2022-07-09T03:26:39.868174Z","iopub.status.idle":"2022-07-09T03:26:39.872558Z","shell.execute_reply":"2022-07-09T03:26:39.87187Z"},"papermill":{"duration":0.018719,"end_time":"2022-07-09T03:26:39.874897","exception":false,"start_time":"2022-07-09T03:26:39.856178","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["(42000, 28, 28, 1)\n","(28000, 28, 28, 1)\n"]}],"source":["print(X_train.shape)\n","print(test.shape)"]},{"cell_type":"markdown","id":"78d1651f","metadata":{"papermill":{"duration":0.009745,"end_time":"2022-07-09T03:26:39.895762","exception":false,"start_time":"2022-07-09T03:26:39.886017","status":"completed"},"tags":[]},"source":["Now that the data has been scaled and reshaped we can now split our training data into training and validation sets. The split will be 90% to train on and 10% to use as validation. Which leaves us with the test data which will we be making our predictions on."]},{"cell_type":"code","execution_count":9,"id":"4a423b60","metadata":{"execution":{"iopub.execute_input":"2022-07-09T03:26:39.916836Z","iopub.status.busy":"2022-07-09T03:26:39.91652Z","iopub.status.idle":"2022-07-09T03:26:40.161171Z","shell.execute_reply":"2022-07-09T03:26:40.16033Z"},"papermill":{"duration":0.257658,"end_time":"2022-07-09T03:26:40.163268","exception":false,"start_time":"2022-07-09T03:26:39.90561","status":"completed"},"tags":[]},"outputs":[],"source":["X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=5)"]},{"cell_type":"code","execution_count":10,"id":"bf153da3","metadata":{"execution":{"iopub.execute_input":"2022-07-09T03:26:40.184697Z","iopub.status.busy":"2022-07-09T03:26:40.184379Z","iopub.status.idle":"2022-07-09T03:26:40.188853Z","shell.execute_reply":"2022-07-09T03:26:40.188164Z"},"papermill":{"duration":0.016987,"end_time":"2022-07-09T03:26:40.190433","exception":false,"start_time":"2022-07-09T03:26:40.173446","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["(37800, 28, 28, 1)\n","(37800,)\n","(4200, 28, 28, 1)\n","(4200,)\n"]}],"source":["print(X_train.shape)\n","print(Y_train.shape)\n","print(X_val.shape)\n","print(Y_val.shape)"]},{"cell_type":"markdown","id":"b8fd1f05","metadata":{"papermill":{"duration":0.006067,"end_time":"2022-07-09T03:26:40.202847","exception":false,"start_time":"2022-07-09T03:26:40.19678","status":"completed"},"tags":[]},"source":["Let us take a look at an image from the training set so we have a visual of the images we are working with"]},{"cell_type":"code","execution_count":11,"id":"1b503689","metadata":{"execution":{"iopub.execute_input":"2022-07-09T03:26:40.217099Z","iopub.status.busy":"2022-07-09T03:26:40.216392Z","iopub.status.idle":"2022-07-09T03:26:40.347736Z","shell.execute_reply":"2022-07-09T03:26:40.347198Z"},"papermill":{"duration":0.140224,"end_time":"2022-07-09T03:26:40.349182","exception":false,"start_time":"2022-07-09T03:26:40.208958","status":"completed"},"tags":[]},"outputs":[{"data":{"text/plain":["<matplotlib.image.AxesImage at 0x7fa4d18db090>"]},"execution_count":11,"metadata":{},"output_type":"execute_result"},{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOwElEQVR4nO3df5BV9XnH8c+zy6+IoIBKKDL1R0wdkzagK5rKGKzTiDgNJJmgTMbS1s7aVjo4tWNp0lZnOp0yNjSjJrUhlQGtwWTGWPmDNBKkZdSUuDD8lrLIYIXyI4YiS1Rgd5/+sUdn1T3fu9x77j0XnvdrZufee5499zxe+ey593zPPV9zdwE4+7WU3QCAxiDsQBCEHQiCsANBEHYgiCGN3NgwG+4jNLKRmwRCeVe/1Ek/YQPVagq7mc2Q9LCkVkn/4u6LUr8/QiN1nd1cyyYBJKz3Nbm1qt/Gm1mrpG9LulXSVZLmmtlV1T4fgPqq5TP7VEm73X2Pu5+U9LSkWcW0BaBotYR9oqQ3+j3ely37ADNrN7MOM+s4pRM1bA5ALep+NN7dl7h7m7u3DdXwem8OQI5awr5f0qR+jy/OlgFoQrWE/RVJV5jZpWY2TNIdklYW0xaAolU99Obu3WY2X9KP1Tf0ttTdtxfWGYBC1TTO7u6rJK0qqBcAdcTpskAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBNHQS0mjOkMm/kqyfuqSi/LX3fk/yXV7fnGkqp5w5mHPDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBMM5eAP/sZ5L13X/UmqwvnbYsWR/b+l/J+oUt3bm1N3rSs/D84ebfTdaHrzw/WR/31MZk3U8w5VezYM8OBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0Ewzj5Ib331+tzawgf/Nbnubee8Vdu2e3uS9a7e/NqUYem/5xuuTffecq0l65+Y1p6s/9ofb82tMQbfWDWF3cz2SuqS1COp293bimgKQPGK2LPf5O5vFvA8AOqIz+xAELWG3SU9b2YbzGzAD29m1m5mHWbWcUp8RgPKUuvb+Gnuvt/MLpK02sx2uvu6/r/g7kskLZGk0TbWa9wegCrVtGd39/3Z7WFJz0qaWkRTAIpXddjNbKSZjXrvvqTPS9pWVGMAilXL2/jxkp41s/ee53vu/u+FdNWEPrNgc26t0jj6oZ53kvXPPfPnyfqlK08m60OO5R8L6bp8VHLdty5L/73f/KffStZ33fKdZP2GufNza2OW/TS5LopVddjdfY+k9FUbADQNht6AIAg7EARhB4Ig7EAQhB0Iwtwbd1LbaBvr19nNDdtekVbtz79k8n0H0+cSdd42LlnvPnioqp6aQefyq5P1l256JLc27457kuvay/nDnRjYel+jY35kwO8ls2cHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSC4lHQBXvheepx9wsGXG9RJ432yfXuy/jfrbsmtHf3628l1x9xWVUvIwZ4dCIKwA0EQdiAIwg4EQdiBIAg7EARhB4JgnH2Qpm35Sm7t4mU7k+umJ1w+s1WadnnrI1Nyay899O3kutPuzL8MtSSd/ySXoj4d7NmBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAiuG4+6ajnnnNzalJeOJ9fd2TU+WX/ntneT9d6urmT9bFTTdePNbKmZHTazbf2WjTWz1WbWmd2OKbJhAMUbzNv4ZZJmfGjZQklr3P0KSWuyxwCaWMWwu/s6SUc+tHiWpOXZ/eWSZhfbFoCiVXtu/Hh3P5DdPygp98OVmbVLapekEcr//Aagvmo+Gu99R/hyj/K5+xJ3b3P3tqEaXuvmAFSp2rAfMrMJkpTdHi6uJQD1UG3YV0qal92fJ+m5YtoBUC8VP7Ob2QpJ0yVdYGb7JD0gaZGkH5jZXZJelzSnnk3izNX7dv614b+/9jeT6+66/Z+S9S+Mm53edsBx9pSKYXf3uTklzo4BziCcLgsEQdiBIAg7EARhB4Ig7EAQXEoapRm1l31NI/FqA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBB8nx2lOXnjsbJbCIU9OxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EwTh7E3iz/bPJ+slb30rWn79mSW7totZzkus+ffzCZP2vXvhysn7lfduS9dSUzVdedCi5LopVcc9uZkvN7LCZbeu37EEz229mm7KfmfVtE0CtBvM2fpmkGQMs/6a7T85+VhXbFoCiVQy7u6+TdKQBvQCoo1oO0M03sy3Z2/wxeb9kZu1m1mFmHad0oobNAahFtWF/TNLlkiZLOiBpcd4vuvsSd29z97ahGl7l5gDUqqqwu/shd+9x915J35U0tdi2ABStqrCb2YR+D78oKT3+AqB0FcfZzWyFpOmSLjCzfZIekDTdzCZLckl7Jd1dvxabX8uoUcn6/z55cbK+5ppvJOtjWj6WrPcqv94rT64759zDyfodX/jnZP3x6en/toefmJ1be+DjT6Wf+1j6uf1o+vwDfFDFsLv73AEWP16HXgDUEafLAkEQdiAIwg4EQdiBIAg7EARfcS3A7r/+dLK+49pvVXiGEcnqrlPvJuuP/vy3cmurX5iSXLf7vJ5k/bkZjyTrvz/6jXR9/qPJesonf/SVdP1oR9XPXdHUX0+WO+9Mf3X4Dz73n8n6m6fOza29ek13ct1qsWcHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAYZy9A9+j0WHUlK7rGJ+vfnzktvf09e3Nrl+mn1bT0vr+c+KVk/e9efDZZ/9Sw6v+J/cn1a5P1J+6/JVlvTZ+ekLR4wXeS9RtHnEzWt59Mj5Xf/9X23Jppc3LdarFnB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgGGcvwAU/a03WW37HkvXFj81J1j++5+Vk3a75VG7t2CfSl7k+ML03Wd9d4VLS0tAK9er92ZjOdH1Bul6LVkvvB//tl+cl64/efXuyPuTlDafdU63YswNBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEOaentK3SKNtrF9nNzdse40yZFJ6auGbf7QjWZ87ekuy3tmdf41xSbpkyPHc2oTW9HTPlbQofY5ApSmhb9qaf+33j/19eqx6/7R077fP+Y9kfe75r+TWVhy9Nrnu8s3XJ+tX/u3RZL2nc0+yXi/rfY2O+ZEB/6dV3LOb2SQzW2tmO8xsu5ktyJaPNbPVZtaZ3Y4punEAxRnM2/huSfe5+1WSrpd0j5ldJWmhpDXufoWkNdljAE2qYtjd/YC7b8zud0l6VdJESbMkLc9+bbmk2XXqEUABTuvceDO7RNIUSesljXf3A1npoKQBL6RmZu2S2iVphNLzYwGon0EfjTezcyU9I+ledz/Wv+Z9R/kGPFLj7kvcvc3d24ZqeE3NAqjeoMJuZkPVF/Sn3P2H2eJDZjYhq0+QdLg+LQIoQsWhNzMz9X0mP+Lu9/Zb/g+SfuHui8xsoaSx7n5/6rnO1qG3So7NTQ/jPPfQ4mR9XEt6COq17ndyaxvfTQ8LPrwnf7pnSfq/n6Uvcz1p9dvJ+pDNr+XWeru6kuvWquU3rszf9paddd12WVJDb4P5zH6DpDslbTWzTdmyr0laJOkHZnaXpNclpb+UDaBUFcPu7i9KuWdWxNtNA2coTpcFgiDsQBCEHQiCsANBEHYgCL7i2gR6bro6WX/nwmHJ+nnbjuQ/945dVfWEM1NNX3EFcHYg7EAQhB0IgrADQRB2IAjCDgRB2IEgmLK5CbSu3Zispy8kLfUU1wrOYuzZgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IIiKYTezSWa21sx2mNl2M1uQLX/QzPab2absZ2b92wVQrcFcvKJb0n3uvtHMRknaYGars9o33f0b9WsPQFEGMz/7AUkHsvtdZvaqpIn1bgxAsU7rM7uZXSJpiqT12aL5ZrbFzJaa2ZicddrNrMPMOk7pRG3dAqjaoMNuZudKekbSve5+TNJjki6XNFl9e/7FA63n7kvcvc3d24ZqeO0dA6jKoMJuZkPVF/Sn3P2HkuTuh9y9x917JX1X0tT6tQmgVoM5Gm+SHpf0qrv/Y7/lE/r92hclbSu+PQBFGczR+Bsk3Slpq5ltypZ9TdJcM5ssySXtlXR3HfoDUJDBHI1/UdJA8z2vKr4dAPXCGXRAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgzN0btzGzn0t6vd+iCyS92bAGTk+z9tasfUn0Vq0ie/tVd79woEJDw/6RjZt1uHtbaQ0kNGtvzdqXRG/ValRvvI0HgiDsQBBlh31JydtPadbemrUvid6q1ZDeSv3MDqBxyt6zA2gQwg4EUUrYzWyGmf23me02s4Vl9JDHzPaa2dZsGuqOkntZamaHzWxbv2VjzWy1mXVmtwPOsVdSb00xjXdimvFSX7uypz9v+Gd2M2uVtEvSb0vaJ+kVSXPdfUdDG8lhZnsltbl76SdgmNmNko5LesLdP50te0jSEXdflP2hHOPuf9EkvT0o6XjZ03hnsxVN6D/NuKTZkn5PJb52ib7mqAGvWxl79qmSdrv7Hnc/KelpSbNK6KPpufs6SUc+tHiWpOXZ/eXq+8fScDm9NQV3P+DuG7P7XZLem2a81Ncu0VdDlBH2iZLe6Pd4n5prvneX9LyZbTCz9rKbGcB4dz+Q3T8oaXyZzQyg4jTejfShacab5rWrZvrzWnGA7qOmufvVkm6VdE/2drUped9nsGYaOx3UNN6NMsA04+8r87WrdvrzWpUR9v2SJvV7fHG2rCm4+/7s9rCkZ9V8U1Efem8G3ez2cMn9vK+ZpvEeaJpxNcFrV+b052WE/RVJV5jZpWY2TNIdklaW0MdHmNnI7MCJzGykpM+r+aaiXilpXnZ/nqTnSuzlA5plGu+8acZV8mtX+vTn7t7wH0kz1XdE/jVJXy+jh5y+LpO0OfvZXnZvklao723dKfUd27hL0jhJayR1SvqJpLFN1NuTkrZK2qK+YE0oqbdp6nuLvkXSpuxnZtmvXaKvhrxunC4LBMEBOiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0I4v8BX6h9QkdHXfwAAAAASUVORK5CYII=\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"needs_background":"light"},"output_type":"display_data"}],"source":["plt.imshow(X_train[18])"]},{"cell_type":"markdown","id":"50d6d803","metadata":{"papermill":{"duration":0.006237,"end_time":"2022-07-09T03:26:40.362139","exception":false,"start_time":"2022-07-09T03:26:40.355902","status":"completed"},"tags":[]},"source":["# 4. Model"]},{"cell_type":"markdown","id":"a191d68a","metadata":{"papermill":{"duration":0.006034,"end_time":"2022-07-09T03:26:40.374626","exception":false,"start_time":"2022-07-09T03:26:40.368592","status":"completed"},"tags":[]},"source":["The orginal plan for this model was to use a pretrained model. That plan did not work, I was unable to find a pretrained model that fit these images. If you know of any please feel free to let me know in the comments. The second plan was to use data augmentation but to my surprise the training results were worst than the orginal model. The third plan was to add additonal layers to the orginal model. Adding batch normalization and dropout layers to the orginal model did improve the validaton loss and validation accuracy from the orginal model."]},{"cell_type":"code","execution_count":12,"id":"46e4d0e5","metadata":{"execution":{"iopub.execute_input":"2022-07-09T03:26:40.38883Z","iopub.status.busy":"2022-07-09T03:26:40.388499Z","iopub.status.idle":"2022-07-09T03:26:40.468659Z","shell.execute_reply":"2022-07-09T03:26:40.467868Z"},"papermill":{"duration":0.089149,"end_time":"2022-07-09T03:26:40.470088","exception":false,"start_time":"2022-07-09T03:26:40.380939","status":"completed"},"tags":[]},"outputs":[{"name":"stderr","output_type":"stream","text":["2022-07-09 03:26:40.444874: I tensorflow/core/common_runtime/process_util.cc:146] Creating new thread pool with default inter op setting: 2. Tune using inter_op_parallelism_threads for best performance.\n"]}],"source":["model = keras.Sequential([\n","    \n","    layers.BatchNormalization(),\n","    layers.Conv2D(filters=32, kernel_size=(5,5), activation=\"relu\", padding='same',\n","                  input_shape=[28, 28, 1]),\n","    layers.MaxPool2D(),\n","    layers.Dropout(.1),\n","    \n","    layers.BatchNormalization(),\n","    layers.Conv2D(filters=64, kernel_size=(3,3), activation=\"relu\", padding='same'),\n","    layers.MaxPool2D(),\n","    layers.Dropout(.1),\n","    \n","    layers.BatchNormalization(),\n","    layers.Conv2D(filters=128, kernel_size=(3,3), activation=\"relu\", padding='same'),\n","    layers.MaxPool2D(),\n","    layers.Dropout(.1),\n","    \n","    layers.BatchNormalization(),\n","    layers.Conv2D(filters=128, kernel_size=(3,3), activation=\"relu\", padding='same'),\n","    layers.MaxPool2D(),\n","    layers.Dropout(.1),\n","    \n","    layers.Flatten(),\n","    layers.Dense(units=256, activation=\"relu\"),\n","    layers.Dense(units=10, activation=\"softmax\"),\n","])"]},{"cell_type":"code","execution_count":13,"id":"9c35c09b","metadata":{"execution":{"iopub.execute_input":"2022-07-09T03:26:40.484976Z","iopub.status.busy":"2022-07-09T03:26:40.4841Z","iopub.status.idle":"2022-07-09T03:26:40.505049Z","shell.execute_reply":"2022-07-09T03:26:40.504448Z"},"papermill":{"duration":0.030047,"end_time":"2022-07-09T03:26:40.506725","exception":false,"start_time":"2022-07-09T03:26:40.476678","status":"completed"},"tags":[]},"outputs":[],"source":["model.compile(\n","    optimizer=tf.keras.optimizers.Adam(epsilon=0.01),\n","    loss='sparse_categorical_crossentropy',\n","    metrics=['accuracy']\n",")"]},{"cell_type":"code","execution_count":14,"id":"3ab3ae41","metadata":{"execution":{"iopub.execute_input":"2022-07-09T03:26:40.521571Z","iopub.status.busy":"2022-07-09T03:26:40.521053Z","iopub.status.idle":"2022-07-09T03:37:22.851483Z","shell.execute_reply":"2022-07-09T03:37:22.850507Z"},"papermill":{"duration":642.345308,"end_time":"2022-07-09T03:37:22.858899","exception":false,"start_time":"2022-07-09T03:26:40.513591","status":"completed"},"tags":[]},"outputs":[{"name":"stderr","output_type":"stream","text":["2022-07-09 03:26:40.670455: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n"]},{"name":"stdout","output_type":"stream","text":["Epoch 1/20\n","296/296 - 34s - loss: 0.5794 - accuracy: 0.8252 - val_loss: 1.0146 - val_accuracy: 0.5755\n","Epoch 2/20\n","296/296 - 32s - loss: 0.1074 - accuracy: 0.9664 - val_loss: 0.0624 - val_accuracy: 0.9824\n","Epoch 3/20\n","296/296 - 32s - loss: 0.0730 - accuracy: 0.9775 - val_loss: 0.0472 - val_accuracy: 0.9864\n","Epoch 4/20\n","296/296 - 32s - loss: 0.0580 - accuracy: 0.9819 - val_loss: 0.0394 - val_accuracy: 0.9879\n","Epoch 5/20\n","296/296 - 31s - loss: 0.0468 - accuracy: 0.9859 - val_loss: 0.0371 - val_accuracy: 0.9890\n","Epoch 6/20\n","296/296 - 32s - loss: 0.0394 - accuracy: 0.9871 - val_loss: 0.0361 - val_accuracy: 0.9898\n","Epoch 7/20\n","296/296 - 33s - loss: 0.0344 - accuracy: 0.9886 - val_loss: 0.0328 - val_accuracy: 0.9898\n","Epoch 8/20\n","296/296 - 32s - loss: 0.0309 - accuracy: 0.9902 - val_loss: 0.0327 - val_accuracy: 0.9912\n","Epoch 9/20\n","296/296 - 32s - loss: 0.0280 - accuracy: 0.9911 - val_loss: 0.0355 - val_accuracy: 0.9890\n","Epoch 10/20\n","296/296 - 33s - loss: 0.0236 - accuracy: 0.9928 - val_loss: 0.0317 - val_accuracy: 0.9914\n","Epoch 11/20\n","296/296 - 32s - loss: 0.0233 - accuracy: 0.9926 - val_loss: 0.0329 - val_accuracy: 0.9907\n","Epoch 12/20\n","296/296 - 32s - loss: 0.0215 - accuracy: 0.9927 - val_loss: 0.0358 - val_accuracy: 0.9905\n","Epoch 13/20\n","296/296 - 32s - loss: 0.0174 - accuracy: 0.9947 - val_loss: 0.0316 - val_accuracy: 0.9917\n","Epoch 14/20\n","296/296 - 31s - loss: 0.0162 - accuracy: 0.9947 - val_loss: 0.0362 - val_accuracy: 0.9907\n","Epoch 15/20\n","296/296 - 32s - loss: 0.0144 - accuracy: 0.9956 - val_loss: 0.0317 - val_accuracy: 0.9926\n","Epoch 16/20\n","296/296 - 32s - loss: 0.0145 - accuracy: 0.9951 - val_loss: 0.0307 - val_accuracy: 0.9914\n","Epoch 17/20\n","296/296 - 32s - loss: 0.0142 - accuracy: 0.9951 - val_loss: 0.0272 - val_accuracy: 0.9924\n","Epoch 18/20\n","296/296 - 32s - loss: 0.0122 - accuracy: 0.9959 - val_loss: 0.0335 - val_accuracy: 0.9910\n","Epoch 19/20\n","296/296 - 32s - loss: 0.0112 - accuracy: 0.9961 - val_loss: 0.0349 - val_accuracy: 0.9912\n","Epoch 20/20\n","296/296 - 32s - loss: 0.0109 - accuracy: 0.9966 - val_loss: 0.0331 - val_accuracy: 0.9912\n"]}],"source":["history = model.fit(\n","    x = X_train,\n","    y = Y_train, \n","    validation_data= (X_val,Y_val),\n","    batch_size = 128,\n","    epochs=20,\n","    verbose=2,\n",")"]},{"cell_type":"code","execution_count":15,"id":"6e1bc079","metadata":{"execution":{"iopub.execute_input":"2022-07-09T03:37:22.878929Z","iopub.status.busy":"2022-07-09T03:37:22.878453Z","iopub.status.idle":"2022-07-09T03:37:23.211247Z","shell.execute_reply":"2022-07-09T03:37:23.210446Z"},"papermill":{"duration":0.345362,"end_time":"2022-07-09T03:37:23.213047","exception":false,"start_time":"2022-07-09T03:37:22.867685","status":"completed"},"tags":[]},"outputs":[{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkmklEQVR4nO3de3hc9X3n8fd3brqOLFuSJXy3N75wcRKo4CFpIdcmQBPYJJsYQi6QFJ4mAUKTZcOWlFI22TTJs+Fpt2xYmhIgJQXn0qxTnLi5sKF0gdq4xsY2No5rGxlfJNnYus/tt3+cM9JYGklje6TRnPm8nuc856o5X41Gn3Pmd27mnENERMpfqNQFiIhIcSjQRUQCQoEuIhIQCnQRkYBQoIuIBESkVCtubm52S5YsKdXqRUTK0gsvvNDlnGvJN69kgb5kyRI2bdpUqtWLiJQlM9s/3jw1uYiIBIQCXUQkIBToIiIBUbI2dBGpTMlkko6ODgYHB0tdyoxWXV3NggULiEajBf+MAl1EplVHRwfxeJwlS5ZgZqUuZ0ZyztHd3U1HRwdLly4t+OfU5CIi02pwcJCmpiaF+QTMjKamptP+FqNAF5FppzCf3Jm8R+UX6PufhV/eA7rtr4jIKSYNdDN7yMyOmtlL48w3M/srM9tjZlvN7KLil5njtc3wzH0w+PqUrkZEgqu+vr7UJUyJQvbQHwaumGD+lcByv7sZ+PbZlzWBOv+K176uKV2NiEi5mTTQnXNPA8cmWOQa4FHneQ5oNLNzilXgGHXNXr+vc8pWISKVwTnHHXfcwQUXXMDq1at54oknADh06BCXX345b37zm7ngggv453/+Z9LpNDfccMPwsvfdd1+Jqx+rGKctzgdezRnv8KcdGr2gmd2MtxfPokWLzmxt2kMXCYw//+l2drx2sqived68Bv7s/ecXtOyPf/xjtmzZwosvvkhXVxcXX3wxl19+Od///vd573vfy1133UU6naa/v58tW7Zw8OBBXnrJa31+/fXXi1p3MUzrQVHn3IPOuXbnXHtLS96bhU1uONC1hy4iZ+eZZ57huuuuIxwO09raytve9jY2btzIxRdfzHe/+13uuecetm3bRjweZ9myZezdu5dbb72Vn//85zQ0NJS6/DGKsYd+EFiYM77AnzY1apu8vvbQRcpeoXvS0+3yyy/n6aef5sknn+SGG27gC1/4Ap/4xCd48cUX2bBhAw888ABr167loYceKnWppyjGHvo64BP+2S6XAiecc2OaW4omHIXqRu2hi8hZu+yyy3jiiSdIp9N0dnby9NNPc8kll7B//35aW1u56aab+MM//EM2b95MV1cXmUyGD33oQ3zlK19h8+bNpS5/jEn30M3s74G3A81m1gH8GRAFcM49AKwHrgL2AP3AjVNV7LC6FgW6iJy1D3zgAzz77LO86U1vwsz4xje+QVtbG4888gjf/OY3iUaj1NfX8+ijj3Lw4EFuvPFGMpkMAF/72tdKXP1Y5kp0gU57e7s74wdcPHQlWAhufLK4RYnIlNu5cyfnnntuqcsoC/neKzN7wTnXnm/58rtSFLxTF7WHLiJyijINdDW5iIiMVr6BPnAc0qlSVyIiMmOUaaA3Aw4GJrqAVUSkspRxoKNmFxGRHGUa6LpaVERktDIPdF0tKiKSVeaBrj10EZlaE907fd++fVxwwQXTWM3EyjPQqxvBwgp0EZEcxbg51/QLhXRxkUgQ/OxOOLytuK/Zthqu/ItxZ995550sXLiQz33ucwDcc889RCIRnnrqKY4fP04ymeQrX/kK11xzzWmtdnBwkM985jNs2rSJSCTCt771Ld7xjnewfft2brzxRhKJBJlMhh/96EfMmzePj3zkI3R0dJBOp/nTP/1T1qxZc1a/NpRroIN/cVF3qasQkTKzZs0abr/99uFAX7t2LRs2bOC2226joaGBrq4uLr30Uq6++urTelDz/fffj5mxbds2Xn75Zd7znvewe/duHnjgAT7/+c9z/fXXk0gkSKfTrF+/nnnz5vHkk97tS06cOFGU3618A722SXvoIuVugj3pqXLhhRdy9OhRXnvtNTo7O5k9ezZtbW388R//MU8//TShUIiDBw9y5MgR2traCn7dZ555hltvvRWAVatWsXjxYnbv3s1b3vIWvvrVr9LR0cEHP/hBli9fzurVq/niF7/Il770Jd73vvdx2WWXFeV3K882dNDl/yJyxj784Q/zwx/+kCeeeII1a9bw2GOP0dnZyQsvvMCWLVtobW1lcHCwKOv66Ec/yrp166ipqeGqq67i17/+NStWrGDz5s2sXr2aL3/5y9x7771FWVf57qHXtei0RRE5I2vWrOGmm26iq6uL3/zmN6xdu5a5c+cSjUZ56qmn2L9//2m/5mWXXcZjjz3GO9/5Tnbv3s2BAwdYuXIle/fuZdmyZdx2220cOHCArVu3smrVKubMmcPHPvYxGhsb+c53vlOU36uMA70ZEj2QHIBoTamrEZEycv7559PT08P8+fM555xzuP7663n/+9/P6tWraW9vZ9WqVaf9mp/97Gf5zGc+w+rVq4lEIjz88MNUVVWxdu1avve97xGNRmlra+NP/uRP2LhxI3fccQehUIhoNMq3v/3tovxe5Xk/dIAXHoGf3ga3vwSNCydfXkRmBN0PvXCVcT900MVFIiKjlHGTiy7/F5HpsW3bNj7+8Y+fMq2qqornn3++RBXlV8aB7t9xsV+BLlJunHOndY53qa1evZotW7ZM6zrPpDm8jJtcdAtdkXJUXV1Nd3f3GQVWpXDO0d3dTXV19Wn9XPnuocfqIVKtQBcpMwsWLKCjo4POTv3vTqS6upoFCxac1s+Ub6Cb6Vx0kTIUjUZZunRpqcsIpPJtcgHdoEtEJEeZB7ou/xcRyQpAoKvJRUQEyj7Qm71A19FyEZFyD/QWSA/BUE+pKxERKbnyDvRanYsuIpJV3oGuy/9FRIaVeaBrD11EJKugQDezK8xsl5ntMbM788xfZGZPmdm/mdlWM7uq+KXmoTsuiogMmzTQzSwM3A9cCZwHXGdm541a7MvAWufchcC1wP8qdqF5De+hq8lFRKSQPfRLgD3Oub3OuQTwOHDNqGUc0OAPzwJeK16JE4hUQdUs7aGLiFBYoM8HXs0Z7/Cn5boH+JiZdQDrgVvzvZCZ3Wxmm8xsU9FuzFPXrFvoiohQvIOi1wEPO+cWAFcB3zOzMa/tnHvQOdfunGtvaWkpzpp1+b+ICFBYoB8Ech/aucCfluvTwFoA59yzQDXQXIwCJ5W9WlREpMIVEugbgeVmttTMYngHPdeNWuYA8C4AMzsXL9CnZ7dZd1wUEQEKCHTnXAq4BdgA7MQ7m2W7md1rZlf7i30RuMnMXgT+HrjBTdfjSOpaoL8bMulpWZ2IyExV0AMunHPr8Q525k67O2d4B/C7xS2tQHUt4DIwcHzkNEYRkQpU3leKgq4WFRHxBSDQdbWoiAgEKtB1pouIVDYFuohIQJR/oNfMBkxNLiJS8co/0ENhqG1SoItIxSv/QAdd/i8iQmACXZf/i4gEJNC1hy4iEpxA1y10RaTCBSfQB09AKlHqSkRESiYgge5f/q+9dBGpYMEKdLWji0gFC0ig634uIiIBC3Q1uYhI5QpIoKvJRUQkGIFe1QDhmAJdRCpaMALdzL+4qLvUlYiIlEwwAh30sGgRqXgBCnRd/i8ilS04gV6rG3SJSGULTqBnm1ycK3UlIiIlEaBAb4HUACT6Sl2JiEhJBCvQQe3oIlKxAhjoakcXkcoUoEDXHRdFpLIFKNDV5CIilS1Aga77uYhIZQtOoEdrIFavNnQRqVjBCXTQ5f8iUtECFui6/F9EKldBgW5mV5jZLjPbY2Z3jrPMR8xsh5ltN7PvF7fMAtW1qMlFRCpWZLIFzCwM3A/8PtABbDSzdc65HTnLLAf+K/C7zrnjZjZ3qgqeUF0zHNxcklWLiJRaIXvolwB7nHN7nXMJ4HHgmlHL3ATc75w7DuCcO1rcMgtU1+Kdh57JlGT1IiKlVEigzwdezRnv8KflWgGsMLN/MbPnzOyKfC9kZjeb2SYz29TZOQVt3XUtkEnB4OvFf20RkRmuWAdFI8By4O3AdcDfmFnj6IWccw8659qdc+0tLS1FWnWO2uy56GpHF5HKU0igHwQW5owv8Kfl6gDWOeeSzrl/B3bjBfz00sVFIlLBCgn0jcByM1tqZjHgWmDdqGV+grd3jpk14zXB7C1emSN+sOlV3nvf06Qzee57rsv/RaSCTRrozrkUcAuwAdgJrHXObTeze83san+xDUC3me0AngLucM5N2RObdx3pYX93nvueK9BFpIJNetoigHNuPbB+1LS7c4Yd8AW/m1Ir2+IA7D7Sw7KW+lNn1jZ5fbWhi0gFKrsrRd8wtx4z2HW4d+zMcARq5ugWuiJSkcou0GtjERbNqWX3kZ78C+jyfxGpUGUX6AArWuO8fPhk/pm6/F9EKlRZBvrK1jj7uvsZTKbHzqxr0h66iFSk8gz0tjjpjGNv5zhnuijQRaQClW2gA/nb0etaYOA4pJPTXJWISGmVZaAvaaojGjZ25Q307MOip+w0eBGRGaksAz0WCbGsuZ7dh8fZQwc1u4hIxSnLQAdY0RYfZw89G+g600VEKkvZBvrK1no6jg/QO5Q6dYYCXUQqVNkG+opW78DoK6P30nXHRRGpUGUb6NkzXXaNbkevboRQRIEuIhWnbAN94exaaqLhse3oZt6DLhToIlJhyjbQQyFjRWv9+Oeiqw1dRCpM2QY6eO3oee+6WKc9dBGpPGUd6Cvb4nT1DtHdO3TqDF3+LyIVqKwDPXumy+4jo/bS61p0paiIVJyyDvRx7+lS1wyJXkj0l6AqEZHSKOtAnxuvYlZNdOyZLtmLi/TkIhGpIGUd6GbGytb42Hu66OIiEalAZR3oACva6tl1pAfvOdU+Xf4vIhWo7AN9ZVsDPYMpDp0YHJmoPXQRqUDlH+j+mS6ntKPrFroiUoHKPtBXtNYDnNqOHquDaK2aXESkopR9oDfWxmhtqMpzpkuzAl1EKkrZBzp4FxiNPRddV4uKSGUJRKCvbI3zypFe0plRZ7oo0EWkggQi0Fe0xRlKZThwLOfK0Fo1uYhIZQlEoA+f6ZJ7YDR7x8Xc89NFRAIsEIG+vLUes1H3dKlrgUwSBk+UrjARkWlUUKCb2RVmtsvM9pjZnRMs9yEzc2bWXrwSJ1cbi7BoTu2oPXRdLSoilWXSQDezMHA/cCVwHnCdmZ2XZ7k48Hng+WIXWYgVrfFRFxf5V4vqBl0iUiEK2UO/BNjjnNvrnEsAjwPX5FnuvwFfBwbzzJtyK1vj/HtXH0OptDdBV4uKSIUpJNDnA6/mjHf404aZ2UXAQufckxO9kJndbGabzGxTZ2dxg3ZFW5x0xrG3s8+boEAXkQpz1gdFzSwEfAv44mTLOucedM61O+faW1paznbVp1jZOuphF7VNXl9t6CJSIQoJ9IPAwpzxBf60rDhwAfB/zWwfcCmwbroPjC5triMSspEDo5EYVM/SHrqIVIxCAn0jsNzMlppZDLgWWJed6Zw74Zxrds4tcc4tAZ4DrnbObZqSiscRi4RY1lI39tRFBbqIVIhJA905lwJuATYAO4G1zrntZnavmV091QWejrFnurSoyUVEKkakkIWcc+uB9aOm3T3Osm8/+7LOzKq2OP+49RB9QynqqiLeqYtdr5SqHBGRaRWIK0WzVow+MKo9dBGpIIEK9JVteQK9vxsy6RJWJSIyPQIV6Atn11IdDbHrcK83oa4FcNB/rKR1iYhMh0AFeihkpz7sQg+LFpEKEqhAh1FnutQq0EWkcgQu0Fe2xunsGeJYX0KX/4tIRQlcoK/IPTCqW+iKSAUJXKCvyg30mtlgId1CV0QqQuACfW68ilk1Ue+eLqGQ/2xRNbmISPAFLtDNjJWt8ZGbdOniIhGpEIELdIAVbfXsOtKDc27kYdEiIgEXyEBf2RqnZzDF4ZODCnQRqRiBDPTsPV12He5Rk4uIVIxAB7p36mIzDJ2EZEkedSoiMm0CGeiz62LMjVd593TJnouuUxdFJOACGejg3XlRFxeJSCUJbqC3xnnlaA/pmuz9XBToIhJsgQ30FW1xBpMZDqXqvQk600VEAi6wgb7SPzD6ck+VN0GBLiIBF9hAX97q7Znv6MpAuEqBLiKBF9hAr41FWDSnll1He3UuuohUhMAGOnjno+8+3KOrRUWkIgQ60Fe21fPvXX2kdcdFEakAgQ70Fa1xUhlHb7gR+rtLXY6IyJQKdKCvamsA4GimwdtDd67EFYmITJ1AB/rS5joiIeNgsg5Sg5DoLXVJIiJTJtCBHouEWNZSx2/7ar0JakcXkQALdKCD147+ck/MG9GpiyISYIEP9JWtcXae1NWiIhJ8gQ/0FW1xut0sb0SBLiIBFvhAX9ka5xjefV3U5CIiQVZQoJvZFWa2y8z2mNmdeeZ/wcx2mNlWM/uVmS0ufqlnZuGcWixazWC4ToEuIoE2aaCbWRi4H7gSOA+4zszOG7XYvwHtzrk3Aj8EvlHsQs9UOGSsaI3zus1Sk4uIBFohe+iXAHucc3udcwngceCa3AWcc0855/r90eeABcUt8+ysaI1zJB1XoItIoBUS6POBV3PGO/xp4/k08LN8M8zsZjPbZGabOjunL1xXtsY5nIqT6lGgi0hwFfWgqJl9DGgHvplvvnPuQedcu3OuvaWlpZirnpB3pkucTO/RaVuniMh0KyTQDwILc8YX+NNOYWbvBu4CrnbODRWnvOJY2Rqnm1lEBo9BJlPqckREpkQhgb4RWG5mS80sBlwLrMtdwMwuBP43XpjPuN3g1oYq+iKNhMjAwPFSlyMiMiUmDXTnXAq4BdgA7ATWOue2m9m9Zna1v9g3gXrgB2a2xczWjfNyJWFmVM1q9Ub6deqiiARTpJCFnHPrgfWjpt2dM/zuItdVdA1N8+AEuN6jWMvKUpcjIlJ0gb9SNGtum3dizutdh0pciYjI1KiYQJ83fxEAXUc6SlyJiMjUqJhAX7ZoERlnnNQeuogEVMUE+ux4DScsztCJI6UuRURkSlRMoAP0RWbr8n8RCayKCvRUTROxoW4yGT0sWkSCp6ICPVTfwmx3kgPH+idfWESkzFRUoNfMbqPJTrLrSE+pSxERKbqKCvRZTfNotD72vHas1KWIiBRdRQV6bNZcAF47pHPRRSR4KirQqfNu2dt9dMzNIkVEyl5lBXptMwCDx4/wd8/tp28oVeKCRESKp7IC3d9Dv6AxwZd/8hKX/vdf8ec/3c7ezt4SFyYicvYKuttiYNR5e+hf/N3ZvGP+W3n02X383XP7+e6/7ONtK1r45FsX87YVcwmHrMSFioicvsoK9OpZEIpi/d38zuLZ/M7i2dz1B+fy+L++ymPP7+dTD29i0ZxaPn7pYj7cvoDG2lipKxYRKVhlNbmYec0uOZf/z41Xc9u7lvPMl97JX3/0Qtoaqvnq+p1c+rVfceePtrLjtZMlLFhEpHCVtYcOXrNL39inFkXDId73xnm8743z2PHaSR59dh8/2XKQxze+yiVL5vCJty7mvee3EQ1X1jZQRMpHBQZ6y6Q36DpvXgN/8aE3cueVq/jBpg4efW4ft3z/32htqOKjlyzmuksWMrehepoKFhEpTGUGevcrBS3aWBvjpsuX8anfW8pvdh/lkf+3n/t+uZv7frmb+Y01nDevgXPPaeA8v1swu4aQDqiKSIlUYKDnb3KZSDhkvHNVK+9c1crezl5+vv0wO147yc5DJ/nVziNkb95YXxXh3HPinHvOSNCvbItTHQ1PwS8iInKqCgz0Fkj2w3feDW2rva51NbSeB7G6SX98WUs9n337G4bHBxJpdh3pYechL+B3vHaSH28+SO/QfgBCBkub6zhv3qyRsG9roLWhCjPtzYtI8Zhzpbk3eHt7u9u0adP0r7jnMPzLX8KhrXB4Gwyd8GcYNL1hJOTb3uj1462nvYpMxtFxfIAdh06w41DPcNAffH1geJmaaJhFc2pZ1FTL4jm1LG6qZXFTHYubapnXWKODryKSl5m94Jxrzzuv4gI9l3Pw+gEv2I+85PUPb/WmZdW1jA35pjdA6PSbUU4MJHn5kHf73v3d/ezv7ufAsT72d/czlMoMLxcOGfMba/yQr2XxnDov+JtqWTSnltpY5X2xEhGPAv10DbzuB3xOyHe+DOmENz8Uhfq5UN/qdfHWkeH6Voi3jcyPVE26ukzGcbRniP3dfew/1s+B7n72dfdx4JgX+icGkqcs31xfRUu8iub6GC31VTTHq/x+jOb6quFuTl1MV72KBIwCvRhSCeja7QV958vQe9Rrvuk9Cr2H/QOted7L6sacgPf7dS0QqfbCPlIF4Zg/XJ0zXAVhr38iGaLjZJr9J9Psez3F/uMpuvoSdPUO0dWboLN3iETOHn5WyGBOXW7Ie8NN9VXMqonSUBOhoTpKQ02UhuqI348Si6i5R2SmmijQ9d29UJEYtF3gdfmkU9757b2Hx4Z97xHoOQKvPu8NpwZPa9Wz/O783IkWAgtDKIyrCUMoRIYwGUJkCJHGSLkwKYxkb4hkj5HMGImMkXAhBqhi0MUYoIpuqhhwMQapop8qkqFqXKQWi9VgsVpCVXVEquqIVtcRq6mnqrqO6liE6miI6miE6mjY62JharLDfhcLh0YO/mb7znnvQXIQUgOn308nIVoDsXq/q/O6qtzx0fPiXj8cG6kjyzlwGcikcrr0qPGcaekkpIa8g+upQUgOjOr3j1O7Py9bf1Xcq7EqPmq4HmLxnOF6qGrwhrPf+Jzzahg6CUM9Xn8wZ3jcaT2Q6IVorf+a8Zxu1Hh1w9hp0ToIldkG3znvd+7vhv5jMHDM20HLpCCT9P6u6WSB4/7nIBSGcNT7PIWiI8Nhfzg0ajw7nJ0+ewnUtxT9V1WgF0s4Ag3neN1EnINEn/fPmB7y+wnvnz2V8KflDvvz0glv2exwJg0uDZk05jKQSRN2acI503OXyfadS5NOJkkn+skM9eGSPVjiCJYaIJQeJJIaIEQaMsCg35VQ2iJkwtVeF6nGRaqxcJRweoBwsp9Qqg9L9BX+gqGIF2ajA3wqRGogWu33/S77Laz3KHT/1guaoV5IFvg7hGPe6yT6vbCZtIbqsWHdMN/b+PR3w/F9I2GfLORZu+ZtYMKR4R0KLOztYIRCp04b7o+eHvHel2itt5HN9oeHa70Nx3C/7tRpkWqv5v5jftfthXR/90ho5/YHjo00l54pC3lhHIp4XTbsz/R1/+BbcPGnz66mPBTo083M29Oqqi/N6vH+6OP+4Z3z9kaS/X434PUT3rhL9pMc6GMonWEolSGRTJNIpUmk/PF0hmQq5Q3ndMl0iqGUGx7vy0TozUTpTUfoSUXoSUfpSUU4kQrTm44ySIwhoqSZ/OBzyDI0xzK0VCVpiiVpiiaZE00wO5xgVjhBPDxEQ2iIegaptUFqSOAshAtFwCK4UBhnEe/bTnY8NDJOKILLBlHI+7lQpGb4G0w4VkMoVkM4VkukqoZwVR2RWDWxSJhIOEQ0bERDoYkvOsuk/XDv8QI+0esHbW/OdL9LDnjBNhzUDTl71KOmRU7jBnPpFCR6Tl3XUA8Mnhg7LZMatbOQ8TaSbvSORCbvjgX9xyDxqv/Z6hv5tnM2LAQ1c6B2DtQ2wZylMP8ibzg7rbYJamZ733SyAR2Oehub8cb9v3tezo18a0sn/D35nOHxpresPLvfdRwKdDmVmRcCkRjUNI6dDcT8Lj5FJaQzjsFk2utSmZHhZIahZJqBZJreoRR9Q2l6h5L0DqboGUrRO5iiL5GiazDFvqEUvX0p+ob8eUMppuZwUb/fTS4cMi/cwyHqYhHqqsLUV0Wo8ztvOOzPm0VdVRP1VWFvfqM/P+b1I2EjEjLCISMSChEOG2HLjtuZXbEcjnhhVzP79H+2GDLpnJ2HPq+f6BsZzoZ/atDbYOUGdc1s73jVdDcHmY00q1A7vevOQ4EuM044ZMMhVyzOOfoT2Q1Byr+613nN5/jN6NnxnGFGzwMyzpHOOJL+N5JU2pFMZ0j601KZDIm0I5XOeNOz8/1lh1IZ+hPZDZJXz7G+fvpypuU7yH06zDg18P3hbOBHwyEiYSPm96PhkN+NHY6EQsQiI8PRsHlb9pwNZHYwe5KFyztvbI2W7ZthAAaG+fMimM3CaBxeFjNCBtGBENEer6ZYJEQ03EcsPJAz7v1u0UjI64dHfgfv9/A2etl+dmMY9ofL9RYeBf3HmNkVwF8CYeA7zrm/GDW/CngU+B2gG1jjnNtX3FJFzpxZ8TcSUymZztDnf7PIDf6+oRR9iTTpTIZUxtuwpNJ+P+OGp2eGx3P7meHlUxnnb4xO3eAMJjP0DqZI+NOy80cvm5V7fNmP5OFpuZGYPTCeneZtRN3Yjak/052ysR1ZbrqYMRzsYcsJfr8b3hANb3xyfkfLP29kgwW3v3sF73/TvKLXPemn28zCwP3A7wMdwEYzW+ec25Gz2KeB4865N5jZtcDXgTVFr1akQkTDIRprY3rISh7ZjZV3vMbbECXS2WM1btS4Pz/lT/enpTJu+JvWcOcc6bTXH94g+sPpDKQzGW+ZzNhvcrkbJvJsiHLHcdBYG52S96aQ3ZVLgD3Oub0AZvY4cA2QG+jXAPf4wz8E/trMzJXqJHcRCaxQyIiFzLteYvLr9ipKIUcQ5gOv5ox3+NPyLuOcSwEngKbRL2RmN5vZJjPb1Nk58T3JRUTk9EzrIWHn3IPOuXbnXHtLS/FPqhcRqWSFBPpBYGHO+AJ/Wt5lzCyCd2FjdzEKFBGRwhQS6BuB5Wa21MxiwLXAulHLrAM+6Q//J+DXaj8XEZlekx4Udc6lzOwWYAPeaYsPOee2m9m9wCbn3Drgb4Hvmdke4Bhe6IuIyDQq6KRc59x6YP2oaXfnDA8CHy5uaSIicjrK7LZpIiIyHgW6iEhAlOwBF2bWCew/wx9vBrqKWE6xqb6zo/rO3kyvUfWducXOubznfZcs0M+GmW0a74kdM4HqOzuq7+zN9BpV39RQk4uISEAo0EVEAqJcA/3BUhcwCdV3dlTf2ZvpNaq+KVCWbegiIjJWue6hi4jIKAp0EZGAmNGBbmZXmNkuM9tjZnfmmV9lZk/48583syXTWNtCM3vKzHaY2XYz+3yeZd5uZifMbIvf3Z3vtaawxn1mts1f96Y8883M/sp//7aa2UXTWNvKnPdli5mdNLPbRy0z7e+fmT1kZkfN7KWcaXPM7Bdm9orfz/sUZTP7pL/MK2b2yXzLTEFt3zSzl/2/3z+YWeM4PzvhZ2GKa7zHzA7m/B2vGudnJ/x/n8L6nsipbZ+ZbRnnZ6flPTwrzrkZ2eHdCOy3wDK8h8y/CJw3apnPAg/4w9cCT0xjfecAF/nDcWB3nvreDvxjCd/DfUDzBPOvAn6G95jDS4HnS/i3Pox3wURJ3z/gcuAi4KWcad8A7vSH7wS+nufn5gB7/f5sf3j2NNT2HiDiD389X22FfBamuMZ7gP9cwGdgwv/3qapv1Pz/AdxdyvfwbLqZvIc+/Og751wCyD76Ltc1wCP+8A+Bd5nlPrZ26jjnDjnnNvvDPcBOxj7Jaaa7BnjUeZ4DGs3snBLU8S7gt865M71yuGicc0/j3TE0V+7n7BHgP+b50fcCv3DOHXPOHQd+AVwx1bU55/7JeU8JA3gO73kFJTPO+1eIQv7fz9pE9fnZ8RHg74u93ukykwO9aI++m2p+U8+FwPN5Zr/FzF40s5+Z2fnTWxkO+Ccze8HMbs4zv5D3eDpcy/j/RKV8/7JanXOH/OHDQGueZWbCe/kpvG9c+Uz2WZhqt/jNQg+N02Q1E96/y4AjzrlXxplf6vdwUjM50MuCmdUDPwJud86dHDV7M14zwpuA/wn8ZJrL+z3n3EXAlcDnzOzyaV7/pMx7aMrVwA/yzC71+zeG8757z7hzfc3sLiAFPDbOIqX8LHwb+A/Am4FDeM0aM9F1TLx3PuP/n2ZyoM/4R9+ZWRQvzB9zzv149Hzn3EnnXK8/vB6ImlnzdNXnnDvo948C/4D3tTZXIe/xVLsS2OycOzJ6RqnfvxxHsk1Rfv9onmVK9l6a2Q3A+4Dr/Q3OGAV8FqaMc+6Icy7tnMsAfzPOukv6WfTz44PAE+MtU8r3sFAzOdBn9KPv/Pa2vwV2Oue+Nc4ybdk2fTO7BO/9npYNjpnVmVk8O4x38OylUYutAz7hn+1yKXAip2lhuoy7V1TK92+U3M/ZJ4H/k2eZDcB7zGy236TwHn/alDKzK4D/AlztnOsfZ5lCPgtTWWPucZkPjLPuQv7fp9K7gZedcx35Zpb6PSxYqY/KTtThnYWxG+/o913+tHvxPrwA1Xhf1fcA/wosm8bafg/vq/dWYIvfXQX8EfBH/jK3ANvxjtg/B7x1Gutb5q/3Rb+G7PuXW58B9/vv7zagfZr/vnV4AT0rZ1pJ3z+8jcshIInXjvtpvOMyvwJeAX4JzPGXbQe+k/Ozn/I/i3uAG6eptj14bc/Zz2D2rK95wPqJPgvT+P59z/98bcUL6XNG1+iPj/l/n476/OkPZz93OcuW5D08m06X/ouIBMRMbnIREZHToEAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiATE/we25wWWLIDETAAAAABJRU5ErkJggg==\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"needs_background":"light"},"output_type":"display_data"},{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAktElEQVR4nO3deXRc5Z3m8e+vFlVpsSV5wTvY0AaMMY6xMSR0gxuHbkgTSOg2y6GZxElg0mk4LNNNOyQBJsn0pCeZyQlzSLrNDBC6Q0gCQ0IYQgIBhpwGbMwSFpvFQQbLeJElS5ZUkmp75497qyjLWsp2SaW69XzOqVN3eavq1a2qp65+de9b5pxDREQqX6jcHRARkdJQoIuIBIQCXUQkIBToIiIBoUAXEQmISLkeeNq0aW7+/PnlengRkYr04osv7nXOTR9qXdkCff78+WzatKlcDy8iUpHM7L3h1qnkIiISEAp0EZGAUKCLiATEqIFuZneZ2R4ze32Y9WZmt5vZVjN71cxOLX03RURkNMXsod8DnDfC+vOBhf7lauAHR94tERE5VKMGunPuGaBjhCYXAfc6z/NAk5nNKlUHRUSkOKWooc8BthfMt/rLRERkHI3rcehmdjVeWYajjz56PB9aRI6Qc45UxpHOZgmZETIjEjJCISt31w6QzTpS2SypjCOVzpLKZBnwr1MZRyqTJescWQdZ53DO4Rz5+Wx+/uDrfJusI+McmazXPpP1HjftL89mC9d5y7MF61YvmsHSeU0l/9tLEeg7gHkF83P9ZQdxzq0H1gOsWLFCA7GL+LJZR28yTXe/d+kZSLE/N92fprs/5S/3LkfyMwYOL5iT6QzJdJZkJksq7RjIZL35dMZf760rvB5OLtgjISPsXyKhA0O/8No4+EPADuFzIZ31gjmVzpL0/5ZcWKezEz9ajpocn7CB/jBwjZndD5wOdDnndpbgfkWGlQvARDJDz0CaxIB/nUz71xl6B9L0DmQOWNbflyCU3E/GhciGImQtQsaiOAthZpgZ3g6nYQaGFzQhy837C50XjADO+RdcPmidv5yDljkyDnoHBgV18uCQNrLESBEnSZwkdaEkzTVZmqJZMhYlSYQkNaT86RRRkkTJEB41HWORENFwiJpIiJqwMSmUYkZNH5Nr+mkMJWggQQN9NJCgzvVRR4K6bC+12QQ12T6yFiEVqiFlMZIWIxWqYcBipPCuk9QwYDUM4F2S1NCfu7goYZch4pJEXIqoSxL2p735VMHyFBGXzC+LuBRhlyYbqSETi5MJ15GJ1OIitWSjtbhoPS5Si4vWYTX1WLQWYvWEauqIRqLUREL5D538c+o/57n5kHkfN6GQt9z8/0ZC/vNvRv5DK2T+B5gZ4bB3HQrhzRd+kBW0G8v/aEYNdDP7MbAKmGZmrcCtQBTAOffPwKPAJ4CtQAJYO1adlQLZLGRTkElCJgXZ9IfTmZS/bpjpbPoIHzsDqT5IJbxLMkE2mSDd30N6oIfMQAI30ItL9kIqgaX7CKUShNJ9RDL9hF2SgVA9feEGEqEGEqF6eq2BHqun2+rpdvXsp4791NGZraPL1bMvU0tHtpaOdC1dmSjhVA9N1ksjPTRZD0300mQ9NNJDo/XSRC/zzFvXbL00WS+T6SFOctg/K0WENBEyFiZFlDTh/HSGMGnCXhuL4DAcIbL+NQZZQgcutxBg3jIzf30IA2pDKWotRW04Saw+SawuSY1LEnEDRLIDRDL9hLLD9DU12hNkEIlDpAbCMYjEIFzz4bJQFJK9MNANvfu9a5cZ/XmP1kFsEtTUQyYN6T5I9XvXR/qaKpaFvL8lPQAc4p54OAY1dd7fgYHLehfch9MuW/AJPcL6I/UX34EVnzvy+xlk1EB3zl0+ynoH/G3JelTJshnvjVIQdN5174cBmJsufDMMed3vtyu4Tvd/uK6YN+A4SrkIfcRIEKPPxQZNN5Fghj9dQzYUZbLrpzHbR6P1MokEU2jjaNdLg+slzsDIDxb2L8PIRmpx8WaobSJUNxWr/SOobf7wEpvkvSkzSf/DzvswjGZTRA/4EEx66wqnc7cZ/KY/4I2fGT4ccu0ice8SbfSv4wXLake+jsT8D+ikF2yZAUgn/euBIZYVrMskvUvDUd52iE32r/1LvLFgftC6cHT4jT444AdfpwcOfB2HogUfNIXXMe9DJxIfel3YjyznvPtJJiDlv6fy772+od+H+ek+7z683XPvQyJ3YdD84Db480OUjA7JzKVHdvthlG1wrgnPOejvhK5W79K5Hbq2e9M9uyHZc/ALJTNKEA0lHPPfzLWDruMQm4RrmE7KYiSyUbozUfanw3Snw/RnIwy4MAPZEAPZEP3ZMH3ZEP2ZEH3ZMH0Zoy8TIpEJkUgbPekQKeftYWb8PcnRmEE8EiIWjRCPhohHwsSi3nyopp5ovI5wvIGa2npqYzEaYhHqaiLUx8LU10Soj0VojoWZG4v482HqaiKER/uXMz0A/fuhv8u/dBZMd3lv1vjkA0M6d4k3EYrGD/15kCMTjkDYD/7xYOZ9wEVrganj85gVoHoDPZOG7p1+YBeEdef2D0M82X3gbcI10DgXJs2Chpn+v2/13osqN11T57/Q6kdYX5ffG0s72NM9wI7OPj7o7GNHZx879nnTH7T3s6Ozj56BA/+dDYeM2miYeDRELOJdx6NhamNh4rnl0TDxSJjJ0RBH+cu824SJRcPURcPU1niXuqgXtLU1IWprIvl1sYhXVx53kRg0TPcuIlK06gv0956Fh/4jdO04uGxRO8UL7KnHwbFnQ+M8b75xHjTNg7ppEDq0Q/fTmSw7u/rZ3pFg+74E2zt6ad3XxgedXljv2t9PZtC38s11UWY31XLM1Do+etxU5jTVMqe5ltlNtcxuijOtPjbhDhUTkfKrvkBv+R10vg9/8ndeSOcCu3Gu92XPIcpmHW09A2zvSNC6r68guPvYvi/Bzq4DAzscMmZOjjOnuZbTF0zxQ9oL7DlNcWY31VJXU31Pi4gcuepLjsReiDfB6q8d+k2Taf7fW208+4d23veDe8e+PgYGHZ87fVKMec21LD+mmXnNdcybUutf1zGrMU4krEEuRaT0qi/Qe9ugvvjabHd/iiff3MOvXtvF02/voT+VpSEW4ZipdRx/1CRWn3gU86bU5YN7bnMd8egIh2CIiIyRKgz0vVA/bcQmnYkkj2/ezWOv7+J37+wlmcly1KQYa5bP4/yTZ7JywRTtZYvIhFOdgT7tjw5a3NY9wG827+Kx13fx3B/aSWcdc5pqufKjx3D+yTM59ehmfREpIhNa9QV6Yi/UnQHArq5+Hnt9J796fRcvbOsg62D+1DquOutYzj95JkvmNJbnsD0RkcNQXYGezeIS7bzcHuYb3/93Xn6/E4DjZzRwzTkLOf/kmZw4c5JCXEQqUnUFet8+zGX5xTspUjOz/P2fn8B5J8/kuOkN5e6ZiMgRq65A720DIN54FI9c+ydl7oyISGlV1aEa6R4v0GfNnjdKSxGRylNVgb699X0Ajj3mmDL3RESk9Koq0Hf4gb5o4bFl7omISOlVVaB37PF+GW/69Nll7omISOlVTaBns45E5256w40fDpIvIhIgVRPob+3uZlKmk0ytBsMXkWCqmkDf2NLBVOumZvJR5e6KiMiYqJ5A39bBjHA3sUYFuogEU1UEunOOjS0dTA91Y4cwdK6ISCWpikDf1p6gvbuP+sz+UYfOFRGpVFUR6Btb2mmmG8Md0o9biIhUkqoI9A0tHRxb1+/N1OkoFxEJpqoI9I0tHXxspv+7n9pDF5GACnyg7+jso3VfHx+ZmvYWqIYuIgEV+EB/oaUDgBMmDXgL6hToIhJMgQ/0DS0dTIpFmBnpAQzqppS7SyIiYyLwgf7Ctg5WzG8mlNjrhXkoXO4uiYiMiUAH+t6eAbbu6WHlgqnQu1dfiIpIoAU60Ddt8+rnKxdM8QJd9XMRCbBAB/qGlg7i0RBL5jRCYq+OcBGRQAt0oG9s6eDUo5upiYS8H4hWoItIgAU20Pf3p9i8c79XbsmkoW+faugiEmiBDfQXt+3DOVg5fwok2r2FOu1fRAIssIG+oaWDSMhYdnSzVz8H7aGLSKAFNtBf2NbBKXMbqa0Je0e4gGroIhJogQz0vmSGV1s7vePPwftCFLSHLiKBVlSgm9l5ZvaWmW01s3VDrD/GzH5rZq+a2dNmNrf0XS3ey9v3kco4Tl/gn+afr6FrD11EgmvUQDezMHAHcD5wEnC5mZ00qNl3gHudc6cAXwf+a6k7eig2tnRgBsvnN3sLetvAQlDbXM5uiYiMqWL20FcCW51z7zrnksD9wEWD2pwEPOlPPzXE+nG1saWDk2ZNZnI86i3o3esd4RIKZIVJRAQoLtDnANsL5lv9ZYV+D1zsT38amGRmZTlGMJnO8tL7+zhtfsGoir1tKreISOCVapf174Czzexl4GxgB5AZ3MjMrjazTWa2qa2trUQPfaDXdnTRn8p+WD8Hr4auI1xEJOCKCfQdwLyC+bn+sjzn3AfOuYudc8uAr/jLOgffkXNuvXNuhXNuxfTpY3PEyUb/By1OWzBoD12BLiIBV0ygvwAsNLMFZlYDXAY8XNjAzKaZWe6+vgzcVdpuFm9jSzvHTa9nWkPsw4UaOldEqsCoge6cSwPXAL8GtgA/dc69YWZfN7ML/WargLfM7G1gBvBfxqi/I8pkHZve2/fh8ecAmRT0d6qGLiKBFymmkXPuUeDRQctuKZh+AHigtF07dG/u2k93f/rg+jmo5CIigReo4/hy9fOVg+vnoEAXkcALXKDPba5ldlPthwt7NTCXiFSHwAS6c46NLR3ecLmFcoGuGrqIBFxgAv0Pbb209yYPLLdAwdC5CnQRCbbABPqQ9XPwx3EJQ7xp/DslIjKOAhTo7UxriLFgWv2BK3r9H4fWOC4iEnCBSbmNLR2cvmAKZnbgikS76uciUhUCEeit+xJ80NV/cLkFdNq/iFSNQAT6sPVz+LDkIiIScIEJ9MnxCCfMmHTwSo3jIiJVIjCBftr8KYRCg+rn6SQMdKmGLiJVoeIDfU93P+/u7R263JI/Br0sv7UhIjKuKj7QX2jZB4xQPweVXESkKlR8oG9saac2GubkOY0Hr8wNzKWSi4hUgYoP9A0tHSw/pploeIg/JT90rvbQRST4KjrQuxIp3trdPXS5BQqGzlUNXUSCr6IDfdN7HTg3TP0cvBp6KKJxXESkKlR0oG9s6aAmHOIj85qGbtDb5tXPBw8HICISQBUd6BtaOjhlbiPxaHjoBol21c9FpGpUbKD3DqR5fUfX8OUW8MdxUf1cRKpDxQb6y+93ks66UQJdp/2LSPWo2EDf2NJOyGD5Mc3DN+rdq2PQRaRqVGygb2jpYPHsRibFo0M3SA9AslsjLYpI1ajIQB9IZ3h5e+fo5RZQoItI1ajIQH+ttYtkOjv6F6KgGrqIVI2KDPQN/g9anDZ/hEDPjbSoGrqIVImKDPSNLR0sPKqBKfU1wzdSyUVEqkzFBXo6k+XF9/aNXG4BBbqIVJ2KC/QtO7vpGUgXEehtEIpCbPL4dExEpMwqLtA3tHhD4o4a6An/pCKN4yIiVaLiAv2PF07jq3+xiFmNtSM37N2r0/5FpKpEyt2BQ3XizMmcOLOIMopO+xeRKlNxe+hFyw2dKyJSJYIb6Bo6V0SqTDADPdUHyR7V0EWkqgQz0PPHoGsPXUSqR0AD3R/HRTV0EakiwQz0hHesuvbQRaSaFBXoZnaemb1lZlvNbN0Q6482s6fM7GUze9XMPlH6rh6CfMlFNXQRqR6jBrqZhYE7gPOBk4DLzeykQc2+CvzUObcMuAz4fqk7ekg0dK6IVKFi9tBXAludc+8655LA/cBFg9o4IHe2TyPwQem6eBgSeyEcg5qGsnZDRGQ8FXOm6Bxge8F8K3D6oDa3Ab8xs2uBeuDjJend4erVOC4iUn1K9aXo5cA9zrm5wCeAfzWzg+7bzK42s01mtqmtra1EDz0EjeMiIlWomEDfAcwrmJ/rLyv0eeCnAM6554A4cNAxg8659c65Fc65FdOnj2F9u7dN9XMRqTrFBPoLwEIzW2BmNXhfej48qM37wGoAM1uEF+hjuAs+isReHYMuIlVn1EB3zqWBa4BfA1vwjmZ5w8y+bmYX+s3+E3CVmf0e+DHwWeecG6tOj6p3r36pSESqTlHD5zrnHgUeHbTsloLpzcCZpe3aYUr2QiqhQBeRqhO8M0VzJxWp5CIiVSZ4gZ7QwFwiUp2CF+j50/61hy4i1UWBLiISEAEMdA2dKyLVKXiBntgLkVqoqS93T0RExlXwAj13DLrGcRGRKhPcQBcRqTLBC3Sd9i8iVSp4gZ4bOldEpMoEK9Cd09C5IlK1ghXoyV5I92kPXUSqUrACPaFxXESkegUr0HWWqIhUMQW6iEhABCzQddq/iFSvYAV6QnvoIlK9ghXovXshWqdxXESkKgUv0LV3LiJVKmCB3qb6uYhUrWAFekKn/YtI9QpWoKvkIiJVLDiBnh/HRYEuItUpOIGe7IHMgGroIlK1ghPouZOKVEMXkSoVoEBv965VchGRKhWgQM/toSvQRaQ6BSfQNXSuiFS54AS69tBFpMoFKNDboaYBorXl7omISFkEKNDboE6/JSoi1Ss4ga7T/kWkygUn0HvbVD8XkaoWoEBvV6CLSFULRqA7p6FzRaTqBSPQB/ZDNqUauohUtWAEeq9+S1RERIEuIhIQRQW6mZ1nZm+Z2VYzWzfE+u+a2Sv+5W0z6yx5T0eSO0tUNXQRqWKR0RqYWRi4AzgXaAVeMLOHnXObc22cczcUtL8WWDYGfR1ebhwX1dBFpIoVs4e+EtjqnHvXOZcE7gcuGqH95cCPS9G5oqnkIiJSVKDPAbYXzLf6yw5iZscAC4Anh1l/tZltMrNNbW1th9rX4fXuhdhkiMRKd58iIhWm1F+KXgY84JzLDLXSObfeObfCObdi+vQSlkcSezWOi4hUvWICfQcwr2B+rr9sKJcx3uUW8E/7V/1cRKpbMYH+ArDQzBaYWQ1eaD88uJGZnQg0A8+VtotF0Gn/IiKjB7pzLg1cA/wa2AL81Dn3hpl93cwuLGh6GXC/c86NTVdHoIG5RERGP2wRwDn3KPDooGW3DJq/rXTdOgTO+TV0BbqIVLfKP1O0vxOyae2hi0jVq/xA7233rvWlqIhUuQAEeu60fx22KCLVrfIDXaf9i4gAQQj03B66augiUuUCEOh+DV1HuYhIlQtAoLdBrBEiNeXuiYhIWVV+oCf2qtwiIkIQAr1XgS4iAoEJdB3hIiJS+YGuoXNFRIBKD/RsVnvoIiK+yg70/k5wGdXQRUSo9EDv1VmiIiI5lR3oudP+VUMXEanwQM+f9q89dBGRCg/0XMlFNXQRkWAEukouIiIVHuiJvRBvgnC03D0RESm7yg50/Ti0iEhehQe6TioSEcmp/EBX/VxEBKj0QE9oD11EJKdyAz2bhUS7augiIr5IuTtw2Pr2gctqD12kRFKpFK2trfT395e7KwLE43Hmzp1LNFr8UXyVG+g67V+kpFpbW5k0aRLz58/HzMrdnarmnKO9vZ3W1lYWLFhQ9O0qt+Si0/5FSqq/v5+pU6cqzCcAM2Pq1KmH/N9SBQe6TvsXKTWF+cRxOM9FBQe69tBFRApVbqAn2r3r2inl7YeIyARRuYHe2+aFebhyv9cVkfJIp9Pl7sKYqNw07N2r+rnIGPnPv3yDzR/sL+l9njR7Mrd+cvGo7T71qU+xfft2+vv7ue6667j66qt57LHHuPnmm8lkMkybNo3f/va39PT0cO2117Jp0ybMjFtvvZW//Mu/pKGhgZ6eHgAeeOABHnnkEe655x4++9nPEo/HefnllznzzDO57LLLuO666+jv76e2tpa7776bE044gUwmwz/8wz/w2GOPEQqFuOqqq1i8eDG33347P//5zwF4/PHH+f73v89DDz1U0m10pCo80FU/Fwmau+66iylTptDX18dpp53GRRddxFVXXcUzzzzDggUL6OjoAOAb3/gGjY2NvPbaawDs27dv1PtubW3l2WefJRwOs3//fn73u98RiUR44oknuPnmm3nwwQdZv34927Zt45VXXiESidDR0UFzczNf+tKXaGtrY/r06dx999187nOfG9PtcDgqN9ATe2Ha8eXuhUggFbMnPVZuv/32/J7v9u3bWb9+PWeddVb+eOwpU7zvzZ544gnuv//+/O2am5tHve81a9YQDocB6Orq4jOf+QzvvPMOZkYqlcrf7xe/+EUikcgBj3fllVfyb//2b6xdu5bnnnuOe++9t0R/celUbqD3tsExHyt3L0SkhJ5++mmeeOIJnnvuOerq6li1ahUf+chHePPNN4u+j8LD/QYfx11fX5+f/trXvsaf/umf8tBDD7Ft2zZWrVo14v2uXbuWT37yk8TjcdasWZMP/ImkMr8UzWYg0aGSi0jAdHV10dzcTF1dHW+++SbPP/88/f39PPPMM7S0tADkSy7nnnsud9xxR/62uZLLjBkz2LJlC9lsdsQad1dXF3PmzAHgnnvuyS8/99xz+Zd/+Zf8F6e5x5s9ezazZ8/mm9/8JmvXri3dH11ClRnoiQ7AQZ2+FBUJkvPOO490Os2iRYtYt24dZ5xxBtOnT2f9+vVcfPHFLF26lEsvvRSAr371q+zbt4+TTz6ZpUuX8tRTTwHwrW99iwsuuICPfexjzJo1a9jHuummm/jyl7/MsmXLDjjq5Qtf+AJHH300p5xyCkuXLuW+++7Lr7viiiuYN28eixYtGqMtcGTMOVeWB16xYoXbtGnT4d14zxb4/hnwV3fDyReXtmMiVWrLli0TNqgmimuuuYZly5bx+c9/flweb6jnxMxedM6tGKp9UXvoZnaemb1lZlvNbN0wbS4xs81m9oaZ3TdUm5LJnyWqPXQRGR/Lly/n1Vdf5a//+q/L3ZVhjVrVN7MwcAdwLtAKvGBmDzvnNhe0WQh8GTjTObfPzI4aqw4DBeO4qIYuIuPjxRdfLHcXRlXMHvpKYKtz7l3nXBK4H7hoUJurgDucc/sAnHN7StvNQXKn/auGLiKSV0ygzwG2F8y3+ssKHQ8cb2b/bmbPm9l5Q92RmV1tZpvMbFNbW9vh9Rj8kotBncZxERHJKdVRLhFgIbAKuBy408yaBjdyzq13zq1wzq2YPv0IyiW9e70wD4UP/z5ERAKmmEDfAcwrmJ/rLyvUCjzsnEs551qAt/ECfmz0tql+LiIySDGB/gKw0MwWmFkNcBnw8KA2P8fbO8fMpuGVYN4tXTcHSbSrfi4iMsioge6cSwPXAL8GtgA/dc69YWZfN7ML/Wa/BtrNbDPwFPD3zrn2seq0t4euQBepZg0NDeXuwoRT1GAEzrlHgUcHLbulYNoBN/qXsaehc0XG1q/Wwa7XSnufM5fA+d8q7X1OAOl0esKM61J5p/5n0tCncVxEgmbdunUHjM1y22238c1vfpPVq1dz6qmnsmTJEn7xi18UdV89PT3D3u7ee+/Nn9Z/5ZVXArB7924+/elPs3TpUpYuXcqzzz7Ltm3bOPnkk/O3+853vsNtt90GwKpVq7j++utZsWIF3/ve9/jlL3/J6aefzrJly/j4xz/O7t278/1Yu3YtS5Ys4ZRTTuHBBx/krrvu4vrrr8/f75133skNN9xwuJvtQM65slyWL1/uDkv3budunezchvWHd3sRGdLmzZvL+vgvvfSSO+uss/LzixYtcu+//77r6upyzjnX1tbmjjvuOJfNZp1zztXX1w97X6lUasjbvf76627hwoWura3NOedce3u7c865Sy65xH33u991zjmXTqddZ2ena2lpcYsXL87f57e//W136623OuecO/vss93f/M3f5Nd1dHTk+3XnnXe6G2+80Tnn3E033eSuu+66A9p1d3e7Y4891iWTSeeccx/96Efdq6++OuTfMdRzAmxyw+TqxPg/4VDotH+RQFq2bBl79uzhgw8+oK2tjebmZmbOnMkNN9zAM888QygUYseOHezevZuZM2eOeF/OOW6++eaDbvfkk0+yZs0apk3z8iM31vmTTz6ZH988HA7T2Ng46g9m5AYJA++HMy699FJ27txJMpnMj90+3Jjt55xzDo888giLFi0ilUqxZMmSQ9xaQ6vAQNdp/yJBtWbNGh544AF27drFpZdeyo9+9CPa2tp48cUXiUajzJ8//6AxzodyuLcrFIlEyGaz+fmRxla/9tprufHGG7nwwgt5+umn86WZ4XzhC1/gH//xHznxxBNLOhRv5dXQc3voOmxRJHAuvfRS7r//fh544AHWrFlDV1cXRx11FNFolKeeeor33nuvqPsZ7nbnnHMOP/vZz2hv9w7Cy411vnr1an7wgx8AkMlk6OrqYsaMGezZs4f29nYGBgZ45JFHRny83NjqP/zhD/PLhxuz/fTTT2f79u3cd999XH755cVunlFVXqDnxnHRHrpI4CxevJju7m7mzJnDrFmzuOKKK9i0aRNLlizh3nvv5cQTTyzqfoa73eLFi/nKV77C2WefzdKlS7nxRu/AvO9973s89dRTLFmyhOXLl7N582ai0Si33HILK1eu5Nxzzx3xsW+77TbWrFnD8uXL8+UcGH7MdoBLLrmEM888s6ifzitW5Y2H/ub/hVfug0v+FUKV93kkMlFpPPTxdcEFF3DDDTewevXqYduMyXjoE8qJfwGX/UhhLiIVqbOzk+OPP57a2toRw/xwVN6XoiIivtdeey1/LHlOLBZjw4YNZerR6Jqamnj77bfH5L4V6CKS55zDzMrdjaItWbKEV155pdzdGBOHUw5X3UJEAIjH47S3tx9WkEhpOedob28nHo8f0u20hy4iAMydO5fW1laO6MdnpGTi8Thz5849pNso0EUEgGg0mj/DUSqTSi4iIgGhQBcRCQgFuohIQJTtTFEzawOKG5jhYNOAvSXsTqmpf0dG/TtyE72P6t/hO8Y5N+TYJ2UL9CNhZpuGO/V1IlD/joz6d+Qmeh/Vv7GhkouISEAo0EVEAqJSA319uTswCvXvyKh/R26i91H9GwMVWUMXEZGDVeoeuoiIDKJAFxEJiAkd6GZ2npm9ZWZbzWzdEOtjZvYTf/0GM5s/jn2bZ2ZPmdlmM3vDzK4bos0qM+sys1f8yy3j1T//8beZ2Wv+Yx/081Dmud3ffq+a2anj2LcTCrbLK2a238yuH9Rm3Lefmd1lZnvM7PWCZVPM7HEze8e/HvI3w8zsM36bd8zsM+PUt2+b2Zv+8/eQmTUNc9sRXwtj3MfbzGxHwfP4iWFuO+L7fQz795OCvm0zs1eGue24bMMj4pybkBcgDPwBOBaoAX4PnDSozZeAf/anLwN+Mo79mwWc6k9PAt4eon+rgEfKuA23AdNGWP8J4FeAAWcAG8r4XO/CO2GirNsPOAs4FXi9YNl/A9b50+uAfxridlOAd/3rZn+6eRz69mdAxJ/+p6H6VsxrYYz7eBvwd0W8BkZ8v49V/wat/+/ALeXchkdymch76CuBrc65d51zSeB+4KJBbS4Ccj+x/QCw2sZpdH7n3E7n3Ev+dDewBZgzHo9dQhcB9zrP80CTmc0qQz9WA39wzh3umcMl45x7BugYtLjwdfZD4FND3PTPgcedcx3OuX3A48B5Y90359xvnHNpf/Z54NDGWy2xYbZfMYp5vx+xkfrnZ8clwI9L/bjjZSIH+hxge8F8KwcHZr6N/6LuAqaOS+8K+KWeZcBQv3v1UTP7vZn9yswWj2/PcMBvzOxFM7t6iPXFbOPxcBnDv4nKuf1yZjjndvrTu4AZQ7SZCNvyc3j/cQ1ltNfCWLvGLwvdNUzJaiJsvz8Bdjvn3hlmfbm34agmcqBXBDNrAB4ErnfO7R+0+iW8MsJS4H8CPx/n7v2xc+5U4Hzgb83srHF+/FGZWQ1wIfCzIVaXe/sdxHn/e0+4Y33N7CtAGvjRME3K+Vr4AXAc8BFgJ15ZYyK6nJH3zif8+2kiB/oOYF7B/Fx/2ZBtzCwCNALt49I77zGjeGH+I+fc/xm83jm33znX408/CkTNbNp49c85t8O/3gM8hPdvbaFitvFYOx94yTm3e/CKcm+/ArtzpSj/es8Qbcq2Lc3ss8AFwBX+B85BingtjBnn3G7nXMY5lwXuHOaxy/pa9PPjYuAnw7Up5zYs1kQO9BeAhWa2wN+Luwx4eFCbh4Hc0QR/BTw53Au61Px62/8Gtjjn/scwbWbmavpmthJve4/LB46Z1ZvZpNw03pdnrw9q9jDwH/yjXc4AugpKC+Nl2L2icm6/QQpfZ58BfjFEm18Df2ZmzX5J4c/8ZWPKzM4DbgIudM4lhmlTzGthLPtY+L3Mp4d57GLe72Pp48CbzrnWoVaWexsWrdzfyo50wTsK4228b7+/4i/7Ot6LFyCO96/6VmAjcOw49u2P8f71fhV4xb98Avgi8EW/zTXAG3jf2D8PfGwc+3es/7i/9/uQ236F/TPgDn/7vgasGOfntx4voBsLlpV1++F9uOwEUnh13M/jfS/zW+Ad4Algit92BfC/Cm77Of+1uBVYO05924pXe869BnNHfc0GHh3ptTCO2+9f/dfXq3ghPWtwH/35g97v49E/f/k9udddQduybMMjuejUfxGRgJjIJRcRETkECnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISED8fxdnpHaVSAlRAAAAAElFTkSuQmCC\n","text/plain":["<Figure size 432x288 with 1 Axes>"]},"metadata":{"needs_background":"light"},"output_type":"display_data"}],"source":["history_frame = pd.DataFrame(history.history)\n","history_frame.loc[:, ['loss', 'val_loss']].plot()\n","history_frame.loc[:, ['accuracy', 'val_accuracy']].plot();"]},{"cell_type":"markdown","id":"c871cfe9","metadata":{"papermill":{"duration":0.014464,"end_time":"2022-07-09T03:37:23.242186","exception":false,"start_time":"2022-07-09T03:37:23.227722","status":"completed"},"tags":[]},"source":["# 5. Predictions"]},{"cell_type":"code","execution_count":16,"id":"e3ab0908","metadata":{"execution":{"iopub.execute_input":"2022-07-09T03:37:23.272945Z","iopub.status.busy":"2022-07-09T03:37:23.272636Z","iopub.status.idle":"2022-07-09T03:37:33.10386Z","shell.execute_reply":"2022-07-09T03:37:33.102619Z"},"papermill":{"duration":9.848945,"end_time":"2022-07-09T03:37:33.105665","exception":false,"start_time":"2022-07-09T03:37:23.25672","status":"completed"},"tags":[]},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>ImageId</th>\n","      <th>Label</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>3</td>\n","      <td>9</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>4</td>\n","      <td>9</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>5</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>6</td>\n","      <td>7</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>7</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>8</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>9</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>10</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>11</td>\n","      <td>5</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>12</td>\n","      <td>7</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>13</td>\n","      <td>4</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>14</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>15</td>\n","      <td>4</td>\n","    </tr>\n","    <tr>\n","      <th>15</th>\n","      <td>16</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>16</th>\n","      <td>17</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>17</th>\n","      <td>18</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>18</th>\n","      <td>19</td>\n","      <td>9</td>\n","    </tr>\n","    <tr>\n","      <th>19</th>\n","      <td>20</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["    ImageId  Label\n","0         1      2\n","1         2      0\n","2         3      9\n","3         4      9\n","4         5      3\n","5         6      7\n","6         7      0\n","7         8      3\n","8         9      0\n","9        10      3\n","10       11      5\n","11       12      7\n","12       13      4\n","13       14      0\n","14       15      4\n","15       16      3\n","16       17      3\n","17       18      1\n","18       19      9\n","19       20      0"]},"execution_count":16,"metadata":{},"output_type":"execute_result"}],"source":["predictions = model.predict(test)\n","\n","predictions = np.argmax(predictions,axis = 1)\n","\n","submissions=pd.DataFrame({\"ImageId\": list(range(1,len(predictions)+1)),\n","                         \"Label\": predictions})\n","submissions.to_csv(\"submissions.csv\", index=False, header=True)\n","\n","submissions.head(20)"]},{"cell_type":"markdown","id":"7a32cfa9","metadata":{"papermill":{"duration":0.009212,"end_time":"2022-07-09T03:37:33.12472","exception":false,"start_time":"2022-07-09T03:37:33.115508","status":"completed"},"tags":[]},"source":["# 6. Conclusion"]},{"cell_type":"markdown","id":"ec8d6af9","metadata":{"papermill":{"duration":0.009473,"end_time":"2022-07-09T03:37:33.143744","exception":false,"start_time":"2022-07-09T03:37:33.134271","status":"completed"},"tags":[]},"source":["We started with the task of improving on the result from our first model, a result of 98% accuracy. By applying batch normalization and dropout layers to the orginal model we were able to improve the validation loss to .029 and the validation accuracy to 99%, achieving our objective of improving our first model."]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.12"},"papermill":{"default_parameters":{},"duration":678.987195,"end_time":"2022-07-09T03:37:36.210513","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2022-07-09T03:26:17.223318","version":"2.3.4"}},"nbformat":4,"nbformat_minor":5}