evaluation_n_inception.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from ast import literal_eval\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "df = pd.read_csv(\"inception_n.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['activation', 'aux1_accuracy', 'aux1_loss', 'aux2_accuracy',\n",
       "       'aux2_loss', 'dataset', 'dropout_rate', 'epochs', 'input_dims', 'loss',\n",
       "       'lr_rate', 'main_accuracy', 'main_loss', 'model_name', 'test_accuracy',\n",
       "       'test_loss', 'val_aux1_accuracy', 'val_aux1_loss', 'val_aux2_accuracy',\n",
       "       'val_aux2_loss', 'val_loss', 'val_main_accuracy', 'val_main_loss',\n",
       "       'opt'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def literal_return(val):\n",
    "    try:\n",
    "        return literal_eval(val)\n",
    "    except (ValueError, SyntaxError) as e:\n",
    "        return val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in list(df.columns):\n",
    "    df[col] = df.apply(lambda x: literal_return(str(x[col])),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>main_accuracy</th>\n",
       "      <th>val_main_accuracy</th>\n",
       "      <th>loss</th>\n",
       "      <th>opt</th>\n",
       "      <th>test_accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[0.80731344, 0.9772637, 0.9834826, 0.988607, 0...</td>\n",
       "      <td>[0.87580806016922, 0.7410101294517517, 0.96974...</td>\n",
       "      <td>[1.6078689948006055, 0.2451490221450578, 0.177...</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.9600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[0.8104975, 0.97537315, 0.9788806, 0.9870398, ...</td>\n",
       "      <td>[0.9563131332397461, 0.9661616086959839, 0.980...</td>\n",
       "      <td>[1.447963404519048, 0.250147792361862, 0.22745...</td>\n",
       "      <td>adam</td>\n",
       "      <td>0.9874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[0.80634326, 0.9706219, 0.9814179, 0.98544776,...</td>\n",
       "      <td>[0.9034343361854553, 0.95252525806427, 0.97202...</td>\n",
       "      <td>[1.56470524057227, 0.29654361480504127, 0.2005...</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.9859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[0.25155222, 0.4315224, 0.50325376, 0.5520597,...</td>\n",
       "      <td>[0.22963635623455048, 0.46515151858329773, 0.4...</td>\n",
       "      <td>[5.855830362519222, 4.678803367728618, 4.15245...</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.6306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[0.24068657, 0.40020895, 0.49325374, 0.5469851...</td>\n",
       "      <td>[0.3187878727912903, 0.40090909600257874, 0.40...</td>\n",
       "      <td>[5.79272524728348, 4.6710144616881415, 4.09117...</td>\n",
       "      <td>adam</td>\n",
       "      <td>0.6440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[0.28922388, 0.4237612, 0.47519404, 0.51044774...</td>\n",
       "      <td>[0.3824242353439331, 0.4492727220058441, 0.494...</td>\n",
       "      <td>[5.666933932517892, 4.686836285263745, 4.32059...</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.5795</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       main_accuracy  \\\n",
       "0  [0.80731344, 0.9772637, 0.9834826, 0.988607, 0...   \n",
       "1  [0.8104975, 0.97537315, 0.9788806, 0.9870398, ...   \n",
       "2  [0.80634326, 0.9706219, 0.9814179, 0.98544776,...   \n",
       "3  [0.25155222, 0.4315224, 0.50325376, 0.5520597,...   \n",
       "4  [0.24068657, 0.40020895, 0.49325374, 0.5469851...   \n",
       "5  [0.28922388, 0.4237612, 0.47519404, 0.51044774...   \n",
       "\n",
       "                                   val_main_accuracy  \\\n",
       "0  [0.87580806016922, 0.7410101294517517, 0.96974...   \n",
       "1  [0.9563131332397461, 0.9661616086959839, 0.980...   \n",
       "2  [0.9034343361854553, 0.95252525806427, 0.97202...   \n",
       "3  [0.22963635623455048, 0.46515151858329773, 0.4...   \n",
       "4  [0.3187878727912903, 0.40090909600257874, 0.40...   \n",
       "5  [0.3824242353439331, 0.4492727220058441, 0.494...   \n",
       "\n",
       "                                                loss      opt  test_accuracy  \n",
       "0  [1.6078689948006055, 0.2451490221450578, 0.177...      sgd         0.9600  \n",
       "1  [1.447963404519048, 0.250147792361862, 0.22745...     adam         0.9874  \n",
       "2  [1.56470524057227, 0.29654361480504127, 0.2005...  adagrad         0.9859  \n",
       "3  [5.855830362519222, 4.678803367728618, 4.15245...      sgd         0.6306  \n",
       "4  [5.79272524728348, 4.6710144616881415, 4.09117...     adam         0.6440  \n",
       "5  [5.666933932517892, 4.686836285263745, 4.32059...  adagrad         0.5795  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df[0:3][:,J]\n",
    "d_opt = df[['main_accuracy','val_main_accuracy','loss','opt','test_accuracy']][0:6]\n",
    "d_opt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhsAAAF1CAYAAABBMSgKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd3gc1dXH8e+xJFdcAJvmCqEETDUOzRQTIGCFxBACAQKhlwQCCSRACBBTA0kgQEIJnYTeXiBBlNACmGrTbZoxNrbBvctF7bx/nBFay5K1kne1u9Lv8zzz7O7M7Mxd6e7dM/feudfcHREREZFs6ZDrBIiIiEjbpmBDREREskrBhoiIiGSVgg0RERHJKgUbIiIiklUKNkRERCSrFGy0MjO7w8wuaeVzjjOz4a15ThGRTDCzQWbmZlac67QAmNkkM9u7Fc83wMwWm1lRa50zGxRstAPuPtjdX1zVPvn2hZbWUej/97ZSEIs0xt2/dPc13L16VfuZ2dFm9kprpau5FGyISMFqKwWx5IaC1NajYKMZzOxsM5tmZovM7BMz28vMupjZnWY2z8w+MrOzzGxqynu2M7O3k/fcD3RO4zzDzWxqcqyZZva1mR1gZqVm9qmZzTWzc1P2H2VmD5jZP5PzjDOzoSnbv6n2M7MdzGyMmS00sxlmdlWy20vJ4/zkSnHnjPzRREQaYGbnmNnnSZk13swOTNYXmdlfzGy2mU0Evl/vfcckZe0iM5toZifV235WUmZ+ZWbHJzV3Gyfb7jCzG8yszMzKgT3N7Ptm9k5SJk4xs1H1jnekmU02szlm9vs0P9soM3vQzO5K0vmBmW1qZr9LyvQpZva9lP1fNLOLzWx0sv8zZtY72bZC7WMSOE9M9vvCzH5qZpsDNwI7J+X3/Gb+O7LP3bWksQCbAVOADZLXg4BvAZcD/wPWBPoB7wNTk306ApOBXwMlwI+BSuCSJs41HKgCLkjedwIwC7gH6A4MBpYCGyb7jwKWAaVAEfBH4PWU400C9k6evwYcmTxfA9gp5fM4UJzrv7WWjOTXc4DPgUXAeODAZH0R8BdgNjAROCX1/w4cA3yUvG8icFK9fDkVOAuYCXwNHJDku0+BucC5aaRtFPAgcFdyng+ATYHfJcedAnwvZf8XgYuB0cn+zwC9G8q3wNFJuhcBXwA/BTZPvh/VwGJgfq7/P1oc4GBgA+Ki9ydAObA+cDLwMdAfWAt4od7/+PtE2WvAHsASYEiybT9gelJGdk3ymAMbJ9vvABYAw5Lzdk7y9VbJ662BGcAByf5bJHlmd6ATcBVRNu+dRh5fBuwLFAP/TPLj76kr07+ol8c/T74HXZLXl9fP40A3YCGwWbJtfWBwSt5/Jdf/10b/JrlOQKEswMZJQbg3UJKyfiKwb8rr46kLNnYHvgIsZfurpBdsLAWKktfdk8y2Y8o+Y1O+EKOAZ1O2bQEsTXk9ibpg4yXgQpLCOmWfbzJ0rv/WWjKSX7NRkA+nGUHwKtKmglhLQ/niXWAk8Dxwcsr6762qbAIeBU5Pnt8G/DFl28asHGz8s4l0XA38NXl+AXBfyrZuQAXpBRv/TXn9AyJoqV+m90pevwicl7L/L4Cnkuf18/h84CCgS71z5nUeVzNKmtx9AvArIhPNNLP7zGwDokCfkrJr6vMNgGme5ITE5DRPOcfr2qGXJo8zUrYvJWomak1Peb4E6NxIp7/jiEL7YzN7y8z2TzM9UkDc/UF3/8rda9z9fuAzYAfgEOBqd5/i7nOJWrDU9z3h7p97+B9Ri7Bbyi6VwKXuXgncB/QGrnH3Re4+jqhF2SaNJL7s7k+7exVRy9GHCCBqjzvIzHql7H+7u3/q7kuBB4BtGzluDbClmXVx96+TNEkeMrOfmdm7ZjY/qfbfkshP9cvUyfXeN8LMXk+ak+cTNWu9k82rKo8bXGdmO5rZC2Y2y8wWEAF5g8dz93JgTpofsX55PbuBMn1VZXjqttTz/yRJ49dm9oSZfTvN9OSUgo1mcPd73H1XYCARaV5BVCX3S9mtf8rzr4G+ZmYp6wZkPaGr4O6fufthwDpE+h8ys27E55E2IksFOTQ/CG6MCuJ2zMwGAjcDpwJru3sv4EOiRu1rVixHB6S8rxPwMNEUuG7yvrLkfbDq8rhW/bLuHuBxoL+79yT6PqQe75tjmFlXYO20P2gWJEH6PkTN3cfE3xHyvAxXsJEmM9vMzL6bZPZlRIFYQ1xl/c7M1jSzvsSXp9ZrRLXzaWZWYmY/Iq4uc8bMjjCzPu5eQ1THQXyOWcnjRjlLnGREFgvyvFeoBXE7VHuBMwui0ycREEOUqaeZWT8zW5Pof1SrI9F3YhZQZWYjiGYWUt57jJltngQG56eRlu7AXHdfZmY7AIenbHsI2N/MdjWzjsBF5PB308zWNbORyQXicqJppibZPAPol6Qz7yjYSF8nojPobOIqax2iQ9tFRKe5L4Bnicy5HMDdK4AfEW1pc4mrrkdaOd317QeMM7PFwDXAoe6+1N2XAJcCo5Or4Z1ymkpZHdkqyPNaIRfE7Y27jweuJC7IZhAdNEcnm28GngbeA94mpcx090XAaUQ+nkcEBo+nbH8SuJboizQBeD3ZtHwVyfkFcJGZLSL6aDyQcrxxRCfqe4hAfR5R3udKB+AMoi/gXKJf1c+Tbc8D44DpZjY7N8lrnK3YnUBWl5n9nPgB3yPXaZH2y8wuJQqhGqID5vbAv4gOcn8GfkZ0pvwL8Hei03OVmZ1CFLidgH8THTYnuPt5FqPQ3uXu/ZJzFBN9ODZ090nJuleAG939rlWkbRTRYe+I5PXewC3uPqjecfu7+1QzezE57y3J9qOB4919VzMbRAT6JUS/j/uI/hxOdDj8hbuPT4KM/wN2BmrcPbVpSNqo5JbQD4FOSf8gyREFG6vJzNYnmh5eAzYBngD+7u5X5zRhIiLtkMV4HWXEra93EsHlAblNlagZZfV1BP5B3Nf/PPAYcH1TbzKzc5PBV+ovT2Y5vSIibdlJxDAFnxNjq/x81bu3jJk92UgZfm7T725/VLMhIhmXBM27NbDpMne/rLXTIyK5pWBDREREskrNKCIiIpJVOZtWunfv3j5o0KBcnV7amLFjx8529z6teU7lYcmkXORhUD6WzFlVHs5ZsDFo0CDGjBmTq9NLG2Nm6Q4DnzHKw5JJucjDoHwsmbOqPKxmFBEREcmqJoMNM7vNzGaa2YeNbDczu9bMJpjZ+2Y2JPPJFBERkUKVTs3GHcQQ140ZQQxmtQlwInDD6idLRERE2oom+2y4+0vJkMCNGQn8M5lG/XUz62Vm67v71xlKo+SKO1RUwPLlKy41NdC1K3TrFo8lJWAFM1eXiEjb5h7ldFUVVFbGY2NL/e0N7V9/3Z57woDmTWCeiQ6ifVlxyuqpybqVgg0zO5Go/WBAMxPaJrjHP6qiIv55tY/LljV/Wbo0fvirqxtfqqpWvb1+EFF/qahI73MVFdUFHqlBSEPPO6bMg5U6xks6zzfYAM5JnTes9bX7PJwu94aXmppVr0/nMfV5bT5vaGlqW3Fx5MeSkpWXxtaXlER+bygIT12WLWt8229+A5065fTfo3zcSiorYdYsmDkTZsyoe6x9PnNmlOW1ebmxx8a2rSqIyKb/+7+cBBtpc/ebgJsAhg4dmp+jiVVXw8KFsGABzJ8fj6nPG1q3cGHdj3Nl5YqBROpjSgZwoJxuLKI7AB2oaXQxfMV1BtalM9apI15UTE1RyTdLdYfkeYfilR6rO5RQ06EjNUUldO+4nF49O0eh19zFLL4gS5ZAeXk81n9e+/qrr1bctnz5irUgzXk+eHDOg42CyMNNqayMQu7rr2OZPj2WRYvSCnSXLIGZS9ZgxtIezFzeg1kVPelQU01XL6cbi+nKErpRTleWrPS86JtJWFfmwDI6s4CezKcXC+i5yufldGM9pjOISQxkMoOYxCAm0YdZ5Es9mwPL6cRi1iD+Omuw+Uk/p0OOg402k48XL15xWbQoyuNFi5q/uNddGNUuXbqsvK7++i5domxrKJiYM2eFJDtQRTEVHbtTuU5fqnuvy5rdoUNRUQS4RUXQocOqH2ufd+gQwW9xcXpL6r5FRSu+buA41R1KqKSBxYtZZ/O16dbMf1cmgo1pQP+U1/2SdblVXQ3z5sU/e84cmDu37nljr+fPj0zXlK5doWdPvEdP5nfvz7SSrZnbdW3md+nJgpruzK/pwfzq7syvWiOWyq4sqOzK/OVdYlnWmfnLOlNd08KbgRxYkiwt1K0bfPppVBZIYaupdqZ8XsH4dytYPH0xxQvnxrJgDiULZlM8fzbFc2fGMmcGxfNnU0IFxVR9sxRRzcKOfZhZ0pcZxX2ZWbQRM2w9ZtKHmTW9mVHdm5mVazGzsheLq7q0OK2diqvoWlJJ15IqunWspHNJFUsqS5i/tBMLlnWisrpole/vYDX07FpJzy4VdO1YzYsLuzB38Yo/3F06VTNo3WUMWn8Zg9avYFDfSgb1q4plQA191jGsqENd9XBFBVVLKylfVEP5ohoWL3LKFzvl5VC+2FlcbpQvSZalHShfVsTiqs6UV3VicVUnyis7sriiI+UVJZQvL2bxsmLKlxWzeGkHypd2oKZmxdBnXpHTq8V/wTbGPcreL76ASZPiccaMlYOI2kAieb5kUTWTK9dnMgOZxKBvHnfjZX5Rv9tghw7QvfvKyzrrrPjaLIKG2gup1GXmzJXXLVkCNTX8hj/zPCOo7NCJyqIuVBZ1otI6xtKlhEovorKmiMrqDlRVJ2V+BdEGMDUqCA49FA47DLbZJrst0uPGwYMPwhNPxM9d7bVxQ8uqBhd/9FEYObJ5585EsPE4cKqZ3QfsCCzISX+N5cvhqKNgzJgIHubPb3zfoiJYa61Y1l47/tvbbgtrrgk9e0KvXtCzJ+Wd1mJKxbpMKV+LKQt7MmX+GkyZ1YUvp3ZgyhSYMiUu2BvTrVscqlcv6NkH1u0Fm/WqW9erV10er60dbmhJrT2uv6QGubVLQ+tSty1fDqefDtdcA1dckfl/hayGMWOgrGzF2qDycigvx8uX8OW87oybtwHjF/Vn3NINGbd8E8bXbEY5axCzwncH1m/ZuSuSJdGhA/TpA+uuG+XyRuvUPa9d1l0XeveOPFq/UqvhCq9iysuLV9i3W7f42qV89b55rP98jTU6YNYp+axh4UKYPDl+q2IpYtKkbkya1I03XorfslRdukSQXVERaVu8OP0Ww1qdO8Maa0Tav3lcE9bqVm9dvcdu3aBzl3ypd2klCxasGEykPk6atPIFXpcuLOy2PpM7b8ak4o2Z3GEQk2oGMLmyL5OWrcvkpX2YVdl9hbeUFFXTuWMNT3Y4mBOeP4GStVKCiC5dsvML7s6c6ZX8tV8Jg7dwNtmkQ1qtcKkLwHPPwVVXwZ/+BN/+dgQdhx0Gm2ySkSTy4YcRYDz0EHz0Ufwphg2D73wnvTQ2tGy3XfPT0mSwYWb3AsOB3mY2FfgDUBIfxG8kpvItBSYQ19rHND8ZGXD77XD//XDAAdCvXwQRtcFE7VL7ukePKEkT7nDDDfFPmTKWbwKJ+oUUwHrrQf/+sMUWsO++EafUnq62YKwtHGszUz565RW48UY499xIq+SBefNg773xBQuY0nFjxnXcjnFFWzPON2d81aaMX/YtFtd0/Wb39bosYHDfGRy77gcM7jufLfovYs0NulC1Zh+q1uxNVa8+VHXtQVW1NdlHrKoqvhapQcRaa63wNclbPXrAVlvF0pDUYKT2cdq0aBFMDQLqBwWNve7aNYJ2acTbb8Nll8HEiRFU1L/wW2MN2HDDWPbck7nrfJtXl2zL6Okb8erHa/PBR8XMm73iWzp3hoEDYeBgGDIoeT4QBiXP11+/iMcfL+JHP4LXlm3H7hu3wuc045kXO1JTAzffYuy4Y8sO8+tfx/Xxww/DvffCqFHwhz/A9ttH0PGTn8RvTLrc4YMPIsB48EH45JP4Hu++O5x6Khx4IKzfwuuR1ZGzidiGDh3qGRu1bvnyCAP7949f0WZGsc8/D3vtFUHCwIFxmNRlwIB47Nt3xf6NhezttyMzX3EFnHVWrlOz+sxsrLsPbc1ztjQPV1dHF4kvv4xlypTk+dPjmfLpUj7tui2Ll9T9mq23XgS3gwfHUvt8rbUy+Wkk13KRhyHDZXF1dUR9M2bATjtFNLDhht88+sBBfDZnLUa/aoweDa++GlfbEF0FhgyJcmnQoLpAYtCgCICbKtYXLoxatjPOgMsvz8zHacqRR8KTT8bHzVQAOm1aXDffe29UdJpFoHDYYfDjH8eFbX3u8P77dQHGp59GgLHHHnDwwfCjH8UFRLatKg/nbLjyjLr99iixb721RdVld9wRV/dffRU1bu3BkCGw995w9dXRpJLj/mpNmjYtrkiHDct1Spo2f34DgUTK82nTVu4s3qN7DQMWOwPWN4b9uOiboGKLLRouXETy0r33RvTwwANw8MEsWwZjx8Lo0TD6XxFczE5qLdZcE3bZJX6whw2DoUOj1qilevSA3XaLVsjWCDZqauCpp2C//TJb09W3bwRMZ5wBn30G990H99wDJ58cNRPf+14EHiNHwoQJdU0kn30WAcaee0ZtyYEHtk6AkTZ3z8my/fbbe0YsW+bev7/7Lru419Q0++0LF7p37ep+wgmZSU4h+e9/477Dm2/OdUoaV1Pjfvvt7j17um+0kXtlZcP7AWM8T/Lwd7+74n2dJSXuG27ovsce7kce6X7uue433uj+xBPuH3zgPn++u595pnuHDu4ffbQ6fy4pYLnIw57Jsriiwn2jjfy9zQ7235xZ4zvv7N6xY933YNNN3Y85Jsqb8ePdq6szc9pUf/lLnOvLLzN/7PreeCPOdffd2T9XTY37O++4n3WW+4ABcd4OHeKxqMh9773d//EP95kzs5+WVVlVHi78YOOGG+JjPPNMi95+++3x9ldeyUxyCklNjfuQIVEIZOOLv7qmTnUvLY3/z267uX/2WeP75lOw8dRT7g8+6P766+5ffZXG33baNPfOnd2POqqJHaUtK/RgY+aV//STud47dKjxTp3chw2LH8dHH229H8Hx46O8+Mc/sn+uP/zB3cx99uzsnytVdXX8Xp19tvtNN7nPmtW651+VthtsrGathrv78OHuG2/c4rcXvPvui1zwyCO5Tkmdmhr3226L2oyuXd2vvbbpH+x8Cjaa7ZRT3IuL3T//PDPHk4JUqMFGRYX7X/9U4T1tvhdR6aefVuNz567WIVuspsZ90CD3kSOzf67vfMd9552zf55Csqo8XAB9zVehtq/GqFEt6qsxaRK8+GLcMdteR9s+6CDYaKPoKOp5MLTPlClQWgrHHhv3nL//Pvzyl4VxV0SLTJ4MN90Exx0X/wiRAvLUU7D11vDrs0rY0V/n/VvHcPU1xppr5iY9ZjBiBDz7bNw3kC0zZsBbb0VZJekp3CJ8+fK4vWqXXaKnYwv861/xeOSRGUxXgSkuhjPPhDfegJdfzl063KN/75Zbwksvwd/+Bi+8AN/6Vu7S1CouvjgiqfPOy3VKRNL26aew//7xw15dVcO/ex7BU8OvYItjd8p10igtjfFTslmePf103bkkPYUbbKxmrYY73Hln9NwdODDzySskxxwTgzflaoCvL7+MHt3HHx+DxXzwQfS6brO1GbU++yxuhTr55ObdSC+SIwsWxNQugwfHj/lf/gIfHn0l+y+4G7v0klwnD4gyvVOnuCslW8rK4k6PbbfN3jnamsIszjNQqzF6NHz+eTShtHddusBpp8UX6IMPWu+87nDLLVGbMXo0/P3vMeZJu2lNuPDCKBVzPN+LSFOqq+Hmm2M4o6uugqOPjtqNM49fQMcr/xiX+LvskutkAjH42vDh2Qs2qqqiZmPEiHZwQZRBhfmnWs1aDYhajW7dos+CwC9+EX+PP/+5dc735ZcxAusJJ8T99e+/D6ec0o6+vOPGxc3zv/xljNolkqdeeim+oyeeCJttFgNN3XxzMobDX/8aI99edFGuk7mC0tIYOfPzzzN/7DfeiLF01ITSPIVXtC9fDpdeulq1GkuXxpgzBx0UI+dKjEZ5wgkxJs+XX2bvPO7RH3LLLWOAn+uvj85c7aY2o9Yf/hCZ77e/zXVKRBo0eXIMlb3HHjGc9v33R+AxZEiyw5w5Uc3xox/FsJ95pDYQePLJzB+7rCwG8dpnn8wfuy0rvGDjtttg6tTVqtV49NEY2vboozOasoL361/H41//mp3jT50ao9+ddFJMAvThh/Dzn7ej2oxab78dEyGccYaGB5W8dPPNMSnYv/8drX0ffwyHHFKvyP3Tn2IWuzyr1QDYeONo8slGU0pZWYx42ktT9zZLYRXzGeirAdGEMnBgROxSZ8CAGAb35psbnoRudcyZA9/9Lrz+ekx69+yzMedBu3TBBTFWc210J5JnNt005rT85JPIrisNIz59etwydvjh0Vs0D5WWxh1tS5Zk7pjTpsG776oJpSUKK9jIQK3GV1/Bf/8bt7u2uyvqNPz2t3Hb2HXXZe6Yy5fHOP1ffhn35Z98cvsd14TXXoMnnojZ7zTdruSpPfaIJtX+/RvZ4bLLoKIiyuI8VVoKy5bFWEqZ8tRTdceW5imcn9sM1WrcdVdMoPOzn2UwbW3IVlvFF+naa6Nvy+pyj1taX3457vIshInUsur882MKy1/+MtcpEWmZL7+Ef/wj7pnfuDXmcm+Z3XePGplMNqWUlcVd6ltumbljthd5F2wsWBDR6EoyUKvhHj94u+wS7XnSsLPPjpkZb7999Y918cUR4F18MRx66Oofr6C98AI89xz87ndx649IIbr44ng8//zcpqMJnTvDXntFRWImRkeuqIha8REj2nHN7GrIq2Bj7tyYUvvSS+ttyFCtxpgxMfuxxtZYtd12gx13hCuvXHkq9Oa455646eKoo+D3v89c+gqSexTOfftGO5JIIZowIa5CTjopOnnludLSmJbik09W/1ivvgqLFqkJpaXyKthYa62IRC+/PIYh+EYGajUgOoZ27hy9qqVxZlG7MXFi3DTREq+8ErWse+wRt7q2+yuBp5+OkcvOOy8yoUghGjUKOnaEc8/NdUrSMmJEPGaiKaWsDEpK4jdKmi+vgg2Iq+mePWMAmZoaMlarsXx5dHg64ADdspSOkSOjR3pLJmibMCH+zoMGwSOPRNnUrrlHkDFoUMwwJ1KICnAguoED42aZTAUbu+8O3buv/rHao7wLNvr0iYDj1VfjijhTtRpPPBHNNGpCSU+HDnFnyjvvRDeDdM2dC9//fjx/4omorWr3HnsMxo6NNqV2H3lJwbrgghiI7qyzcp2SZiktjcHIFi1q+TEmT45YS00oLZd3wQbEnSLf/S6cfbbz1cW3xi0Mq1GrAdGEsv76GvWtOY48Mi5g0p2graIiBhOcNCkGTsvjjuqtp6Ym+mpsuikccUSuUyPSMmPHRjVlAQ5EV1oKlZXNu2iqr3YkUgUbLZeXwYYZ3HgjLF9Szelfn73atRozZ0YV2BFHxDCzkp5OneBXv4oBuMaOXfW+7jHc+f/+F/3Hdt21ddKY9x54IIZKvfBCKC7OdWpEWub886OasgAHohs2LJo+VqcppawMNtww5oaRlsnLYANgkwHLOb/rVTzEwfx7yer1yLnnnrirQk0ozXfyydCjR9MTtF16Kfzzn/GbevjhrZO2vFdVFU0nW26pXslSuEaPjkv7Ah2IrqQkpkkoK2vZLbDLlkWtiG55XT15G2xw6638duF5DB60mFNONRYvbvmh7rwzZi3M01F181rPnhFwPPhg4zMo3ntvXPgceWTe33rfuu66K+bhvvhiDVcrham2c/O668Kpp+Y6NS1WWhpDjX/wQfPf+/LLMeS5mlBWT36WgMkdKB2H7cBNd3VjypSW/4i9/36MZa9ajZY7/fRoAbjqqpW3jR4dE9rtvnvMqaLIP1FREdU8228ft/aIFKLnn4/xvs89t6AHottvv3hsySywZWXRpLznnplNU3uTn8HGrbdGGDpqFLsMM04+OYbPHjOm+Ye6886oRjvssMwns73YYIOotbjttuj/Uuvzz+MW14EDo+9Yp065S2Peue226Cl7ySWKwKQw1dZq9OsXYxEUsA02gO22a1m/jbKyCDRWmoxOmiX/go3acTWGDftm9JQ//jFq8U44oXkjWlZVwd13w/77F1wH6rzzm9/Ev+Zvf4vX8+bFLa41NXGLq/6+KZYujaaTYcNg331znRqRlnniiZim+YIL2sRAdKWlURM7f37675kwIVpC1YSy+tIKNsxsPzP7xMwmmNk5DWwfYGYvmNk7Zva+mbX8X5NSq1F7RdirV9RsvPsuXHNN+od6+mmYMUNNKJnw7W9Ha8B118VYGj/6EXzxRdziqnlm6vnHP2J6YdVqSKGqvWX7W9+KdtI2oLQUqqtjfpN01Ta71I5EKi3XZLBhZkXAdcAIYAvgMDPbot5u5wEPuPt2wKHA9S1KTQO1GrUOOgh+8IMIsidNSu9wd9wBvXsro2TK2WdHjcZ3vhPNuLfeGvOoSD3PPRf5d/jwXKdEpGUefjiu7kaNinboNmDHHePu3eY0pZSVxRA5GjNo9aVTs7EDMMHdJ7p7BXAfUL/HmwM9kuc9ga9alJqaGjjllAavCM3g73+Px5//vOlbmObOhccfh5/+VIM2ZspOO0VwMXFi3NGpMaoa8fjjcfuOSKEqLo5elW2os1tRUbRqPvlkMhVGE5YsiYmadbGaGekEG32BKSmvpybrUo0CjjCzqUAZ8MuGDmRmJ5rZGDMbM2vWrJV36NIlpt9u5IpwwIAYz+Gpp+D++1ed6PvvjxsC1ISSWbfcAtdfH8FGe9RkHo6dYM01WzdhIs3QZD4+8MD4VW5joyCWlkbT+jvvNL3viy9GZbv6a2RGpjqIHgbc4e79gFLgX2a20rHd/SZ3H+ruQ/v06dOiE516aoyZcfrpUaXfmDvvhK22gm23bdFppBGbbho1S+21K0Im8rBIrrXXfLzvvlF2pdOUUlYWd6Dsvnv209UepBNsTAP6p7zul6xLdRzwAIC7vwZ0BnpnIoH1FRXFBG1z5jQ+H9Ann8Abb0StRnv9URQRkUkkLG0AACAASURBVBX16QM77NB0sOEeN+PstVebuBEnL6QTbLwFbGJmG5pZR6ID6OP19vkS2AvAzDYngo1G6phX33bbxRD9t9wSs/nVd+edEZT89KfZSoGIiBSi0tK4GJ09u/F9PvkkbkRQE0rmNBlsuHsVcCrwNPARcdfJODO7yMx+mOx2JnCCmb0H3Asc7d6SUejTN2pUDCZ14onRrlaruhr+9a+oLltvvWymQERECk1padRcPP104/vU1nyoc2jmpNVnw93L3H1Td/+Wu1+arLvA3R9Pno9392Huvo27b+vuz2Qz0RAj595wQ0Sgl19et/7552Hq1DZza7iIiGTQkCGwzjqrbkopK4u5tAYObL10tXX5N4JoM4wYAYceGkNzfPxxrLvzzhgE7Ac/yG3aREQk/3ToEL8dTz0VNeH1LVoUzfNqQsmsgg42AK6+OnoMn3giLFgQc3Qceqg69YiISMNKS2MspjffXHnb889DZaWaUDKt4IONddeFP/85pgE+6KCYlkJja4iISGP22SduImioKaWsDLp3j4GsJXMKPtgAOPbYuBf6uediHIgdd8x1ikREJF+tuSbsssvKwYZ7rNtnH408nWltItjo0CHmvurSJZpTNLaGiIisSmkpvP02fP113boPP4wbDNRfI/PaRLABMSvp1Kkx/oaIiMiq1AYUTz1Vt063vGZPmwk2IGb069CmPpGIiGTDVltB374rNqWUlcUUFxtskLt0tVX6aRYRkXbHLGo3nnkm7j6ZPx9Gj1YTSrYo2BARkXaptBQWLoRXX4X//jfG3VATSnYU5zoBIiIiubDXXlBSEs0ns2bFgJA77ZTrVLVNCjZERKRd6t49hk34z39ikK9994Vi/SpmhZpRRESk3SothfHjYfp09dfIJgUbIiLSbqUGGPvtl7t0tHWqMBIRkXZrs81go41g7bVjNljJDgUbIiLSbpnBo49Cp065TknbpmBDRETata22ynUK2j712RAREZGsUrAhIiIiWaVgQ0RERLJKwYaIiIhklYINERERySoFGyIiIpJVCjZEREQkqxRsiIiISFYp2BAREZGsUrAhIiIiWZVWsGFm+5nZJ2Y2wczOaWSfQ8xsvJmNM7N7MptMERERKVRNzo1iZkXAdcA+wFTgLTN73N3Hp+yzCfA7YJi7zzMzzZ0nIiIiQHo1GzsAE9x9ortXAPcBI+vtcwJwnbvPA3D3mZlNpoiIiBSqdIKNvsCUlNdTk3WpNgU2NbPRZva6me3X0IHM7EQzG2NmY2bNmtWyFIvkkPKwtAXKx9LaMtVBtBjYBBgOHAbcbGa96u/k7je5+1B3H9qnT58MnVqk9SgPS1ugfCytLZ1gYxrQP+V1v2RdqqnA4+5e6e5fAJ8SwYeIiIi0c+kEG28Bm5jZhmbWETgUeLzePo8StRqYWW+iWWViBtMpIiIiBarJYMPdq4BTgaeBj4AH3H2cmV1kZj9MdnsamGNm44EXgN+6+5xsJVpEREQKR5O3vgK4exlQVm/dBSnPHTgjWURERES+oRFERUREJKsUbIiIiEhWKdgQERGRrFKwISIiIlmlYENERESySsGGiIiIZJWCDREREckqBRsiIiKSVQo2REREJKsUbIiIiEhWKdgQERGRrFKwISIiIlmlYENERESySsGGiIiIZJWCDREREckqBRsiIiKSVQo2REREJKsUbIiIiEhWKdgQERGRrFKwISIiIlmlYENERESySsGGiIiIZJWCDREREckqBRsiIiKSVQo2REREJKsUbIiIiEhWpRVsmNl+ZvaJmU0ws3NWsd9BZuZmNjRzSRQREZFC1mSwYWZFwHXACGAL4DAz26KB/boDpwNvZDqRIiIiUrjSqdnYAZjg7hPdvQK4DxjZwH4XA1cAyzKYPhERESlw6QQbfYEpKa+nJuu+YWZDgP7u/sSqDmRmJ5rZGDMbM2vWrGYnViTXlIelLVA+lta22h1EzawDcBVwZlP7uvtN7j7U3Yf26dNndU8t0uqUh6UtUD6W1pZOsDEN6J/yul+yrlZ3YEvgRTObBOwEPK5OoiIiIgLpBRtvAZuY2YZm1hE4FHi8dqO7L3D33u4+yN0HAa8DP3T3MVlJsYiIiBSUJoMNd68CTgWeBj4CHnD3cWZ2kZn9MNsJFBERkcJWnM5O7l4GlNVbd0Ej+w5f/WSJiIhIW6ERREVERCSrFGyIiIhIVinYEBERkaxSsCEiIiJZpWBDREREskrBhoiIiGSVgg0RERHJKgUbIiIiklUKNkRERCSrFGyIiIhIVinYEBERkaxSsCEiIiJZpWBDREREskrBhoiIiGSVgg0RERHJKgUbIiIiklUKNkRERCSrFGyIiIhIVinYEBERkaxSsCEiIiJZpWBDREREskrBhoiIiGSVgg0RERHJKgUbIiIiklUKNkRERCSrFGyIiIhIVqUVbJjZfmb2iZlNMLNzGth+hpmNN7P3zew5MxuY+aSKiIhIIWoy2DCzIuA6YASwBXCYmW1Rb7d3gKHuvjXwEPCnTCdUREREClM6NRs7ABPcfaK7VwD3ASNTd3D3F9x9SfLydaBfZpMpIiIihSqdYKMvMCXl9dRkXWOOA55saIOZnWhmY8xszKxZs9JPpUieUB6WtkD5WFpbRjuImtkRwFDgzw1td/eb3H2ouw/t06dPJk8t0iqUh6UtUD6W1lacxj7TgP4pr/sl61ZgZnsDvwf2cPflTR107Nixs81sciObewOz00hba1Ka0peLdLV6p2Tl4YzJx3S1izwMq8zH+fh/gfxMl9IUGs3D5u6rfKeZFQOfAnsRQcZbwOHuPi5ln+2IjqH7uftnq5taMxvj7kNX9ziZpDSlL1/T1Zry8W+Qj2mC/ExXPqapteXr3yAf06U0Na3JZhR3rwJOBZ4GPgIecPdxZnaRmf0w2e3PwBrAg2b2rpk9nrUUi4iISEFJpxkFdy8DyuqtuyDl+d4ZTpeIiIi0Efk6guhNuU5AA5Sm9OVrulpTPv4N8jFNkJ/pysc0tbZ8/RvkY7qUpiY02WdDREREZHXka82GiIiItBEKNkRERCSrFGyIiIhIVinYEBERkaxSsCEiIiJZpWBDREREskrBhoiIiGSVgg0RERHJKgUbWWZmd5jZJavx/ifN7KiU15eY2Wwzm56ZFIqI5A8zG2RmnkwCmnNmNimZ1byl719sZhslz7uY2b/NbIGZPZi5VOY/BRt5zt1HuPudAGY2ADgT2MLd12vJ8cxsTzN7IcnskxrYPijZvsTMPl6dL5nknzwsyFUQS5vm7mu4+8Tk5Y+BdYG13f3glhzPzE41szFmttzM7mhg+15J2b0kKcsbnfa9NSnYKCwDgDnuPrMlbzazIqAcuA34bSO73Qu8A6wN/B54yMz6tOR8Ik1RQSytKSkDc2kg8Gkym3qzpFwgfAVcQpTj9ffpDTwCnA+sBYwB7m9xajNIwcYqmNnZZjbNzBaZ2SdJQdXFzO40s3lm9pGZnWVmU1Pes52ZvZ28536gc5rnGmlm75rZQjP73Mz2S9a/aGbHJzUM/wU2SK4G70i2P2hm05OrwZfMbHDKMe8wsxvMrMzMyoE93f1Nd/8XMLGBNGwKDAH+4O5L3f1h4APgoJb+DUWaod0WxO2BmZ2TlG2LzGy8mR2YrC8ys78kzcMTge/Xe98xSVm7yMwmmtlJ9bafZWZfm9lXSVnpZrZxsm2lMtDMvm9m7yRl7RQzG1XveEea2WQzm2Nmv0/zsxWZ2bkpn2+smfVPtrmZbWxmFwIXAD9JyvDjzOxbZvZ8cq7ZZna3mfVKOe6k5HfofaDczIrd/RF3fxSY00BSfgSMc/cH3X0ZMArYxsy+nc7nyCp319LAAmwGTAE2SF4PAr4FXA78D1gT6Ae8D0xN9ukITAZ+DZQQV2qVwCVNnGsHYAGwDxEA9gW+nWx7ETg+eT689lwp7z0W6A50Aq4G3k3Zdkdy3GHJcTunbNsbmFTvWAcCH9Vb93fgb7n+f2hpMr+eA3wOLALGAwcm64uAvwCziQDzFMCB4mT7McBHyfsmAielHHM4MBU4C5gJfA0cAJQCnwJzgXPTSFsRcG5K+sYC/ZNtDmwMXAhUJN+XxcBxyffteaJQnQ3cDfRKOe4k4OzkO7i89jMl2y4B7qiXjhOBV1NedwOW1n7XtGQ9jx4MbJCURT8halnXB04GPgb6E0HgC/Xy6PeTvGDAHsASYEiybT9gOjAY6ArcVZunku0rlYFJvt4qeb01MAM4INl/iyT/7U6UqVcBVcDeTXy23xIXZpsl6dyGqKGjXnpGAXelvG9jotzvBPQBXgKurpfH303+Nl3qnbOhPH4NcEO9dR8CB+X6/6+ajcZVExlgCzMrcfdJ7v45cAhwmbvPc/epwLUp79mJCDKudvdKd38IeCuNcx0H3Obu/3X3Gnef5u4fp5NId7/N3Re5+3LqotieKbs85u6jk+Mua+JwaxBfzFQLiGBG8tvnwG5AT+KH+y4zWx84Adgf2A4YSgTAqWYm23sQgcdfzWxIyvb1iAK6L3FVdjNwBLB9cr7zzWzDJtJ2BnAYEaT0IALkJak7uPsfgMuA+z2aVm4lCu0/Ej9QmxMF7qh6xz6M+DHq5U3XiAwG3ks5Zznxdxvc6DskYzyutr9KyqL7gc+IC61DiDJzirvPJf7nqe97wt0/9/A/4Bki75G893Z3H+fuS1g5f0C9MtDdX3T3D5LX7xNNx3sk+/4Y+I+7v5SUqecDNWl8vOOB89z9kySd77l7QzUP9f8mE5Jyf7m7zyKCmz3q7XZt8rdZmkY68rYMV7DRCHefAPyKyLwzzew+M9uAKPimpOya+nwDYJon4WRichqn608Ues2SVN1dnlTdLSSiYIDejaSvKYuJH4NUPYirUcljWSrIIWoaLnX3SuA+Im9dkwS444halG2aSJ4KYsHMfpY0Fc83s/nAlkR+ql+mTq73vhFm9rqZzU3eV0pdGbeq8rjBdWa2Y9JfZ5aZLSBqVho8XhKQNplXaXkZvm7y2zItKcPvYsXye6X0NyFvy3AFG6vg7ve4+65EW7IDVxBVyf1Sduuf8vxroK+ZWcq6AWmcagpRTdhchwMjiSaRnkRTD8QVYS0nfeOAjcwstfDdJlkveSxLBTlEh+Tq5HntD/qMlO1LiR/xVVFB3M5ZdMS9GTiVaF7oRVTvG1FuppajA1Le1wl4mGgKXDd5Xxl1ZdyqyuNa9cvAe4DHiaa8nsCN9Y73zTHMrCvRWb4pLS3DL0vSt5W79yBqDa3ePs0tw78J/s2sW5KunJfhCjYaYWabmdl3k8y+jChUa4AHgN+Z2Zpm1pf48tR6jWjfO83MSszsR8TVZVNuBY6x6IDawcz6ptmhpzvRVj2HaK+8LI3P1cHMOhPNPWZmnc2sI4C7f0q0D/4hWX8g0ab5cBppkRzJYkGeKSqIpRvxv5oF0emTCIghytTTzKyfma1J9D+q1ZFozp4FVJnZCOB7KdsfIMrOzZPA4Pw00tIdmOvuy8xsB+KirdZDwP5mtmtSLl5Eer+TtwAXm9kmFrY2s3SClO5EELwg+T1p7C7Bb5hZcVKGFwFFSVld20H6/4AtzeygZJ8LgPfTbZbPJgUbjetEdAadTXRAWgf4HZH5pgJfAM8SmXM5gLtXEL2BjyY6z/2E6P2+Su7+Jkl7OVGt+z+iNqUp/ySuVKcR1dmvp/Ge3YnAqYz44VlKVJ3XOpRo259HfP4fJ1XYkr+yVZBnigrids7dxwNXEhdkM4gOmqOTzTcDTxP9ad4mpcx090XAaUQ+nkcEBo+nbH+S6Df3AjCBujJw+SqS8wvgIjNbROSBB1KON47oRH0PEajPI8r7plyVHOcZYCFxAdkljfddSNwBuAB4gjR+L4DziHL7HCIAX5qsIymrDwIuTdK+I1Gm515Le5Zq+aan78+B/+U6HVra90IULnOJ4PgqImA9Higmgtg5RIBc/26UU4jCfz7wL6JfxiXJtuGk3P2UHMuBQSnrXgGOaCJtRURh+AXRZPEW0C/Ztqqe+oOJO1cWEzVuZ9ZLzyTq3SWQHMPrLaNStu9N3PmwlLjTa1BTf1sthbMQHYmrSbkzSUt+LJb8gyRNSQ//jYgIfRMiGv27u1+d04SJiLRDSXNvGdGUfCdQ4+4H5DZVUp+aUZqvI/AP4grteeAx4Pqm3mQx4MviBpYns5xeEZG27CTiFu7PiVqNn2fjJBbzVDVUhp+bjfO1NarZEJHVlgTNuzWw6TJ3b7Ljsoi0bQo2REREJKvUjCIiIiJZlbNppnv37u2DBg3K1emljRk7duxsd2/V2WmVhyWTcpGHQflYMmdVeThnwcagQYMYM2ZMrk4vbYyZpTMsfEYpD0sm5SIPg/KxZM6q8nCTzShmdpuZzTSzDxvZbmZ2rZlNMLP3bcVJnERERKSdS6fPxh3ENL6NGUGMN7EJMYXzDaufLBEREWkrmgw23P0lYmTCxowE/unhdaBXMvCViIiISEbuRunLijMvTk3WiYiIiLTura9mdqKZjTGzMbNmaW4vKTzKw9IWKB9La8tEsDGNFaew7pesW4m73+TuQ919aJ8+rX6Hl8hqUx6WtkD5WFpbJoKNx4GfJXel7AQscPevM3BcERERaQOaHGfDzO4lpprubWZTgT8AJQDufiMx214pMAFYAhyTrcSKiIhI4Wky2HD3w5rY7sApGUuRiIiItCmaG0VERESySsGGiIiIZJWCDREREckqBRsiIiKSVQo2REREJKsUbIiIiEhWKdgQERGRrFKwISIiIlmlYENERESySsGGiIiIZJWCDREREckqBRsiIiKSVQo2REREJKsUbIiIiEhWKdgQERGRrFKwISIiIlmlYENERESySsGGiIiIZJWCDREREckqBRsiIiKSVQo2REREJKsUbIiIiEhWKdgQERGRrFKwISIiIlmlYENERESySsGGiIiIZJWCDREREcmqtIINM9vPzD4xswlmdk4D2weY2Qtm9o6ZvW9mpZlPqoiIiBSiJoMNMysCrgNGAFsAh5nZFvV2Ow94wN23Aw4Frs90QkVERKQwpVOzsQMwwd0nunsFcB8wst4+DvRInvcEvspcEkVERKSQpRNs9AWmpLyemqxLNQo4wsymAmXALxs6kJmdaGZjzGzMrFmzWpBckdxSHpa2QPlYWlumOogeBtzh7v2AUuBfZrbSsd39Jncf6u5D+/Tpk6FTi7Qe5WFpC5SPpbWlE2xMA/qnvO6XrEt1HPAAgLu/BnQGemcigSIiIlLY0gk23gI2MbMNzawj0QH08Xr7fAnsBWBmmxPBhurmREREpOlgw92rgFOBp4GPiLtOxpnZRWb2w2S3M4ETzOw94F7gaHf3bCVaRERECkdxOju5exnR8TN13QUpz8cDwzKbNBEREWkLNIKoiIiIZJWCDREREckqBRsiIiKSVQo2RESyraoKxo6Fm27KdUpEciKtDqIiItIMy5bBW2/BSy/Byy/Dq6/CokWx7fvfh771B2EWadsUbIiIrK6FCyOgePnlWN54AyoqYtvgwfDTn8Luu8NuuynQkHZJwYaISHPNmlUXWLz0Erz7LtTUQFERDBkCv/xlBBa77gprr53r1IrknIINEZGmLFkC//sfPPkk/Pe/8PHHsb5zZ9hpJ/j976PmYqedYI01cptWkTykYEPy25Il8Oab8Mor4A7nn5/rFEl74A6ffhrBxVNPRaCxbFkEF8OHw1FHRXCx/fbQqVOuUyuS9xRsSH6ZORNGj47llVeiB39VVWzbc08FG5I9ixfDCy/UBRhffBHrN9sMTj4Z9tsvAowuXXKbTpECpGBDcscdJkyIoKJ2+fTT2NapE+ywA/z2t9HuvfPOsOaauU2vtC3uMH58XXDx8svRqbNbN9hrr8h7++0HG26Y65SKFDwFG9J6KiujI11qcDFzZmxbay0YNgyOOy6CC1VPSzaUl8Nzz8F//hMBxpQpsX7LLeG002DEiMiHynsiGaVgQ7KnvBxefz2uGF95BV57LfpgQFwt7rtvBBa77grf/jZ00BhzkgWTJsETT0SA8cILsHw59OgBe+8NF1wQ+bB//1ynUqRNU7AhmTN7dgQVtcHF229Hfwsz2GYbOPbYuuBCYw1ItlRXR5D7n//E8uGHsX7TTeGUU2D//SMPlpTkNp0i7YiCDWkZd5g8uS6wePll+Oij2Jba32K33WCXXaBnz9ymV9q2+fPh6acjuCgrg7lzobg4OnRedVWM2rnpprlOpUi7pWBD0jd7dhTmzzwTwcXUqbG+Z89o5z7yyAguhg6NWwRFsumTT+Df/448+corUaPRu3fUXOy/P3zvewpyRfKEgg1Ztc8+g8ceg8cfj9tRa2pgvfXqhl7ebbfoXFdUlOuUSntQXR0BxtVXx9gXAFtvDWefHQHGDjsoL4rkIQUbsqKamhhE67HHYqltGtlmGzjvPPjhD2M4ZrPcplPalwUL4NZb4e9/j/EvBgyAK66AQw+N5yKS1xRsCCxdGrcDPvZYXDXOmBFXh3vsEYMZ/fCHMGhQrlMp7dFnn8G118Idd8SgW7vuCn/+M4wcGX0yRKQg6NvaXs2eHbcDPvZYdKxbsgS6d49xBkaOjEcNoiW54A7PPgvXXBOdPYuLowbj9NNj/BURKTgKNtoLdxg3rm68gVdfjSaTfv3g6KMjwNhjDw1mJLmzZAncdVfUZIwbB+usE8PT//zn0U+oQNTUxM0xc+bETTFz5tQtc+fG0B6qlJH2Rlm+LVu2DF58sW68gcmTY/2QITFL5ciR6n8huTd1Klx3Hdx0U/wab7st3H571Gbk0V1N5eUxAO6HH8YM8/UDidpgYt68CDgaYhazz/fp07ppF8k1BRttzbRpUfX8n/9EVfSSJdC1K+yzTwQYpaUaUEvyw7hxcMkl8OCDUfM2ciT86ldxh1OOA+ClS+G992DMmFjGjo1pVFKDiDXWiFH21147loED4zF1Xf3XvXppoFxpnxRsFLqamigNa2sv3nkn1g8cCMccE7cDDh+eV1eIzeEOb7wRv0dVVdGMLwXuk0/gwgvhvvti0rPTT4dTT83ZhGfLl8P7768YWHz4YdxlC9Ga853vwEEHxRAy22wT69TiKO1OeXlMlrnhhhE5N4OCjUJUWRm1Fg8+GH0wZs6My6VddoE//jECjMGDc3512FI1NXUBxkMPxVxZHTvCAQdE8FGgH0s+/xwuvhj+9a8Ifs86C37zmxiIqxUtWBB56803I7D44IP4SkEkZehQ+MEPoi/q0KFREag8J+2Ge7QTfvxxDH2Q+ljbFP/oo1ET2QwKNgqFe0xkdvfd8MADcTdJz55x18j++8dU2GuvnetUtlhNTXy8Bx+Ehx+OZvyOHWOOrEsvjbtvNRhkgZo8OZpL7rgjekb+6lcRaKy7bqsmY/r0qBm7/npYuDAuzIYOhTPPjMehQ2PIDgUW0i5UV8d3s35A8dFH0fmoVteusNlmdbNyb7457LRTs0+XVrBhZvsB1wBFwC3ufnkD+xwCjAIceM/dD292amRl48ZFgHHvvTF7ZefO8ct7+OERYBRwXW5NTQxK+tBDEWBMmxYfZ999o4LmBz9QgFHQpk2Dyy6Dm2+OX/CTT4bf/Q422KBVkzFxYgzNcfvtUFEBBx8c0/Zsv70CC2kn5s6NTkjvvhvLe+9Fc+ayZXX79OkTgcSPfxyPm28es3H375+RjkZNBhtmVgRcB+wDTAXeMrPH3X18yj6bAL8Dhrn7PDNbZ7VT1p59+WUEF/fcE43JHTpEB88LL4y2hB49cp3CFquujgCjtgbj668jwBgxAv70p6ikKeCPJxBVCFdcATfcEP/w446Dc89t9ZE+338fLr8c7r8/KlSOOiqCjE02adVkiLSe2gkya4OK2qW2+QNg/fWj49Hee0cwURtUZLlmPJ2ajR2ACe4+EcDM7gNGAuNT9jkBuM7d5wG4+8xMJ7TNmzMnLvHvvjsmOYOoqrr2WjjkkFavcs6G22+P35zp06OCZsSIuMrcf/8YT0wK3OzZETH+/e9RhfCzn8U4Ga3c8fOVV6JmrKws7hg54wz49a9bvUJFJLsqKqLJ491348aA2sBiwYLYbhbNHzvvHGPVbLddBBk5+i1JJ9joC0xJeT0V2LHePpsCmNlooqlllLs/Vf9AZnYicCLAAM1nEJnlkUciwHjqqbjd4tvfjk50hx0G3/pWrlOYMffeGxe4w4bBX/8aAcYaa+Q6Vc2nPNyAefPgyiujQ0R5Ofz0pzFyVStWIbhHX+nLL4+as96942t0yikaCLchyscFxj2G7n/zzbrlnXfiNwSiX8XWW8fvxrbbxrLllnG3V57IVAfRYmATYDjQD3jJzLZy9/mpO7n7TcBNAEOHDvUMnbvwLF4c7dhXXhnt2n37Rqe5n/40Is821pBcVhYXubvvHjFVgd6FCygPr2Dhwph99aqr4mrqkENg1Kiolm0lVVXRTHLFFXFXyYAB8Le/wbHHRvkrDVM+znMzZqwYWLz5ZgxLCxFADB1aN3z/ttvCxhvn/WzH6QQb04D+Ka/7JetSTQXecPdK4Asz+5QIPt7KSCrbitmzoyT829/ianD48Ag69t23zY7088or0d9o661jlvpCDjQksXhxNJX8+c/R8eyAA6I/0dZbt1oSqqriq/OnP0W/6S22gH/+MwYdLSlptWSIrL7y8rgHOzWwqO1jUVQEW20VgfwOO8COO0Yw34qBxZIl0bdu+vRYvv46aqabOzdnOsHGW8AmZrYhEWQcCtS/0+RR4DDgdjPrTTSrTGxeUtqwL7+Mq7+bb47/3MiRcM45Lbp9qJC8915kygEDokZDHT8L3NKl0enz8svjPvzSUrjoolafHO3tt+H446MWeaedovVm//3bbLwubYk7TJgQ9/m/9lrMUfXhh3VD0264YWTq006L4GLIkKxU0VVXx7Vv/SCi9nnq60WLVn7/eutlIdhw9yozfFbgYQAAHZpJREFUOxV4muiPcZu7jzOzi4Ax7v54su17ZjYeqAZ+6+5zmpeUNuijj+LS66674vXhh8PZZ8dlWBs3YUJU2HTvDs88o7kgCtry5REoX3ZZlEB77x1Bxs47t2oyliyJCpQrr4z89PDDcOCBba7VUdqS8vIYlvbVV+sCjNmzY1uPHlFTccABEVh85zsxNO1qqKqKFpivv47lq69WfKx9PmNGw/P39OgRgcR660WcU/t8vfXiJpba5y0Zhy+tPhvuXgaU1Vt3QcpzB85IFnnzzbj6e/TRaDf4xS+iS/zAgblOWauYNi3u1K2ujnng1P+sQFVWxi1El1wSw7juvnsMMb777q2elBdegBNOiEFIjz8+WnCaOVqySHa5R5teaq3Fe+/VjXu/2WZRBbfLLrDzztRstjlffFnEvHnRz7PyI6h4L56nLpWVK6+rqIhrgFmzVgwmZs6MZKQyi+B8gw3q7npdf/26JTWgyGY/J40gminuMYT4H/8YJWOvXnDeeXk5xeM778Q4F6edlvmZu+fMge99L4L3F16Im2ukwFRVRW3cRRfBF19Ete5tt8Fee7V6NcL8+TE2xi23xM1Zzz8Pe+7ZqkkQadjy5dGmN3p0Xc3F9OmxrVu3qK0455wILHbYic/nr83YsdE9Y+y98bhwYctPX1ISNQzrrw/9+kXFyPrr1wUVtc/XWSc/+jEp2FhdNTVx++rll0fu2WAD+Mtf4MQT83LwiJdeiuB60aKY1fvii6PipTgDOWHxYvj+9+Pq88kno8O0FJDq6ri148ILY7KlIUOiM3NpaU7aKh55JG5dnTUrRjcfNQq6dGn1ZIiEOXMiqBg9Opa33oqAA2CjjaJ5ceedqdlpFz7vsiVj3yuOwOLKiElqh7/o2DFqFw4/PLo7rbderKtdSkpWfN3Q+pKSwms+VLCxOl5/PW5ZfeONGFPgllvgiCPydgjxp56KNu5Bg+DGG6MJ/vTT4dZbY76IYcNafuzly+PYY8ZEW7quPguIOzz2WNTEjRsX9+c/8ki0JeegRPv665gE9pFH4q6+J56IuEek1dSOa1EbWIweHfOGQPzSDxkSkfCwYUzqvxtvTOwTMwY/BG+fu2JgUTv8xfbbxzJ4cKxvbxRstMTUqVE9dvfdUVd1xx0RZOTxfc4PPRSR9JZbwtNPR8vO7rvD//1fxEu77hrDOV9xRfMHmKuujo//7LPxp2jmZICSS5MnR6H5xBPR5nXffTGsaw5u7XCPwPc3v4ng9fLLo6tTPlQBSxtXURFXSrWBxauvRpUaxKhwu+wSgwUNG8bcb32H51/rwrPPwrNnRU0u1AUWhx4atbrtObBokLvnZNl+++294JSXu194oXvXru6dOrmfe677okW5TlWTbr/dvUMH92HD3OfNW3n74sXuv/ude0mJe8+e7n/7m3tlZXrHrqlxP/54d3C/6qqMJrtZiDujlIfTVVXl/te/unfrFvn5yivT/6dnwWefue+5Z+Sj4cPdP/00Z0nJmVzkYS/0fLw6Kircn3zS/eijo+CLeNd9443djzrK/aab3MeN86Xl1f7cc+7nnOM+dKi7Wey2xhruP/iB+zXXuI8d6758ea4/UO6tKg8r2EhHTY37vfe69+8ff7KDD3afODHXqUrLtddGkvfZJ4KKVfn449gP3Lfd1v3VV5s+/tlnx/6//31m0ttSCjaa4e233bffPv5xI0a4f/FFzpJSWel+xRXunTtHeX/zzfF1a48UbLSCqir35593P+kk97XXju9Ajx4RcDz8sPv06V5dHV+RP/0pysPOnWO34mL3XXd1HzXK/ZVXIlaRFSnYWB1vvRVVAuC+3Xbu//tfrlOUlpoa90svjWQfcID7smXpv+/BB9379Yv3Hnus+8yZDe97xRWxz8kn5/4HQsFGGhYvdv/Nb9yLitzXXdf9vvty+o+bObMu5jnwQPdp03KWlLygYCNLqqvdR492P+009/XWiwzXtav7YYe5P/aY1yxd5hMnRqB7yCF1MQi4Dx7sfvrp7v/+t/vChbn+IPlPwUZLfPVVRLvgvs467rfcElFxPTU17mPGNLgpZ2pq3M86K5J+xBEtqx1ftChqLYqL3Xv1cr/++hU/4803x/EPPTQ/PruCjSY8+eT/t3fvcTbX+R/AX28yrqMsI7mtahG7opKskluby4rEUpqKbispFRU/0epmw0YXaqdYxjCuKa1bZV0qiSLkLtGY0MhlhmHGOO/fH+8zZow55szMOef7PTOv5+NxHs6Z79c57znzme+8z+fy/qjWqWM/tEceUT1yxNFwUlNVmze3T41z5jgaimsw2QigzAvz4MGqtWtbuy9dWvWuu9Qzc5bu3nRSJ01Sve++rA5rQLV6ddX771eNjWXyWxBMNvLj1CnrEihfXjUiwv5qHz+e66l796q2a2fvYvfu/vceBNPZs9bTAKg+9pg9LoytW1XbtrXnu/561TVrVOfOtTkgHTq4Z5ySyYYPBw/aJzhA9ZprVFetcjoiPXvWfl9ErOeaDJONANi82cZ0//AHzRz78HT6q+4aM1/ffytVo6Ozem0B1ago1R49VN95R3XLFud7aMMdkw1/ZI4fZH76u/NO1d27fZ4aE6MaGWmThDI7QNq08ZmXhMSZM9aTAVivRKB+cTwe1VmzLOvPHLts0cLmy7oFk40cPB7VSZNUK1WypPkf/3BHNqyqzzyjjk8odiMmGwV04oRdkJs0UQXUIyV0Z4sHNOa+Vdq7x2mtUSMruaha1YZKJk5kchEMTDbykpiYNRW+USPVZct8nvrzz6rt29upbdtmza2bNs3+CF93nX2YDLXTpy0/AlRfey04r5GcrPrss6qdOjneC38BJhvZ7NhhSzoA1ZYtrXvKJd5+28J64gle6HNisuG/M2dUE1fu0vV3v66Ly3XT/+AB/We1N/SeG3Zo9WoZ55KLatVUe/VSffdd1W3b2OaC7WJtmHU21q8HunSxKizvvmsbL+RSTlPVakg89ZRVc54wAejXL6scQXQ0ULky0L271az49FPbwC8UTp60glqffWYFHwcMCM7rREbavnLkUunp9gN65RUrtfn++8CDD7pmO9QFC6yIXNeuwLhx4VcBkYIvLc32rzx40DYLO3Qo533FoYR0/JZcCoo/AHgu6z8fBK4QoHUboFUroHVroF49tjO3KN7JxkcfAffea1nCV19ZRZZcJCZa9fFFi6wQ1n/+Y9Vpc+rYEVi2zEp2t2hhxbN8PGXAHDtmr7dmjSVDDzwQ3Ncjlzp2zMolf/cd0KsXMH584De+KYR166zY0Q03ADNmuLr+HTkgIwOYOtVK0u/ff/6xcuWAalEZuNxzAHWTtqDl6Z9wecXTqNb6Glx+559xef3LUK2aFSMsX96R8Mkfvro8gn1ztOvO47F1myKqzZqpHjjg87TYWFuNUbasFW/xZ8Llli02CenSS4O7UvbQIRumLFXKJm0WZyjOwyipqaq33moNwYUzLvfssbHyK690ZogxXDjRhtXhdpw5Va5+fRv2uOkm1RkzrI7F7l0eTVm0yuoaXXKJndC+verHH7tjCRxd4GJtuPglG2lpqn372rfes6ddqHPxyy9WHQ6wMhv5rWi4b59N/i9dWvWjjwIQdw7r1tnzly1rqxqLu2KbbJw5o9q1qyXOM2c6Hc0Fjhyxdlqpko2Zk2/FLdn47DOryAmoNmxo10mPR22W/Tvv2BcBazyDBhXPsrJhhslGpsOHVVu1sm97xIhcuyk8HtXp0619lyljVZwLmkQnJVnHSYkStjCgsDweSywy57JWqhQ2NcaCrlgmGx6PVV0DbOaly5w+bb9uERFsp/4oLsnG2rVZJQNq11adMsV7jU1JsZrg5cvbwRtuUJ082ecHQnKfYpNsnD17kdnG27fb2uuICNW4uFxPOXjQKhlmducF4pNYSkrW6pVRowo2Gzo93YZzGjWy56lRQ3XsWGeX2bpNsUw2hgyxBjF8uLNx5OLs2azyHjNmOB1NeCjqycbWrap33aXn6lu8+aZ3NbbHY+PAmQUweve2jITCzsXacJGZIJqSYptWHjpkqybOu535DZGbd6JCiVcR2bklIjdfgchXzz/nyBHg//4POHHCdj4dNCgwk9gqVLBZ+H36AEOHAr/+Cowd698CgZQU27V+3DggIcF2EJwyxbYr5k6Cxdy4cbYt6t//Dowc6XQ0F3jhBSA+HnjtNWuvVHz9/LM10SlTbALnyJHA00/bdRe7dgFPPGGz6Rs3BmbNstn1VOQUmWRj/nzgl1/s2hsRYX+oU1KAlG37kbI1AQdL10dKlauQsuISpHwCnDlz4XPceKP9QjRsGNjYIiKAuDjb1n3cONu5ePJk31tnHzwIvPWWrcQ9dsyWcb37rq12cckqRnJSXJztvd69u63BdtnavpgYYNQo4JFHgCFDnI6GnHL4sCWbEyda6YCBA+0DV1QUgNRUYPgoW6pdpgzw5ptA//65lh2gIsJXl0ewb4HuurvtNpvtfm6YIiMjq1Rhhw4XjDmcPm1zKvbsUd240SZcBnuH7eybo3XseOEurNu327YVERE23697dysPTnlDcRlGWbjQZua3beuaiqDZLVpk+7x16ODojvVhyYk2rEFoxykpqiNHWoXlEiVsWtG+fdlOWLAgq1LzvffabHwqEi7WhotEspGYaH+czw1dJyerdu5s396TT7ruqvf++/ZL2Ly5zVldvdqqf4rY6pV+/TjxOr+KRbKxerUtP7r+eldO2Nmwwcr3N2nCHTILoigkG9u22eojwOZnnFe8ds+erCV+DRuqLl8esNcld7hYGy4SfVYzZlg3XXQ0bIDwjjuALVusi7l/f6fDu8DDD1sdsXvuseJgyclApUo2zj1gAFC1qtMRkuts2WLV22rUABYvBipWdDqi8yQkWHiXXQYsXOgdj6diZd48m5tWtizw+edAu3beA2lpwJgxwKuv2kS4MWNsTMXXODIVSUUi2YiLA5o1A+od/cZqIZ86ZVe89u2dDs2nbt1sTtSIETb0/uCDNpmU6AL79llbLlPG6uAHOBtVBTZutKS3fPmsW7ly9m/p0hefFnL8ONCpk02u/vJLoHr1gIZHLpeRYR+UXn/drsPz5gE1a3oPLl1qn6B27wb+9jfgjTeyHaTiJOyTjc2b7UL59qvJlkpXrQr873+Bn+UZBK1aAStXOh0FuVpSEnD77bYBzqpVAdtwR9V+d2bOtNtPP/k+t0SJCxOQ7LeffgK2b7cOl0aNAhIehYmkJOuhXbbMJue/+aYlp0hIsCUn8+YBdeta0nH77U6HSw4K+2QjLs4mMPc6MN56NJYutcZNFO5SUqzL4OefbZe9APwl37HDVhfOnGkbXpUsaVuqDB8O1K5tOU3mLTU178e//QZ4PEBsrD0PFR/ffgvcdZct5588Gejb13vgo49sTNvjsaGTQYO8GQgVZ2GdbHg8wPTpQId2ZxA1dax10zHRoKIgPd2u5Bs22LruW24p8FPt3QvMnm0JxoYNNiRy663Ak0/aEF5UVODCpuJh0iTg8cdt87OvvrIN9gBYV8dDD1nRo7lzgTp1nAyTXMSvqg0i0kFEdojIbhHxuXJeRLqLiIpI08CF6NuKFbYja/Rl/7VPgVzUT0WBxwPcf7/NsvvgA5vwnE8HDlitlhYtbOTl+eet3ktmgbgVK4B+/ZhoUP6kpdlwycMPAy1b2ibD5xINwOq/pKRYVxcTDcomz54NESkJYAKAvwDYD2CdiCxQ1a05zosEMBDAN8EINDdxcUBkpKLLsoFW8apJk1C9NFFwqLf60axZVvCoTx+fp6Wn28U/Lc3up6YCy5dbD8aKFXZO48ZWYKtnT1v5RFRQCQlAjx7A2rVWnOvll3NUWf70U7sojxgRFnPmKLT8GUZpBmC3qu4BABGZCaArgK05znsZwOsAng1ohD6kplovXY8/bkPZNQnA0OmheFmioPF4gBdvW43ly3shvdqzSIurjbRJWclEZmKRlpZ7BdxM9erZ9b5XL6BBg9DFT0XX8uXWnk6fBj780FbTnSc11brK6te3TIQoB3+SjRoAErI93g/gpuwniMj1AGqp6kIR8ZlsiMijAB4FgNq1a+c/2mw++cR666L3vAzcfLP16REFWSDbcHYZGcBDvU4gdvnN+HPlHajcuBZKl7F5dZm3iIi8H197rXXwuayCObmMv+1YFfjXv2wYrn59SzSuuSaXE196yZYlrVhhS7SJcij0BFERKQHgDQB98jpXVWMAxABA06ZNtTCvGxcH1PzdSbT+dRYw+ZPCPBWR3wLZhjOdOWOT92d/WAEvX/IPvPD9w0BNZgsUPP6045QUm+s5Z44Nn0ye7KNY28aNtrvkQw/Zen6iXPgzQTQRQK1sj2t6v5YpEsCfAKwQkb0AmgNYEMxJoklJwJIlit4SjxLXNrLlgURhKC3NFlHNng2MxWC88HQqix6R43bsAG66ycpkjB5t7TPXROPsWdtxr3JlO5HIB396NtYBqCsiV8KSjLsB9M48qKrHAVTJfCwiKwAMVtVvAxtqllmzgIwMwX2/jQfeHsY+YwpLp07Z6tYlS4B3/vQeHk/4ABiyx+mwiPDxx/ah7rPPgLZtL3LihAnAunVAfDzwu9+FLD4KP3n2bKhqBoABAJYC2AZgtqpuEZGXRKRLsAPMzbRpisblduJPV52yj4VEYebkSaBzZ6tB98Hzu/D4D4/ZwDgv2OQCgwcDP/yQR6KRkAAMGwZ06GCzR4kuwq85G6q6CMCiHF8b4ePc1oUPy7edO4G1awVjEAM8+6yVDyUKI8nJNvL39ddA7FRF9L/7AtWqWZUtIhcoUcIKdvmkalW9PB5g4kT2LlOewu4v9fTpgMCD3lGfA31ecToconw5etQ+CK5fb8OBPcostBKMEyfaRiNE4eDDD21J4NixAduvh4q2sEo2VIG4SafRDl+g+uDeXGJFYSVzT7WtW+1afUens0CTocDVV1tJRqJwcOwY8MQTwHXXWQE6Ij+EVbLx9dfAnsQyeLHcPKAfZz5T+Dh40DYl3rMHWLDAdoxHXLwNjMfHA6VKOR0ikX+GDgUOHbKeDQ5jk5/CqqVMe+soyqI0uvWvDlSs6HQ4RH7Zv98SjcREYNEioE0b2JrX4cPt02HPnk6HSOSfr74C3nvPto8/b1MUoosLm2QjPR2YNb8UupVcgMjnHnM6HCK/7N1rM/oPH7aVJzff7D0QE2MH33vPZuMRuV16OvDoo0Dt2lYxlCgfwibZWDQ1CUfToxB9x1FuVUlhYdcuSzROngSWLQNuvNF7ICXFdrFq08YmcRCFg9GjbcLRwoVAhQpOR0NhJmySjbjXE1EVHvxl/F+dDoUoT1u32tBJRoZtYtW4cbaD48bZbNFRo7hkkMLDjh2WIPfqxYrNVCBh0X97bPdhfPJjA9zTYCMuuSpwm18RBcPGjVlbRKxcmSPRSEoCxoyx0qE33ZTr/ydyFVXb0bVcOWD8eKejoTAVFsnGnIFfIh2lET2yrtOhEF2UKtC/v63KXrUKaNgwxwmvvWbbcb/CGjEUJqZMsd1cR4+24nNEBeD+YZTkZMQtjcI1Ffbjhh4sHkPuJmK7ZKal5VLraN8+K97Vty/QoIEj8RHly6+/AoMGAS1b2q6uRAXk+p6NvaPiserszYiOVg5vU1ioXt1HUcUXX7Rs5MUXQx4TUYE8/bTNcP73v7lqigrF3a3n9GnMeOcIAODe52vlcTKRi/3wAxAbCwwYANRiW6YwsGQJMGOGFfFiTxwVkquTDZ0ai2kn7kTLRkdRp47T0RAVwrBhQGSkXbiJ3O7kSeCxx4D69dlmKSDcm2xkZGD9KwuxHQ1w34DLnI6GqOBWr7Ya5c89B1Su7HQ0RHn75z+t6FxMDFC6tNPRUBHg3gmic+cibn9rRFxyFj3+VtLpaIgKRhUYMsT2637qKaejIfLPwIG2QeCttzodCRUR7kw2VJExagziSy5B5ztKoFIlpwMiKqDFi4EvvgAmTOAW8hQ+qlQB+vRxOgoqQtyZbCxejM83ReEQonDf/U4HQ1RAHo+Nd199NfDII05HQ0TkGHcmG6NGYVq5Z1CptKJjR653pTAVHw9s2mQz+rmFPBEVY+6bIPrll0j58nvMP9MZvXoJ5yZReEpPty3kmzSx/SSIiIox9/VsjBqF+RXux6kTpRAd7XQwRAUUEwP89JPN2WAxJCIq5tyVbBw/DmzahLjL38aVUUCLFk4HRFQAJ07YDpmtWgHt2zsdDRGR49yVbFx6KX754kcsu7oUhg3j7tsUpsaNsz0lPv6YjZiICG5LNgDEz4uAxwMOoVD42rUL6NYNaN7c6UiIiFzBdclGXBzQrBlQr57TkRAVUGysTRAlIiIALks2TpwAKlYEevZ0OhKiQoqIcDoCIiLXcFWyUaECsHKlVXgmIiKiosGvNXki0kFEdojIbhEZksvxZ0Rkq4hsEpFlIvL7wgTFOXVERERFR57JhoiUBDABQEcADQHcIyINc5y2AUBTVb0WwFwAowMdKBEREYUnf3o2mgHYrap7VDUdwEwAXbOfoKrLVTXV+3ANgJqBDZOIiIjClT/JRg0ACdke7/d+zZeHACzO7YCIPCoi34rIt0lJSf5HSeQSbMNUFLAdU6gFtI6yiEQDaApgTG7HVTVGVZuqatOoqKhAvjRRSLANU1HAdkyh5s9qlEQAtbI9run92nlE5DYAwwC0UtW0vJ70u+++Oywi+3wcrgLgsB+xhRJj8p8TcRVqUnJBsA0HjBvjKhZtGLhoO3bjzwVwZ1yMyfhsw6J5rDMVkUsA7ATQDpZkrAPQW1W3ZDvnOtjE0A6ququw0YrIt6ratLDPE0iMyX9ujSuU3PgeuDEmwJ1xuTGmUHPre+DGuBhT3vIcRlHVDAADACwFsA3AbFXdIiIviUgX72ljAFQAMEdEvheRBUGLmIiIiMKKX0W9VHURgEU5vjYi2/3bAhwXERERFREBnSAaQDFOB5ALxuQ/t8YVSm58D9wYE+DOuNwYU6i59T1wY1yMKQ95ztkgIiIiKgy39mwQERFREeFosuHHniulRWSW9/g3IlInyPHUEpHl3n1etojIwFzOaS0ix70TYb8XkRG5PVeA49orIpu9r/dtLsdFRN7yvk+bROT6IMdTP9v3/72IJIvIUznOCfn75AS24XzFxnbsQmzD+YqNbbigVNWRG4CSAH4EcBWACAAbATTMcU5/AO95798NYFaQY7oCwPXe+5GwJb85Y2oN4L8hfq/2AqhykeOdYFVbBUBzAN+E+Od4EMDvnX6fQn1jG853bGzHLruxDec7NrbhAt6c7NnIc88V7+Op3vtzAbQTCd6esKp6QFXXe++nwJb6Xqw0u1t0BRCrZg2Ay0TkihC9djsAP6qqr+JWRRnbcGCxHYce23BgsQ374GSy4c+eK+fOUav3cRxA5VAE5+0qvA7AN7kc/rOIbBSRxSLyxxCEowA+FZHvROTRXI7nd/+aQLobQLyPY6F+n0KNbTh/2I7dh204f9iGC8ivOhvFjYhUADAPwFOqmpzj8HpYN9UJEekE4CMAdYMc0i2qmigiVQF8JiLbVXVVkF8zTyISAaALgKG5HHbifSIvF7ZhgO2Y8oFt2H/h0Iad7NnwZ8+Vc+eIlU2/FMBvwQxKRErBGvh0Vf0w53FVTVbVE977iwCUEpEqwYxJVRO9//4KYD6s6zM7v/avCYKOANar6qGcB5x4nxzANpwPbMeuxDacD2zDBedksrEOQF0RudKbld0NIGeZ8wUAHvDe7wHgf+qd8RIM3nHISQC2qeobPs6pljleKSLNYO9h0H7xRKS8iERm3gdwO4Afcpy2AMD93pnQzQEcV9UDwYopm3vgo9su1O+TQ9iG/Y+L7did2Ib9j4ttuDCcmJWaeYPN3N0Jmw09zPu1lwB08d4vA2AOgN0A1gK4Ksjx3AIbk9sE4HvvrROAfgD6ec8ZAGALbNb2GgAtghzTVd7X2uh93cz3KXtMAmCC933cDKBpCH525WEN9tJsX3PsfXLqxjbMdhzuN7ZhtuFQ3FhBlIiIiIKKFUSJiIgoqJhsEBERUVAx2SAiIqKgYrJBREREQcVkg4iIiIKKyQYREREFFZMNIiIiCiomG0RERBRU/w9JA32FfTologAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 648x432 with 6 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ((ax1, ax2, ax3), (ax4, ax5, ax6)) = plt.subplots(nrows=2, ncols=3, sharex=True, sharey=True, figsize=(9,6))\n",
    "ax1.set_title('sgd_mnist')\n",
    "ax1.plot(d_opt['main_accuracy'][0] ,'r')\n",
    "ax1.plot(d_opt['val_main_accuracy'][0] ,'b')\n",
    "# ax1.plot(d_opt['loss'][0], 'g' )\n",
    "\n",
    "ax2.set_title('adam_mnist')\n",
    "ax2.plot(d_opt['main_accuracy'][1],'r', label=\"adam_acc\")\n",
    "ax2.plot(d_opt['val_main_accuracy'][1] ,'b')\n",
    "# ax2.plot(d_opt['loss'][1] ,'g')\n",
    "\n",
    "ax3.set_title('adagrad_mnist')\n",
    "ax3.plot(d_opt['main_accuracy'][2] , 'r')\n",
    "ax3.plot(d_opt['val_main_accuracy'][2] ,'b')\n",
    "# ax3.plot(d_opt['loss'][2] ,'g')\n",
    "\n",
    "ax4.set_title('sgd_cifar10')\n",
    "ax4.plot(d_opt['main_accuracy'][3] ,'r')\n",
    "ax4.plot(d_opt['val_main_accuracy'][3] ,'b')\n",
    "# ax4.plot(d_opt['loss'][3] ,'g')\n",
    "\n",
    "ax5.set_title('adam_cifar10')\n",
    "ax5.plot(d_opt['main_accuracy'][4] ,'r')\n",
    "ax5.plot(d_opt['val_main_accuracy'][4] ,'b')\n",
    "# ax5.plot(d_opt['loss'][4] ,'g')\n",
    "\n",
    "ax6.set_title('adagrad_cifar10')\n",
    "ax6.plot(d_opt['main_accuracy'][5] ,'r')\n",
    "ax6.plot(d_opt['val_main_accuracy'][5] ,'b')\n",
    "# ax6.plot(d_opt['loss'][5] ,'g')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAADSCAYAAABjNopPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3yUVdr/8c81qfQa6ZBIDagUI00EARUE22JZxS7oro91H3etK12WXcuqj+2n2HtDRUFRuq6KBrDQlKp0gpRASJnMXL8/zkQCG2ACk7knw/V+ve5XMvfcM+eacPLl5NxNVBVjjDGxy+d1AcYYYw7OgtoYY2KcBbUxxsQ4C2pjjIlxFtTGGBPjLKiNMSbGWVAbY0yMO+qDWkTWiEiRiNTfb/1CEVERSQ89fiH0uGupbVqJiJZ6PFtEhpd6fLeIrBaR3SKyTkTeDK1fHFq3W0QCIlJQ6vHdZdQ4SkReifynN/FCRIaKSHaoD20UkY9FpFfouX36T6gf55XqcztKPXdq6Pk79nv/9ND6ktesEZE799vmxlANhSLyQhk19heRZSKyR0RmiUiLg3yeNSJy2hH8SOLKUR/UIauBS0oeiMjxQNUyttsGjAvnDUXkSuBy4DRVrQ5kATMAVLWDqlYPrf8cuLHksaqOP7KPYo42IvK/wMPAeKAB0Bx4Ajj3IC/rWKrP1S61/kpcP7/iAK+rHeq3FwD3isjppZ7bgPv9eK6MGusDk4B7gbpANvBmGB/PYEFd4mX27ZhXAi+Vsd2LwAki0ieM9zwJmKaqKwFUdZOqPn3Ele5HRM4JjdB3hEb0maWeu0NE1ovILhH5SUT6h9Z3DY18ckVks4g8FOm6THSISC1gDHCDqk5S1TxV9avqh6r6t3K+VzVcAN8AtBaRrANtq6rZwGKgU6l1k1T1feC3Ml4yBFisqm+ragEwCugoIu3KWWOKiDwsIhtCy8MikhJ6rr6IfBT6XdgmIp+LiC/0XJm/C5WFBbXzNVBTRDJFJAG4GChrqmEPbtRyX5jveYWI/E1EskLvG1Ei0gZ4HbgVSAOmAh+KSLKItAVuBE5S1RrAAGBN6KWPAI+oak2gJfBWpGszUdMDSAXei8B7DQF2A28D03ADljKJSHfgOGBFmO/dAfi+5IGq5gErQ+vL4x6gO+4/iI5AV+DvoeduA9bhfhcaAHcDeojfhUrBgnqvklH16cBSYP0Btvt/QHMROfNgb6aqrwA34TrFHGDL/vN+EfBHYIqqfqaqfuABoArQEwgAKUB7EUlS1TUlo3vAD7QSkfqqultVv45wXSZ66gFbVbW4nK9bEBp57hCRR0PrrgTeVNUA8BpwsYgk7fe6rSKSD3yFm155P8z2qgM791u3E6hRzrovBcao6hZVzQFG46YYwfXrRkCL0F8Vn6u7mNHBfhcqBQvqvV4GhgJXUfa0BwCqWgiMDS0HpaqvquppQG3gz8BYERkQkWqdxsAvpdoLAmuBJqq6AjfSHoX7T+INEWkc2nQY0AZYJiLfishZEazJRNdvQH0RSSzn67qoau3QcrOINAP6Aq+Gnv8AN1IfvN/r6uNC9zbgVGD/ID+Q3UDN/dbVBHaVs+59+nzo+5J+fT9uhP+piKwq2dl5iN+FSsGCOkRVf8HtVByE2+lxMM/jwndImO/tV9W3gR9wfy5Gygbg9z3nIiJAM0J/Dajqa6raK7SNAv8MrV+uqpcAx4TWvROanzSVz1dAIXDeEb7P5bg8+FBENgGrcEH9X9MfqhpQ1YeAAuB/wnz/xbipCuD3+fCWofXlsU+fx+043RCqa5eq3qaqxwLnAP9bMhd9oN+FysKCel/DgH6h+bMDCv2ZORI44FSGiFwlIoNFpIaI+EJTJR2AeYdZm09EUkstKbi55cGhw56ScKOcQuBLEWkrIv1C2xUA+UAwVNtlIpIWGoGXHJoVPMy6jIdUdScwAnhcRM4TkaoikiQiZ4rIv8rxVlfiphE6lVrOBwaJSL0DvGYCcLuIpAKISGLo+wQgIdRPS0b67wHHicj5oW1GAD+o6rKD1JS0X59PxO2T+buIpIWOJBlBaH+SiJwl7pBZwU2rBIDgwX4XKg1VPaoX3E6F08pYn4j7nzc99PgFYFyp533AIvcj/H3dbGB46PshwH+A7UAu8CNwVRnt/P6ag9Q4KlRL6WVd6Lk/AEtwHXMO0CG0/gTgG9yfltuAj4DGoedeAbbg/hxdDJzn9b+DLUfcjy/FHfKWB2wCpgA9S/WfV0ptq0CrUo+74wIsrYz3XYzbEZceel1iqeck9PxNB+mno0ptfxqwDBeUs0t+tw7wedaU8V7jcKP8R4GNoeVRIDX0mr+EXpeH26l4b2j9AX8XKssioQ9ijDEmRtnUhzHGxDgLamOMiXEW1MYYE+MsqI0xJsZZUBtjTIwL62wmEakNTMSdrKHANar61YG2r1+/vqanp0ekQGP2N3/+/K2qmhbtdq1fm4p0sH4d7mmnjwCfqOoFIpJM2ZcA/V16ejrZ2dnlLNOY8IjIL4feKvKsX5uKdLB+fcigDl1GsTfuGhioahFQFKnijDHGHFw4c9QZQA7wvLi7nkws67oQInJd6BrH2Tk5OREv1BgvWL82sSCcoE4EugBPqmpn3OmZd+6/kao+rapZqpqVlhb16UNjKoT1axMLwgnqdbjrSpRcTOgdXHAbY4yJgkMGtapuAtaG7pIA0B93ESBjDMCLL8Kd//VHpjERE+5x1DcBr4rID7jLH9oNWI0J0R9/YPMT/4JlB7tipzGHL6ygVtXvQvN0J6jqeaq6vaILM6ayuKzLGk6/AgKjR3pdiolTdmaiMUfo3I5/5Mc05dWlb8GPP3pdjolDFtTGHKEL2l9Al/qduLe/UDDq74d+gTHlZEFtzBEaOcJH8qRP+bWG8uSGybBggdclmThjQW3MEapfH76emcYJOSMZ10fYOfour0syccaC2pgjdMMN0KkTbJx0N9t81bh/96fw9ddel2XiiAW1MUcoMRGeegq2bk6m7Xdv8FAP2Dj2gDeoN6bcLKiNiYBu3eBPf4Ll0wZRtLULo31zYe5cr8syccKC2pgIGT8e6tcX6s98h2c6+fh5/G2g6nVZJg5YUBsTIXXqwIMPwublGSQtvJ57ambDjBlel2XigAW1MRF06aXQty/InPt5p8UxfPPAX2xUbY6YBbUxESQCTzwBgcJUUj57hDsaLkKnTvW6LFPJWVAbE2Ht2sEddwiF31/MbE5l2mO32qjaHBELamMqwN13Q8axStKUp7m92a8E33/P65JMJWZBbUwFqFIFnnhc8G9tzY8r/8rrz9wCwaDXZZlKyoLamAoycCBccKEic+/ljgZJFL75mtclmUrKgtqYCvTwv4XU5ETWf/E4T712GwQCXpdkKqGwglpE1ojIjyLynYhkV3RRxsSLJk1g/H0JsOJM7vWdQu4rz3pdkqmEyjOi7quqnVQ1q8KqMSYO3Xij0KZDHrtmPcJ9744Dv9/rkkwlY1MfxlSwxER46dlqsLsRD236C5ue/z+vSzKVTLhBrcCnIjJfRK6ryIKMiUfdusElV+ZSnH0zN7/7LhQWel2SqUTCDepeqtoFOBO4QUR677+BiFwnItkikp2TkxPRIo3xSiT79eMP1aZK9VzeXvwgPz81IUIVmqNBuHchXx/6ugV4D+haxjZPh+5UnpWWlhbZKo3xSCT7dZ068K8HBdZ3Z+jrOZCfH6EqTbw7ZFCLSDURqVHyPXAGsKiiCzMmHt0wvDYt2i1h/vdjmTbhH16XYyqJcEbUDYAvROR74Btgiqp+UrFlGROfROCdN5tDUTWueq0N5OV5XZKpBA4Z1Kq6SlU7hpYOqnpfNAozJl5lnVCd08+ZxaYVl3H/LU94XY6pBOzwPGM88NZLp5JYexX3vns2+Vt2eF2OiXEW1MZ4oHaNFP5y01cU7mjHVZd87HU5JsZZUBvjkQmjL6FW2/d5e855vP7UZrtktTkgC2pjPOITH0+M3oWmLWPo9Q04o1cuy5d7XZWJRRbUxnho6B8v54lHJ8HAm5n1rXJchyAjRtgh1mZfFtTGeOz6C8fy4l/TCd7YjlrNXmfsWOjQAaZM8boyEyssqI2JAVf0/1/evHg82y+7nLaD+5K4awtnnQV/+AP88ovX1RmvWVAbEyMu7Ho1ky58h9UnziXlD035e9c3+PRTJTMTJkyAoiKvKzResaA2JoacfdwQPrr8Y1Y2Vt496RJm9/gjA0/zc9dd0LEjzJzpdYXGCxbUxsSY01udwcdXTWftMSlc0uFt/p3TkSkvbqWoCPr3h6FDYeNGr6s00WRBbUwM6pPeh8+umc1vadXp3XMZbf55PIveWMSIEfDuu9C2LTzyCBQXe12piQYLamNiVPem3Zk5bC559WvRe/AW1lzUndG9Z7BoEfTsCbfeCllZ8MknEAx6Xa2pSBbUxsSwzo06M2fYFwTr16PPxQV8f8UAWn/5Ih9/DO+8A7/9BmeeCZmZ8OijkJvrdcWmIlhQGxPjOhzTgbnDviClfkP6Xi18e/dVyJjRnD9EWbECXnkF6taFW25xdz2/4QZYutTrqk0kWVAbUwm0qdeGz4d9Qe20pvQfnsQXz42Ca64hRYq49FL46iv49ls4/3yYOBHat4fTToMPPoBAwOvqzZGyoDamkkivnc7nV39B47RjGXBNEjPmvACDB8POnYCbr37hBVi3DsaPh59+gvPOg5Yt4V//ctMkpnKyoDamEmlSswlzrppDy2PaMfjKRKaunQnHHecmrEOX30tLg7vugtWr3REiGRlwxx3QtCkMGwYLF3r8IUy5hR3UIpIgIgtF5KOKLMgYc3ANqjdg1pWzOK5RR84b6uP5zoJeeCEMHEjpy+8lJsKQITBrFvzwA1x5JbzxBnTpAr16ue/9fg8/iAlbeUbUtwC2i8KYGFCvaj1mXDGDns1P5poT13LmuHb8svhLN7q+917Ys2ef7Y8/Hp56yk2LPPQQbNoEl1ziRtk33QT/+Y8d4hfLwgpqEWkKDAYmVmw5xphw1UqtxYwrZvDowEf5QtbS4fogj1x7PIH7xrnL73344X+9pk4d+Mtf4Oef4aOP4JRT4Jln3Ag7IwNuvx0WLMBuYhBjwh1RPwzcDhzw/1wRuU5EskUkOycnJyLFGeO1WO/XCb4Ebup2E0tuWELv9D7cmjafkx/IZHFDH5xzjltWr/6v1/l8bj/kO+/Ali3w8stuMP7vf8OJJ0K7djBypB3mFysOGdQichawRVXnH2w7VX1aVbNUNSstLS1iBRrjpcrSr5vXas6UoVN45Q+vsKJ4C50HrWXUuNMonDPDHas3bhwUFpb52po14bLL3PWvN22Cp592UyJjx7qXdurkrt5XRt6bKAlnRH0ycI6IrAHeAPqJyCsVWpUxptxEhEtPuJSlNyzlog4XMbp4Ol3GNOWrC3u4eevjj4dPPz3oe9SrB9deCzNmwPr17noiVau6o0iOPRZ69HDrNmyI0ocyQBhBrap3qWpTVU0HLgZmquplFV6ZMeawpFVL45UhrzBl6BR2BfM5ueVsbn7qXHYlBmHAALjwQrdX8RAaNYKbb4Yvv3Sj6QkToKDAXWOkaVPo2xfuvx/mz7eTaiqaHUdtTJwa1HoQi/9nMTd2vZHHNk3muGuL+HjMFW4vYrt2LmXDPD4vPd0di71wISxZAiNGuLnt2293J9qkpblDAR9/3M1r287IyBKtgJ9oVlaWZmdnR/x9jQEQkfmqmhXtditzv/5y7ZcMnzycpVuXcmnGuTz8Xj713//UBfbYsS5lfeUft23c6G5mMHOmmy4puW1Y48bQr59b+veH5s0j/IHi0MH6tY2ojTkK9GzWk4V/WsiI3iN465epZPZcwGsv3IYKbiokKwumTi33ULhRI7j0Unj2WTc9smKF2xl5yikwbRpccw20aAGtWsGf/gRvvgkxePBMzLMRtal0bER9ZBZtWcTwycOZt34eA1sO4MGCPrQf/4xL2p493REiffsecTuqsGjR3tH2nDl7L8N6wgkuzLt1c0urVoc1oI8rB+vXFtSm0rGgPnKBYIDHvnmMEbNHsLtoN8M6Xs3o9W1oNP5Rd7hH//4usLt3j1ibxcVux2NJcH/9NeTluedq14auXV1ol3yN4aMhK4QFtYkrFtSRs3XPVsbNHccT3z5BUkISfz3pVv66qAY1/vGQm6MYPNgFdqdOEW87EHA7JufNg2++cV8XLdp7KntGxr7h3aULVKkS8TJihgW1iSsW1JG3cttK7p55N28tfosG1RowqsddDJuzi6T7H4QdO9w89ujR7lYyFSgvz426S4f32rXuucREN2XStatbOnVyJ+SkpFRoSVFjQW3iigV1xZm3bh5/++xvfP7r57St15YJPe7l3A+WIf9+2F3o6bLL3Lnlxx4btZo2btwb2t98426QUDLXnZjoDlzp1Ak6dty7HHNM1MqLGAtqE1csqCuWqvLhzx9yx/Q7WLZ1Gb2a9+L+rLvp/tJMeOwxN9k8bBj8/e/uzJcoCwbdRaW+/37fZf36vds0arRvcHfqBK1bu2CPVRbUJq5YUEdHcbCYZxc8y8jZI9mct5kL2l/AP467lVaPveYuuefzuWul/vnPbi5CxNN6t2797/BesmTvOT2pqe7CUx07ugtPZWW5s+pTUz0t+3cW1CauWFBH167CXTz41YPc/+X9FAWKuD7reu7NuJK0Rya6y+7l5UHnznD99TB0KFSr5nXJvysqcmdKlg7v777be1uyxEQX1llZbjnxRPc4OTn6tVpQm7hiQe2Njbs2Mmr2KCYunEj15OrcefKd3NL+aqq+9R48+ST8+KO7FN/ll7vQ7tDB65LLpAq//grZ2W7HZXa2W7Zvd88nJ7udliXBnZXlPkpSUsXWZUFt4ooFtbeW5izlzhl3MvmnyTSs3pB7TrmHazsPJ+XbBS6w33rLDWVPOcVNi5x/fswfmqHqzvcpHdzz5/9+32BSUtyUSVaW++MhM9MtdetGrgYLahNXLKhjwxe/fsE9M+9h7i9zaV6rOSP7jOSKjleQuG2Hux36U0/BypXuzJVrrnHnkGdkeF122IJBWLVqb3BnZ7u73+zatXebBg32hnZmpjtcMDPT7cws75S9BbWJKxbUsUNV+WzVZ9wz8x6yN2TTpl4bRp86mos6XIRPgenTXWBPnuySb+BAN8oePBgSErwuv9yCQXfhqSVL3Nx3ybJkyd7RN7gZoNLBXbKkpx/4Y1tQm7hiQR17VJUPfvqAe2fdy6ItizihwQmM7TuWs9ucjYi4619PnOiOFtmwAZo1g6uugosuchPAHh8xcqRU3d1xSgd3yfebNu3dLjXV3RW+rDPzLahNXLGgjl2BYIA3F7/JyNkjWbFtBV2bdOW+fvfRP6O/C2y/310P+6mn3Gg7GHRnrFx0kTv7MQ5Ce3/bt+87+r7jjrKvY2JBbeKKBXXs8wf8vPj9i4yZM4a1uWvp06IP9/W7j5Obn7x3o82bYdIkePttd2m9YNDND1x44d6R9lHkiK5HLSKpIvKNiHwvIotFZHTkSzTGxJOkhCSGdxnO8puW88jAR9wZjs/3YtCrg1iwcYHbqEEDdxjfzJluOuSJJ9y6cePcmSnt27vT1Rcv9vbDxIBwrgBbCPRT1Y5AJ2CgiETu2ofGmLiVkpjCzd1uZuXNK5nQfwJfr/uaE58+kQveuoBFWxbt3bAktGfNcueCP/64Wzd27N7QHjXqqA3tcG5uq6q6O/QwKbTYHdGMMWGrllyNO3rdwepbVjOi9wimrZzG8U8ez5mvnsn0VdPZZwq2YUP4n/9xob1hw97QHjPGhXaHDi60f/zxqLk5Y1j3VBCRBBH5DtgCfKaq8yq2LGNMPKqVWovRfUez5pY1jOs7joUbF3L6y6fT+f915qXvX6IoULTvC8oK7bQ0F9onnABt28Kdd7pL6sVxaJdrZ6KI1AbeA25S1UX7PXcdcB1A8+bNT/yl5C6XxkRYNHcmWr+uWAXFBbz242s8+NWDLMlZQuMajbm5681cd+J11KlS58Av3LwZ3n8f3n3XzXEHAu4OukOGuDMhe/SodMdpR/SoDxEZAexR1QcOtI3tHTcVyY76iD+qyrSV03jgyweYsXoG1ZKqMazzMG7tfisZdQ5xNuO2be6EmkmT4NNPobDQjcTPO8+Fdp8+FX+hjgg40qM+0kIjaUSkCnA6sCyyJRpjjmYiwsBWA5l+xXQW/mkhQzKH8ET2E7T6v1Zc+PaFfL3u6wO/uG5dd/LM5Mnu9mGvvw69esFLL8Hpp7vQvvpqd/x2YWHUPlMkhTNH3QiYJSI/AN/i5qg/qtiyjDFHq04NO/HSH15izS1r+FvPvzF91XR6PNuDk587mfeWvkcgGDjwi2vUgIsvdsdm5+TAe+/BoEHu69lnu/ntoUPhtdfc85WEnfBiKh2b+ji67C7azXMLn+Phrx9m9Y7VtKzTklu738rlJ1xOrdRa4b1JUZG79fmkSW5ue+tWdwZkly7u+iMDBrjzuj2cIrEzE01csaA+OhUHi3l/2fs88OUDzFs/j6pJVbmow0Vc2+VaejTt4U5RD0cg4K5hOm2aW77+2q2rWRP69dsb3OnpFfp59mdBbeKKBfXRTVWZv3E+z8x/htcWvcbuot20T2vP8M7DuaLjFdSrWq98b7hjhxttlwT3r7+69W3a7A3tPn0q/M41FtQmrlhQmxK7i3bz5qI3eWbBM8xbP4/khGSGZA7h2i7Xcmr6qfgkrFNF9lKFn36CTz5xoT17NhQUuNu+nHKKC+0BA9yJN75yvvchWFCbuGJBbcryw+YfmLhgIi//8DI7CnbQsk5LhncZzlWdrqJh9YaH96YFBfD553uDu+QU9rQ06N9/7xKBGyJYUJu4YkFtDibfn8+7S9/lmQXPMPeXuSRIAme3PZtru1zLgJYDSPAdwYkw69a5y7POmOGWjRvd+owMOO00F9r9+pV9HdNDsKA2ccWC2oTrp60/MXHBRF78/kVy9uTQrGYzrul8DX/s8Efa1W8X/g7Isqi6C0yXhPasWZCb657r2HHvaLt3b6he/ZBvZ0Ft4ooFtSmvokARHyz7gIkLJ/LZys9QlPTa6QxqNYhBrQfRN6MvVZOqHlkjxcXuaJKS4P7Pf9wJNomJ7tC/kuDu1s3Nee/HgtrEFQtqcyTW7lzL1OVTmbpiKtNXTWePfw8pCSn0zejLoFaDOLP1mbSq2+rIG8rPd2FdEtzZ2W4UPmeOG2Xvx4LaxBULahMphcWFzP1l7u/B/fNvPwPQum5rBrV2o+3eLXqTmph65I1t3+6OIhk82EbUJv5ZUJuKsnLbSj5e8TFTl09l1ppZFBQXUDWpKv0z+v8e3M1rNa+Qtg/WrxMrpEVjjKmEWtZtyY1db+TGrjeyx7+H2WtmM3X5VKYsn8KHP38IQPu09gxoOYABLQfQu0VvqiRVqfC6bERtKh0bUZtoU1V++u0npi6fyicrPmHuL3MpDBSSkpBC7xa9XXC3GkCHtA6HfSSJTX2YuGJBbbyW789n7i9zmbZyGtNWTmNJzhIAmtRowhktz+CMlmdw+rGnl+t0dpv6MMaYCKqSVIUBrdwoGtyRJJ+u/JRpK6fx/rL3ef675xGErMZZDGg5gDNankH3pt1JSji8q/PZiNpUOjaiNrEsEAyQvSH799H2vHXzCGiAmik16ZfRj/H9xpOZlvlfr7MRtTHGREmCL4FuTbvRrWk3RvQZwY6CHcxcPZNpK1xwH86JNRbUxhhTgWqn1mZI5hCGZA5BVQ9rZ2M490xsJiKzRGSJiCwWkVsOq1pjjDnKHe4RIeGMqIuB21R1gYjUAOaLyGequuSwWjTGGFMuhxxRq+pGVV0Q+n4XsBRoUtGFGWOMccp1iwIRSQc6A/PKeO46EckWkeycSnR3X2MOxvq1iQVhB7WIVAfeBW5V1dz9n1fVp1U1S1Wz0g7jotnGxCLr1yYWhBXUIpKEC+lXVXVSxZZkjDGmtHCO+hDgWWCpqj5U8SUZY4wpLZwR9cnA5UA/EfkutAyq4LqMMcaEHPLwPFX9AjiCG4sZY4w5EuU66sMYY0z0WVAbY0yMs6A2xpgYF72gzsuDZ58Fvz9qTRpjTDyIWlAHZ85m6vB3Cb71TrSaNMaYuBC1oP6g6EwGM5XJI+dDBdyswBhj4lXUgvrsc30cW38n41dehH7xn2g1a4wxlV7Uglp8AQb8+We+pSsz754erWaNMabSi1pQT/5pMk/SizrVtzH+i1Ng9epoNW2MMZVa1IL6nLbnkNmwJcn9n2Em/fn6rg+i1bQxxlRqUQvqBF8CI/qMYHP7sVRLyeUf77SGnTuj1bwxxlRaUT3h5cL2F9K+aQuq9HmOyYHBLBr3fjSbN8aYSimqQZ3gS2Bkn5Fs7TSGlIQ8JjxZE4qLo1mCMcZUOlE/hfyC9hdwXHoTqvV8mdfzzmHVU59GuwRjjKlUoh7UPvExss9Itp00Bp8Uc/+4gmiXYIwxlYonF2UakjmEE1qnUe3EN3hu82A2TlngRRnGGFMphHMrrudEZIuILIpYo6FR9c6eY/CTyEO3rY/UWxtjTNwJZ0T9AjAw0g2f1+48TsisTvXjJvHkT33Z9sO6SDdhjDFx4ZBBrapzgW0Rb1h8jOozil2njCGP6jx2/eJIN2GMMXHB0xsHnNfuPDp1TKRa6yk88tVJ7N6028tyjDEmJkUsqEXkOhHJFpHsnJyccF/DqD6jyOs9lm1al6evXxipcoyJiMPp18ZEWsSCWlWfVtUsVc1KS0sL+3XntD2HLl39pDafzYMftqZwTyBSJRlzxA63XxsTSZ7fM7FkVF3QZxwbAg156a8/eF2SMcbElHAOz3sd+ApoKyLrRGRYpIs4q81ZnHjyDpIbfss/n0+zs8qNMaaUcI76uERVG6lqkqo2VdVnI12EiDC672iK+oxnZUFT3v7nqkg3YYwxlZbnUx8lBrUexEm915FUdwnjH0i02yoaY0xIzAS1iDD69DH4+/yDRTuaM+Wl37wuyRhjYkLMBDXAwFYDOenUZSTUXM24u3fbqNoYY4ixoBYRxgwaS6DX/czb0II5n+R7XZIxxngupoIaYEDLAXTtMw9f1U3cd9sWr8sxxogXA5UAAAlTSURBVBjPxVxQiwhjh4wn2PMhpi9tQfY3Qa9LMsYYT8VcUAOc3vIMuvaajqRs575bN3ldjjHGeComg1pEuO+S8Wi3x/jgq4YsXep1RcYY452YDGqA/q0H0K3He5CYz/jb7WI4xpijV8wGtYgwfuhINOtpXptShzVrvK7IGGO8EbNBDdD3+HPolvUqQQkyYVSu1+UYY4wnYjqoRYR//PEW6PgSz72awubNXldkjDHRl+h1AYfSt9flnNS5K99+dzU9ewQ4qWsCmZmQmQnt2kGbNpCa6nWVxhhTcWI+qAHuP3sop/42jG0/XspHn2Xy5luNQd0fA+ILUqfRThq22E6Tlrto0XIPGa0LaNWmmAb1k6mSVIWqSVWplVKLxjUak+BL8PjTGGNM+VSKoO5z7i3cMaM9nzV8kT2JkCep7N7Vhj07Mincnsm239qxbUUmS+a1hUCp4XW1TZC2FOrPh7rLSajxGw0aCC2apNKqWU0ym6fRqn46x9Y5low6GdRJrYOIePdBjTGmDJUiqBFhwv8tZUIwCFu2wLp1sHYtrF2LrltL0bp32bNxLbs3rufnnCosDrZhebAdK4KZrN6UydpfL2FPsDYBYENo+arkvatshWpboNoPJNXcTq26haQdA00bJZHepCptW9ShfXoaGQ3q0KxBTapVqdgfWSAAO3fCjh2wfTsUFUGzZtC4Mfhieo+CMaaiVI6gLuHzQcOGbsnKAkCAlNBSB2gWDNJ/8+a9Yb7uW/TXSWxftZ2cdYVs3hhky1YfWwpqsDbhGFZyDOsKj2Fz4TFs25rJzp+OYWtRXQ54jk1CIb7UXSSm5pFUZQ+pVQuoUs1PtRoBatQKUquWj9q1E6lbN5H6tZM5pm4qSb5kducmsTs3kd25CezamUjuzgR27fSRm5vAzh1C7g4fO3cKubllj+iTkyE9HTIy4Nhj3dfS39epE/GftjEmRoQV1CIyEHgESAAmquqECq3qSPh80KiRW046CXBhXje0tC3Zbvdu2Lix1LLAfd2wgaL1OWxZl8+SnUEWaxV+TjmGrQm12C61yKUmu6jJ7mAt8oM1KdxVk13balHsr0mwqBYU1AI9xI81ZSekboPU7ZC6wy31tkOTHXsfh54TXxDJbU7x9gyW7ziW5T+mo3PTIb/evu+ZupOEumtIqPtraFlLYt21JFXNJylJSE7CfU0WkpOElGQfSYnua8mSmpJAanICKUlJJCckA1BYXEhhoJCC4gIKA4UUFu/7/e/PFZc8l09hcSH+YDEpviSqJKRSJTHFfU1IJTUhhSqJqaGlSmhx36eWrEuqQlbTrpzW4ezI9QtjKrFDBrWIJACPA6cD64BvRWSyqi6p6OIqVPXq0Lq1W/aTDDQNLWcUFMDmzbBnDxQUlLFsgIJVkJ9PMH8Pufm5bM7LY92uYjbmBdiYpxQFCqjq20oqOSTrVtSfT8BfSKCokIC/yH2PEvBBoBgCeyCQD0GBgA8UCKaCNnJLUKCguCbbCzLYmZ/Bzvxj2ZGfQW5+BrlrWpG75DQCwSqH/7ORIPj8kOBHfH7EV4z4ivH5ivH5/G6RYhLETwJ+EqSIBC0iET+1xE8ixSTiXq+hJZDoJzehiO2JfoKJ7nFxyZLkFn+Sn+KkneDbyvnVV3DaKxbUxkB4I+quwApVXQUgIm8A5wKVO6jDlZoKLVqEtakPqB1a2h5i232oQmEh5OX997JnD/j9bikqOsj328C/GYrmEiwqZvPOVFb/VpO8PYK/IEBxYQB/QQB/YdAtRUpxkfta8jb+gLiI1ST8gdBCkluXWBV/UhX8iVUpllT8Can4fan4E1Io9tXAL8luKdleE/EHfPj9QlG+D3+xD39A3LqAj6JiH6oH3nFb9+xfy/MTNCauhRPUTYC1pR6vA7rtv5GIXAdcB9C8efOIFHfUEHH/IaSmQr16h97+EHxAo9BSLsGg+yshP98tADVrur8+KmBPZiCw9/+a/Zdq1WKjD1m/NrEgYjsTVfVp4GmArKwsu4lWZeTzQdWqbomChAS3xPIJS9avTSwIZ5i0HmhW6nHT0DpjjDFREE5Qfwu0FpEMEUkGLgYmV2xZxhhjShxy6kNVi0XkRmAa7vC851R1cYVXZowxBghzjlpVpwJTK7gWY4wxZbCTko0xJsaJauR3ZItIDvBLGU/VB7ZGvMHwWNvx024LVU2roPc+oIP0azg6/429bDseP/MB+3WFBPWBiEi2qmZFrUFr27O2vfzMXjga/429bPto+8w29WGMMTHOgtoYY2JctIP66Si3Z21717aXn9kLR+O/sZdtH1WfOapz1MYYY8rPpj6MMSbGRS2oRWSgiPwkIitE5M4otttMRGaJyBIRWSwit0Sr7VD7CSKyUEQ+inK7tUXkHRFZJiJLRaRHFNv+S+hnvUhEXheRGL7s0pHzom973a9DNVjfjlLfjkpQl7r5wJlAe+ASEWkfjbaBYuA2VW0PdAduiGLbALfAge/sVYEeAT5R1XZAx2jVICJNgJuBLFU9DnfZgYuj0bYXPOzbXvdrsL4dtb4drRH17zcfUNUioOTmAxVOVTeq6oLQ97tw/6hNotG2iDQFBgMTo9FeqXZrAb2BZwFUtUhVd0SxhESgiogkAlVx9xOOV570bS/7NVjfjnbfjlZQl3Xzgah1qhIikg50BuZFqcmHgduBYJTaK5EB5ADPh/40nSgi1aLRsKquBx4AfgU2AjtV9dNotO0Rz/u2B/0arG9HtW8fNTsTRaQ68C5wq6rmRqG9s4Atqjq/otsqQyLQBXhSVTsDeUC05k7r4EaUGUBjoJqIXBaNto9G0e7XoTatb0e5b0crqD29+YCIJOE686uqOilKzZ4MnCMia3B/DvcTkVei1PY6YJ2qloyw3sF17mg4DVitqjmq6gcmAT2j1LYXPOvbHvVrsL4d9b4draD27OYDIiK4+aylqvpQNNoEUNW7VLWpqqbjPu9MVY3K/76quglYKyIl99jtT/RuRvwr0F1EqoZ+9v3xZodTtHjSt73q12B924u+HbF7Jh6MxzcfOBm4HPhRRL4Lrbs7dI3teHYT8GooPFYBV0ejUVWdJyLvAAtwRyYsJI7PUvSwbx+t/RqOwr5tZyYaY0yMO2p2JhpjTGVlQW2MMTHOgtoYY2KcBbUxxsQ4C2pjjIlxFtTGGBPjLKiNMSbGWVAbY0yM+//lPqNduVGVLgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ((ax1, ax2)) = plt.subplots(nrows=1, ncols=2, sharex=True, sharey=True,figsize=(6,3))\n",
    "ax1.set_title('MNIST Loss')\n",
    "ax1.plot(d_opt['loss'][0],'r' ,label=\"sgd_val_acc\")\n",
    "ax1.plot(d_opt['loss'][1],'g' ,label=\"adam_val_acc\")\n",
    "ax1.plot(d_opt['loss'][2],'b' ,label=\"adagrad_val_acc\")\n",
    "ax2.set_title('CIFAR10 Loss')\n",
    "ax2.plot(d_opt['loss'][3],'r' ,label=\"sgd_val_acc\")\n",
    "ax2.plot(d_opt['loss'][4],'g' ,label=\"adam_val_acc\")\n",
    "ax2.plot(d_opt['loss'][5],'b' ,label=\"adagrad_val_acc\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAJsAAACMCAYAAABrstuvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAKC0lEQVR4nO3de4ycVR3G8e8DFDCiWNjGVKAXFANogdLGlGCABCwtarkptEZsCVAlIInGP2w0UC5/cAkhIopUbAKSlCKJBikUq5WAQIGWi6VAoRS5NBgKLUQoAm0f/zhn7Muy251u3z2d2fl9ksm+tzPnzMyz78w775zzyjYhlLDTjm5A6BwRtlBMhC0UE2ELxUTYQjERtlDMLn1tIGku8A3gddtf7mG9gF8AJwAbgBm2H8vrpgM/z5teZvumvurr6uryqFGjmn4AobUsW7bsDdvDelxpe6s34CjgcOCpXtafANwNCJgAPJyX7wWszn+H5umhfdU3btw4h/YFLHUvr22fb6O27wPWbWWTE4Gbc11LgM9IGg4cDyyyvc72emARMKmv+sLgVcdntn2AVyrzr+ZlvS0PHaolDhAkzZS0VNLStWvX7ujmhAHS5wFCE9YA+1Xm983L1gDHdFt+b093YHsOMAdg/PjxbXeyVher6W19UdmH10ptq2PPdgfwPSUTgLdtvwbcA0yUNFTSUGBiXhY6VDNffcwj7aG6JL0KXAQMAbD9G+Au0hHpKtJXH2fmdeskXQo8mu/qEttbO9AIg1yfYbM9rY/1Bs7rZd1cYG7/mvZRrfR2EPqnJQ4QQmeIsIViImyhmAhbKCbCFoqJsIViImyhmAhbKCbCFoqJsIViImyhmAhbKCbCFoqJsIViImyhmAhbKCbCFoppKmySJklaKWmVpJ/2sP4aSU/k23OS3qqs21RZd0edjQ/tpZk+CDsDvwK+Rur7+aikO2w/3djG9o8q2/8QGFu5i/dsH1Zfk0O7ambP9hVgle3Vtj8AbiX1gu/NNGBeHY0Lg0szYWu6Z7ukkcBoYHFl8e65A/ISSSf1u6Wh7dXRSblqKnC77U2VZSNtr5G0P7BY0nLbL1QLSZoJzAQYMWJEzU0KraKZPVtvPd57MpVub6G21+S/q0k94sd2L2R7ju3xtscPG9bzaEuh/TUTtkeBAySNlrQrKVAfO6qUdCBpaKyHKsuGStotT3cBRwJPdy8bOkMznZQ3SjqfNHTCzsBc2yskXUIai6sRvKnArbnTcsNBwA2SNpOCfXn1KDZ0lqY+s9m+izTMQnXZhd3mZ/dQ7kFgzHa0LwwicQYhFBNhC8VE2EIxEbZQTIQtFBNhC8XUfboq9EHNj2kIwGC6HGzs2UIxg3LP1sl7j1YWe7ZQTIQtFBNhC8VE2EIxg/IAIfTPQB9YxZ4tFBNhC8VE2EIxEbZQTF3DL8yQtLYyzMLZlXXTJT2fb9PrbHxoL7UMv5DNt31+t7J7kS4ZOR4wsCyXXV9L60NbGYjhF6qOBxbZXpcDtgiY1L+mhnZX5/ALp0r6p6TbJTU6NTdVNq4R3xnqOkD4MzDK9iGkvddN21I4esR3hlqGX7D9pu338+yNwLhmy4bOUcvwC5KGV2anAM/k6XuAiXkYhqHAxLwsdKC6hl+4QNIUYCOwDpiRy66TdCkpsACX2F43AI8jtIFahl+wPQuY1UvZucDc7WhjGCTiDEIoJsIWiomwhWIibKGYCFsoJsIWiomwhWIibKGYCFsoJsIWiomwhWIibKGYCFsoJsIWiomwhWIibKGYCFsopq4e8T+W9HTuyve3fEXlxrpNlZ7yH7t0ZOgcdfWIfxwYb3uDpHOBK4HT87r3bB9Wc7tDG6qlR7ztv9vekGeXkLrshfARdfaIbzgLuLsyv3vu7b5E0kn9aGMYJGod5lTSd0mDyBxdWTzS9hpJ+wOLJS23/UK3cjOBmQAjRoyos0mhhdTSIx5A0nHAz4Apld7x2F6T/64G7gXGdi8bwy90hrp6xI8FbiAF7fXK8qGSdsvTXcCRQFwjvkPV1SP+KmAP4A9KQ06/bHsKcBBwg6TNpGBf3sO4bqFD1NUj/rheyj0IjNmeBobBI84ghGIibKGYCFsoJsIWiomwhWIibKGYCFsoJsIWiomwhWIibKGYCFsoJsIWiomwhWIibKGYCFsoJsIWiomwhWLq6hG/m6T5ef3DkkZV1s3Ky1dKOr6+pod202fYKj3iJwMHA9MkHdxts7OA9ba/AFwDXJHLHkzqIPMl0uW6f53vL3Sguq4RfyJbrp58O3CsUs+XE4Fbbb9v+0VgVb6/0IHq6hH//21sbwTeBvZusmzoELX2iO+vao944B1JK5ss2gW80cM9bmP927R5szq1bSN7XEpzYWumR3xjm1cl7QLsCbzZZFlszwHmNNGWj5C01Pb4bS1XQrTt42rpEZ/np+fpbwGLbTsvn5qPVkcDBwCP1NP00G7q6hH/O+D3klaRrhE/NZddIek20pALG4HzbG8aoMcSWpzSDqg9SZqZ34JbTrSth3rbOWyhvcTpqlDMoA2bpH/lYbpK1TdD0nWl6muF+iUdI+nOZrcftGEL/aNkQHLR0mGT9ElJCyQ9KekpSadLOkHSs5KWSbq28Z8laW9Jf5G0QtKNbOs3lH235U+5zhX5S2gknSnpOUmPkAY6bGz7zfyDhMcl/VXSZ/Py2ZJuknS/pJcknSLpSknLJS2UNGSA6x8maVHjOcpt6JI0Kv9Q4mbgKWA/SdcrjYW8QtLFlfuelJ//x4BTtulJtN2yN+BU4LeV+T1Jp79G5/l5wJ15+lrgwjz9dcBAV41t2Sv//UR+QfYBXgaGAbsCDwDX5W2GsuXg62zg6jw9G/gHMAQ4FNgATM7r/gicNMD1XwfMytOTGs8RMArYDEzoob6dScPTHgLsnp//A0j/zLc1nv9mbi1xumorlgNXS7oCuBP4D7A6n9SHFLbGaa6jyP9pthdIWl9zWy6QdHKe3g84A7jX9loASfOBL+b1+wLzJQ0nBeHFyv3cbftDSctJL+TCvHw56UUfyPq/CpwMYHtht+foJdtLKvOn5T3oLsBw0i9+dgJetP18rvMWtjz/fWrpt1HbzwGHk16Iy4ApO6Idko4BjgOOsH0o6SIjz26lyC9Je5kxwPdJe4SG9wFsbwY+9JbvnjbTy5fsNdffm3cr9Y0GfgIca/sQYEGT97FVLR02SZ8DNti+hTRu75HA/pUfZ55e2fw+4Du53GTSW0ld9iT9Xm+DpAOBCaS3s6PzZ8UhwLe7bd84Bzyd7VdX/Q8ApwFImkjvz9GnSeF7O3/em5yXPwuMkvT5PD9tWx5Eq7+NjgGuUhoA+kPgXNIufaGkd0nnbRsuBuZJWgE8SPo8U5eFwA8kPQOsJF3F5jXSZ7CHgLeAJyrbzyYNZr0eWAyMbpH6G8/RGbncv0kfTfaoVmb7SUmNvecrpJBi+7/5rXWBpA3A/cCnmn0QbXcGQdIett+RJNIviJ+3fc2Oblc7ULpMwKZ8vvsI4HoXvK5Yq+/ZenKOpOmkD76Pk66/EJozArgtf4/2AXBOycrbbs8W2ldLHyCEwSXCFoqJsIViImyhmAhbKCbCFor5H55uw93slj0EAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 144x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ((ax1)) = plt.subplots(nrows=1, ncols=1, sharex=True, sharey=True,figsize=(2,2))\n",
    "width = 0.3\n",
    "ind= np.arange(len(d_opt[['opt']])/2)\n",
    "ax1.bar(ind, list(d_opt['test_accuracy'].values)[0:3],width,color='g')\n",
    "ax1.bar(ind + width, list(d_opt['test_accuracy'].values)[3:6],width,color='b')\n",
    "ax1.set_xticks(ind)\n",
    "ax1.set_xticklabels(list(d_opt['opt'].values))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>opt</th>\n",
       "      <th>dataset</th>\n",
       "      <th>lr_rate</th>\n",
       "      <th>test_accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>sgd</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>0.9842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sgd</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>0.9600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>sgd</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.0100</td>\n",
       "      <td>0.9159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>sgd</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>0.1032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>sgd</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>0.1032</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   opt dataset  lr_rate  test_accuracy\n",
       "9  sgd   mnist   0.0001         0.9842\n",
       "0  sgd   mnist   0.0010         0.9600\n",
       "6  sgd   mnist   0.0100         0.9159\n",
       "7  sgd   mnist   0.1000         0.1032\n",
       "8  sgd   mnist   0.5000         0.1032"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUcAAACcCAYAAADsx2kzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAa80lEQVR4nO3de5wU5ZX/8c+XiyIKiuC6GhSId24Dw4AxbAJeUKIbkiyaKEoiP0GD0ZjXrkZJVCBmNzEh0dXEjSaKykYXNKs/YjAqGi9JVC7eQYgiqCPECwJyEeRy9o/n6bGnrZ7pufRU98x5v17zmq6qp6tOVVeffqqqu47MDOecc7W1SzsA55wrRZ4cnXMugSdH55xL4MnROecSeHJ0zrkEnhydcy6BJ0fXqkgaKak6a3iJpJHxsSTNlLRO0oI4brKktyVtktQ9pbBLmqRfSboi7ThaWqrJUdIqSR9J6pEz/llJJql3HL41Dg/LanOoJMsaflTSxKzh70laGXf6akmz4/glcdwmSTslbc0a/l5CjNMk/Xczra9JOrQ55uUKY2b9zOzROPhPwCigp5kNk9QR+DlwopntZWZrWzK2uP+fUMf0Wok+LWb2TTO7qrnnK6l3fE9k3n+rJF3WgOffKumHzR1XRin0HFcCZ2QGJA0AOie0ex8oaENI+gYwHjjBzPYCqoCHoebNslcc/wRwQWbYzP6jaatS3iR1SDuGIusFrDKzzXF4f6ATsKQxM5PUvrkCS0uJvOb7xPfjqcAVkkalHRAAZpbaH7AKuBxYmDVuBvB9wIDecdythE/4vwMj4rhDQ/g1z3sUmBgf/wK4toDl1zwnz/TRwEfAdmAT8HwcvzdwM7AGeIuQtNtnxfUYsAF4D5gdxz8e12lznNfXEpZ3CPAIsDY+97eEHScz/SDgf4F3Y5tfZE2bBLwMbASWApVxvAGHZrW7FfhhfDwSqAYujdt2FtANuC8uY1183DPr+fsCM4HVcfq9cfxLwBez2nWM6zA4z7adBLxK+NCbCxyYNc2AbwKvAOuBXwLKM5894jqti+t9CVCds4+dAJwDbAV2xu1/Z3wtLA4/EtsfCTwU41oOfDVn2/0XMC8+9wTgQOB3cXutBL6d1X4aMAe4Pb4uS4CqOG0WsAv4MC7/uwnrNjJ7XXKm1bXcYcCTcdutIbwfdsvZvt+K23dl1n7wb8A78TkT6tln8rXtDvwe+ABYSHhv/DnPOvSOsXTIGrcAuCRr+C7CvrmB8B7qF8efS3hffhS33+8L3C6LYmxvAz+vMz+0ZDJM2DiZHXc5cBTQPm74XnwyOf4Q+HZmQ1N3cjyLsHNfQug1tm9Mcszawf87Z9w9wI3AnsA/xBf0vDjtTkJyb0folfxTzk55aB3LOpRw2Lc7sF/cGa6N09oDzwPXxOXWzBs4jZCkhwKK8+mVtEw+uaPvAK6Oy9wj7txjCb33LnHnvDfr+X8AZhOSaEc+/rD6LvGDIA5/CXgxz3oeR0iclXG51wOP52yn+4B9gIPjjj46z7x+TDgC2Jfw4fESCckxPj6brDcqOW/OuF3fBCYAHYDBMc6+WdtuAzA8vr6dgcXAlcBuwKeB14CTsvadrcDJ8fX7EfBUUmx51m0kCckxLruu5Q4BPhPXoTfhQ/M7Odv3objN9sjaD34QX9OTgS1Atzr2mXxt/yf+dQb6xu1ZUHKMMW8BvpLV5v8R9sPdgWuB55L25QK3y5PA+Ph4L+Azdb73WyoR5tk4qwjJ8fK444yOL1oHkpPj7sAbwBeoIznG4TOB+YRP+LXApQnLr/WcPDFOIys5Eg7FtgF7ZI07A/hTfHw7cBNZva2cnTJvckxo/2Xg2fj4GEKS6JDQ7gHgojzzqC85fgR0qiOGQcC6+PgAQm+nW0K7Awm9o65x+G4SekNx2s3AT7KG9yL0AnpnxZz9oTIHuCzPvF4jK3ESehSNTY5fA57Imf+NwNSsbXd71rSjgTdy2k8BZmbtO/OzpvUFPkyKLc+6jSQ5Oda53IT23wHuydknjstZzofU7sG9Q0weCftMYlvCB8B24IisaYX0HNfHeRrhyDHfUcI+sc3euXEV+Ho8DkwHehTy/iuF8w0QDjEeB/oQkksiM9sm6SrgKuD0umZoZr8FfhtPun85Pn7OzB5oYqy9CJ+YayRlxrUjfEJC6EFdBSyQtA74mZndUsiMJe0P/CfwOcKnZTvC4SKEXtHrZrYj4akHASsavioAvGtmW7Ni6EzonY4m9A4BusTzawcB75vZutyZmNlqSX8Bxkq6h/ABdlGeZR4IPJP13E2S1gKfIiQMCIdSGVsICTTfvN7MGn49T7tC9AKOlrQ+a1wHwv6Z8WZO+wNz2rcn9GQzctejk6QOeV7HhsSZd7mSDiechqoi9OA6EHpU2d7MGV6bE1Nd2zxf2/3isrLnnbucJD0ISe8iYBzh/fVR3Of+nXBktB/hgznTfkPCfOp7Pc4h9HiXSVoJTDez+/IFVQoXZDCz1wnnB04mnFOry0zCJ8i/FDjv7WZ2F/AC0L8x4eUMv0noOfYws33iX1cz6xeX93czm2RmBwLnATc04Ar1f8TlDTCzroTTA5kM/CZwcJ4T6G8Szlcm2ULtC1z/WM/6/RtwBHB0jOHzcbzicvaVtE+eZd0WYz4NeNLM3srTbjVhRw4zlvYkHM7na1+XNYSknXFwI+aR8SbwWNbruo+FC3WTs9pYTvuVOe27mNnJBS4vd9s3JM66lvtfwDLgsPgafo+P96OmLrsu7xIOuXtmjTsoT9vawZjtNLOfE05DnB9HjyOcnjmBcJ6/dxyfWZek92be7WJmr5jZGYRTYVcDd8d9L1FJJMfoHEJXf3NdjeIn1lTCRYREks6WdIqkLpLaSfoC0A94uhFxvQ30ltQuLn8N8CDwM0ld4/wPkTQiLvs0SZmdYx3hBdyVNa9P17GsLoSTyxskfYpwzjRjASER/FjSnpI6SRoep/0GuFjSkPhdvkMlZZLPc8A4Se0ljQZG1LO+XQiHOOsl7UvY1mSt+/2EhN9NUkdJn8967r2E84gXUccRAOG87ARJgyTtTvhQeNrMVtUTW5I5wJQYT0/gwkbMI+M+4HBJ4+O6dZQ0VNJRedovADZKulTSHnEb95c0tMDl1bc/ABBf65q/ApbbhXDRYZOkI4HJ+ebdnMxsJ6FzM01S57jsrzdwNj8GvhvXswuhI7KW8AGf+22S3O1X53aRdJak/cxsF+FQHj5+b35CySRHM1thZosKbH4nIVHk8wHh0/INwkb4CTDZzP7ciNDuiv/XSsocCn6dcMJ3KSEB3k04HwfhosjTkjYRrsJeZGavxWnTgNskrZf01YRlTScklw2ECx81vei4432RcK71DcKFq6/FaXcRDj/uIJz3u5dwsh1Covpi3A5nxml1uZZwkv494CngjznTxxPOKy0jnGv6TlaMHxKuFPahjiMAM5sPXBHbriH0eus8TVKH6YRD6ZWED61ZdTfPz8w2AifGWFYTDokzF6uS2u8E/plwXnYlYZv9htDLKcSPgMvj/nBxnjafInxYZf/1qWe5FxN6XRuBXxMuoLWUC2IcmW8/3ElIcIX6A+E9NYnwAfs64YhiKWF/zHYz0Dduv3sLeD1GA0vie/M/gdPjPptI8USlc81C0pXA4WZ2VtqxuPRJuhr4RzP7RtqxNFTJ9Bxd+YuH4ecQrta7NkjSkZIGxtM7wwj7wz1px9UYnhxds5A0iXBC/H4zezzteFxquhBOqWwmHM7/DPj/qUbUSH5Y7ZxzCbzn6JxzCTw5OudcglL5hUxqevToYb179047DOfanMWLF79nZvulHUc+bT459u7dm0WLCv16pXOuuUhqyk89i84Pq51zLoEnR+ecS+DJ0TnnErT5c46u/G3fvp3q6mq2bt1af2PX4jp16kTPnj3p2LFj2qE0SJtPjosXg3Jv5tQM/Lv1Lae6upouXbrQu3dvVIwX0zWambF27Vqqq6vp06dP2uE0SNkcVksaLWm5pFeTKpTF25S9K+m5+DcxaT6u9dm6dSvdu3f3xFiCJNG9e/ey7NWXRc8x3hH4l4T6KtXAQklzzWxpTtPZZnZBiwfoUueJsXSV62tTLj3HYcCrZvaamX1EKODzpZRjcg6A9evXc8MNNzTquddeey1btmxp5ohccyiX5PgpateiqI7jco2V9IKkuyUVdHt21/pIzftXn9aSHHfsaEpZm9anXJJjIX5PqF43kFDB8LZ8DSWdK2mRpEWh7IVzjXfZZZexYsUKBg0axCWXXMJPf/pThg4dysCBA5k6NVSZ2Lx5M6eccgoVFRX079+f2bNnc91117F69WqOPfZYjj322Lzznzx5MlVVVfTr169mfgALFy7ks5/9LBUVFQwbNoyNGzeyc+dOLr74Yvr378/AgQO5/vrrgfBLsPfeew+ARYsWMXLkSACmTZvG+PHjGT58OOPHj2fVqlV87nOfo7KyksrKSv7617/WLO/qq69mwIABVFRU1KxzZWVlzfRXXnml1nDZK6REYdp/hLKkD2QNTwGm1NG+PbChsHkPsXBtuXn/XMtZunRpreGWfi1Xrlxp/fr1MzOzBx54wCZNmmS7du2ynTt32imnnGKPPfaY3X333TZx4sSa56xfv97MzHr16mXvvvtunfNfu3atmZnt2LHDRowYYc8//7xt27bN+vTpYwsWLDAzsw0bNtj27dvthhtusLFjx9r27dtrPTd7OQsXLrQRI0aYmdnUqVOtsrLStmzZYmZmmzdvtg8//NDMzP72t7/ZkCFDzMxs3rx5dswxx9jmzZtrzXfkyJH27LPPmpnZlClT7Lrrrktch9zXyMwMWGQlkF/y/ZVLz3EhcJikPpJ2I9T4mJvdQNIBWYNjCIXMnWtRDz74IA8++CCDBw+msrKSZcuW8corrzBgwAAeeughLr30Up544gn23rvQMjMwZ84cKisrGTx4MEuWLGHp0qUsX76cAw44gKFDQ02trl270qFDB+bPn895551Hhw7hWuu+++5b16wBGDNmDHvssQcQvjM6adIkBgwYwGmnncbSpeGa5/z585kwYQKdO3euNd+JEycyc+ZMdu7cyezZsxk3blzhG6vElcXVajPbIekCQvH69sAtZrZE0g8Inz5zgW9LGkMoDfk+oYC7cy3KzJgyZQrnnXfeJ6Y988wzzJs3j8svv5zjjz+eK6+8st75rVy5khkzZrBw4UK6devG2Wef3aivxXTo0IFdu0Khvdzn77nnx9VJr7nmGvbff3+ef/55du3aRadOneqc79ixY5k+fTrHHXccQ4YMoXv37g2OrVSVS88RM5tnZoeb2SFm9u9x3JUxMWJmU8ysn5lVmNmxZrYs3YhdW9GlSxc2btwIwEknncQtt9zCpk2bAHjrrbd45513WL16NZ07d+ass87ikksu4ZlnnvnEc5N88MEH7Lnnnuy99968/fbb3H///QAcccQRrFmzhoULFwKwceNGduzYwahRo7jxxhtrLq68//77QDjnuHjxYgB+97vf5V3ehg0bOOCAA2jXrh2zZs1i586dAIwaNYqZM2fWXDzKzLdTp06cdNJJTJ48mQkTJjRi65WuskmOxTJkSDHOOKa9Vq4lde/eneHDh9O/f38eeughxo0bxzHHHMOAAQM49dRT2bhxIy+++CLDhg1j0KBBTJ8+ncsvvxyAc889l9GjR+e9IFNRUcHgwYM58sgjGTduHMOHh1Llu+22G7Nnz+bCCy+koqKCUaNGsXXrViZOnMjBBx/MwIEDqaio4I477gBg6tSpXHTRRVRVVdG+ffu863L++edz2223UVFRwbJly2p6laNHj2bMmDFUVVUxaNAgZsyYUfOcM888k3bt2nHiiSc2y/YsFW2+hkxVVZX5/RzL28svv8xRRx2Vdhht1owZM9iwYQNXXXVV3jZJr5GkxWZWVez4Gqsszjk650rTV77yFVasWMEjjzySdijNzpOjcyXi6KOPZtu2bbXGzZo1iwEDBqQUUf3uuacsS1IXxJOjcyXi6aefTjsEl6XNX5Bxzrkknhxdq9DWLyyWsnJ9bTw5urLXqVMn1q5dW7ZvwtbMLNzstr4vk5ciP+foyl7Pnj2prq7m3Xf9JiKlKFMmodx4cnRlr2PHjmV3C35X+vyw2jnnErT5nqMX2HLOJfGeo3POJSib5FhA9cHPS3pG0g5Jp6YRo3Ou9SiL5JhVffALQF/gDEl9c5q9QbiH4x0tG51zrjUql3OONdUHASRlqg/WlGY1s1Vx2q40AnTOtS5l0XOk8OqDzjnXLMolOTYrrz7onKtPuSTHt4DsOtQ947hGMbObzKwq3GhzvyYH55xrfcolOdZbfdA555pTWSRHM9sBZKoPvgzMyVQfjBUHkTRUUjVwGnCjpCXpReycK3dtvoaMVGXQ/DVk2vhmda5epV5Dpix6jsXk1Qedc0nafHJ0zrkknhydcy5BwclRUmdJV0j6dRw+TNI/Fy8055xLT0N6jjOBbcAxcfgt4IfNHpFzzpWAhiTHQ8zsJ8B2ADPbAhThTojOOZe+hiTHjyTtARiApEMIPUnnnGt1GnJXnmnAH4GDJP0WGA5MKEZQzjmXtoKTo5k9KGkx8BnC4fRFZvZe0SJzzrkUNeRq9cNmttbM/mBm95nZe5IeLmZwzjmXlnp7jpI6AZ2BHpK68fFFmK74PRWdc61UIYfV5wHfAQ4EFvNxcvwA+EWR4moxxao+6Fw585/ANuDGE5IuNLPrixxPiyvWjSecK2ctkRxbzY0nzOx6Sf0lfVXS1zN/zR1QAVUGd5c0O05/WlLvOL67pD9J2iSp7Hu0zrl0FXy1WtJUYCSh+t88QiXAPwO3N1cwWVUGRxHqxCyUNNfMlmY1OwdYZ2aHSjoduBr4GrAVuALoH/+cc67RGvIl8FOB44G/m9kEoALYu5njqakyaGYfAZkqg9m+BNwWH98NHC9JZrbZzP5MSJLOOdckDUmOH5rZLmCHpK7AO9Su69IcCqkyWNMm3iF8A9C9IQvxAlvOufo05BcyiyTtA/yacNV6E/BkUaIqMjO7CbgJMhdknHOutoKSoyQBPzKz9cCvJP0R6GpmLzRzPIVUGcy0qZbUgXBov7aZ43DOtXEFHVZb+L7PvKzhVUVIjFBYlcG5wDfi41OBR6ytF8JxzjW7hhxWPyNpqJktLFYwZrZDUqbKYHvglkyVQWCRmc0FbgZmSXoVeJ+QQAGQtIrwy53dJH0ZODHnSrdzzhWkIV8CXwYcCrwObCb8UsbMbGDxwis+/xK4c5/kXwJvWM/xpLomSupmZuuaGE+LGzIEFnludM7laMgty16vp8nDQGXTwnHOudLQnNUH/fYNzrlWozmTo18xds61Gl632jnnEvhhtXPOJSgoOUpqH7/KU5fjmyEe55wrCYX+QmYnsFzSwXW0eb/ZonLOuZQ15HuO3YAlkhYQvgQOgJmNafaonHMuZQ1JjlcULQrnnCsxDfkS+GPFDMQ550pJIaVZN5L8HcbMb6u7NntULcirDzrXOK39Xlj1XpAxsy5m1jXhr0tTEmNjC2nFaVPi+OWSTsoaf4ukdyS91Ni4nHMOUvoSeFYhrS8QCnadIalvTrOaQlrANYRCWsR2pwP9gNHADXF+ALfGcc451yRp/UKm0YW04vj/MbNtZrYSeDXODzN7nHCPR+eca5K0kmNTCmkV8lznnGuSNvnbaq8+6JyrT1rJsSGFtMgppFXIc+tkZjeZWVW4C/F+DQzdOdcWpJUcm1JIay5werya3Qc4DFjQQnE759qIVJJjPIeYKaT1MjAnU0hLUubniDcD3WMhrX8FLovPXQLMAZYCfwS+FX/7jaQ7CbW0j5BULemcllwv51zrUXCBrdbKC2w51zhNTR2lXmCrTV6Qcc65+jTkxhOtklcfdM4l8Z6jc84l8OTonHMJPDk651wCT47OOZfAk6NzziXw5Oiccwk8OTrnXAJPjs45l8CTo3POJWjzv5DxAlvONZ/WdKsG7zk651yCoifHIlUZTJynpAviOJPUo9jr5pxrvYqaHItRZbCeef4FOAF4vZjr5Zxr/YrdcyxGlcG88zSzZ81sVZHXyTnXBhQ7ORajyqBXH3TOFV2bvCDj1Qedc/UpdnIsRpVBrz7onCu6YifHYlQZLGSezjnXJEVNjsWoMphvngCSvi2pmtCbfEHSb4q5fs651surD3r1QeeaTUPSSalXH2zzPx/0AlvOuSRt8mq1c87Vx5Ojc84l8OTonHMJPDk651wCT47OOZfAv8ojbQSWpx1HgXoA76UdRIE81uIpp3jrirWXmZXsT9Ta/Fd5gOWl/F2rbJIWeazNr5xihfKKt5xizeWH1c45l8CTo3POJfDkCDelHUADeKzFUU6xQnnFW06x1tLmL8g451wS7zk651yCNpEcm1IBMQ0FxPuvkpZKekHSw5J6pRFnjKXOWLPajY1VIVO7cllIrJK+GrftEkl3tHSMWXHUtw8cLOlPkp6N+8HJacQZY7lF0juSXsozXZKui+vygqTKlo6xUcysVf8B7YEVwKeB3YDngb45bc4HfhUfnw7MLvF4jwU6x8eT04q3kFhjuy7A48BTQFWpxkq4ofKzQLc4/A8lHOtNwOT4uC+wKo1Y4/I/D1QCL+WZfjJwPyDgM8DTacXakL+20HNsSgXENNQbr5n9ycy2xMGnCDf3TUMh2xbgKkLJ3a0tGVyOQmKdBPzSzNYBmNk7LRxjRiGxGtA1Pt4bWN2C8dUOxOxx4P06mnwJuN2Cp4B9JB3QMtE1XltIjk2pgJiGhlZXPIfwqZyGemONh1AHmdkfWjKwBIVs18OBwyX9RdJTkka3WHS1FRLrNOCseOf7ecCFLRNao5RlxVD/hUwZk3QWUAWMSDuWJJLaAT8Hzk45lEJ1IBxajyT0xh+XNMDM1qcaVbIzgFvN7GeSjgFmSepvZrvSDqy1aAs9x6ZUQExDQdUVJZ0AfB8YY2bbWii2XPXF2gXoDzwqaRXhfNPclC7KFLJdq4G5ZrbdzFYCfyMky5ZWSKznEGosYWZPAp0Iv2MuRU2uGJqKtE96FvuP0Bt4DejDxye3++W0+Ra1L8jMKfF4BxNO2B9W6ts2p/2jpHdBppDtOhq4LT7uQTgU7F6isd4PnB0fH0U456gU94Xe5L8gcwq1L8gsSCvOBq1T2gG00At3MqEXsAL4fhz3A0KvC8Kn7l3Aq4Tyr58u8XjnA28Dz8W/uaUaa07b1JJjgdtVhNMAS4EXgdNLONa+wF9i4nwOODHFWO8E1gDbCb3vc4BvAt/M2q6/jOvyYpr7QEP+/BcyzjmXoC2cc3TOuQbz5Oiccwk8OTrnXAJPjs45l8CTo3POJfDk6JxzCTw5OudcAk+OzjmX4P8A515TG8n7WX0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "frames = [df[['opt','dataset','lr_rate','test_accuracy']][0:1],df[['opt','dataset','lr_rate','test_accuracy']][6:10]]\n",
    "df_lr_rate = pd.concat(frames,ignore_index=False)#.reset_index(drop=True)\n",
    "df_lr_rate = df_lr_rate.sort_values(by=['lr_rate'])\n",
    "ax = df_lr_rate.plot.barh(x='lr_rate', y='test_accuracy',figsize=(4,2),color='b',title='MNIST test accuracy on different Learning Rates')\n",
    "df_lr_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>opt</th>\n",
       "      <th>dataset</th>\n",
       "      <th>activation</th>\n",
       "      <th>test_accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>sgd</td>\n",
       "      <td>mnist</td>\n",
       "      <td>tanh</td>\n",
       "      <td>0.0980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>sgd</td>\n",
       "      <td>mnist</td>\n",
       "      <td>relu</td>\n",
       "      <td>0.1135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>sgd</td>\n",
       "      <td>mnist</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>0.9257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sgd</td>\n",
       "      <td>mnist</td>\n",
       "      <td>softmax</td>\n",
       "      <td>0.9600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    opt dataset activation  test_accuracy\n",
       "11  sgd   mnist       tanh         0.0980\n",
       "10  sgd   mnist       relu         0.1135\n",
       "12  sgd   mnist    sigmoid         0.9257\n",
       "0   sgd   mnist    softmax         0.9600"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVwAAACcCAYAAAAziRMsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAf80lEQVR4nO3de5xVZdn/8c+Xgw0goIIZijhW5gGGgWEkkQxQCZLEfNBMFA+JKP5+5vMrTU0TUZ/UJDMtn9TyhGmYppFiIqmoeWIGQRRRQ1QOKoicBMGBuX5/3PfgZjN79hqYvfcwc71fr/2a2et4rbXude173WvvdcvMcM45l3stCh2Ac841F55wnXMuTzzhOudcnnjCdc65PPGE65xzeeIJ1znn8sQTrmv0JBVLMkmt4vvHJJ2aMv4qSR9L+jC+P1bSQkmfSupdqLjzQdLrkgbmYLnd4v5r2dDLLjRJJ0maWpCVm1mDvYB3gc+BzmnDXwEMKI7v74zv+6ZM8/UQzub3TwOjU97/HFgAfAosAibF4a/HYZ8Cm4D1Ke9/nhbHH1LGfQ5Upbx/bBu29zTguYTT3glsBLo05D5vDi+gOJaXVrWM6wZ8Bnw5Zdh84JgCxbpFua1jup3rW+5iGboqR3G/CxyZp31kwNqUc29lIcpOIV65qOEuAE6seSOpBGhby3SfAFclWWCszYwiFIidgXLgXwBm1t3Mdo7DnwX+b817M/tl6nLM7OyUaX9JSNo10363/puajKR2wAhgFXByrtaTYd2t8rm+AugGLDezpSnD9iF8ENdbHvfXCGADMFjSV/K0zsakNOXc26XQweRNDj4lLwVmpAybAFzC1jXc64EPgQFxWMYaLvA74IYE6988T4JpLwfuSXl/CPA8sBKYDQxMGXca8A6whvCBchJwIKE2vYksn9LAKcBC4DzgtbRxuwF3AEuAFcDDKeOOAWYBqwm1tqEp+/nI2raFLz7RzwDeB56Jw/8a9/cq4Bmge8r8bYBfA+/F8c/FYY8C56bF+ypwbIbtHE5IdCvjsTgwrWycH+dfBUwCijIsp2UsNx/H/f5/SKml1Bxn4EhC7bY6HoP74t+aGtT8OP2ewIPAsnj8fpy27x4A7on7eTTQEfgT8AGwmFAxaJlSFp6L8a2Iy/tuHPc/bHmV9bs6ysSTcfqZwPlp477FF2VxYVznGMIV2edx2f9ILQtxGz8DdktZTu+4D1sDX4vrXB6H/RnYJU43Me7Dz+Kyf0ZazTAufzKhovQf4My0fXg/cDfhHHkdKK9j2w34erbhpNTogYGEK9ufAkvjsTk9QRl+Py63pjbdj7QrU+BQYEacbwZwaFpOuRL4d9y2qcQreKCIUG6Wx2M1A9ijzryzvUk2bYfVHPw3CQmpZdxJ+7B1wr0K+HHNhlN3wj05HugLCLXblg2ZcIG94k47itCuPTi+3x1oRzgR94/TdiEmq/QDV8e6/gX8CtiD0KzQJ2Xco4TksyvhxBgQh/eNBWBwjGkv4IB6JNy7Y+xt4vAfAe2BLwE3ALNS5v993Hd7xWN2aJzuB8BLKdOVxv2yUy3b+A1Ckhsct+NnhBNzp5SYXyacuLsBbwBnZ9hfZwPzgL3jtE9RS8JNPREznbhx31UClwE7AV8lJPEhKfuuCvh+nLYN8BBwS9x/X45xn5VyzKuAM+O+Gkv4sFTSMkg4H6qBgwgJ5NW0cWsIV4mtgU5Ar/QElH7Oxf+fZMtEeB3wh5Tza3A8rrsTPnRvqG05aeWoZp8/A9xMSDK9CB9eh6fsw/WE86clcDXwYh3bv60JdyNwRdwvRwHrgF2zlOEttiP9vCWUrxWEK+hWcb+vADqlHM/5hPLdJr6/Jo47C/gH4Qq+JdAH6FDXsc/VTbOJhFrdYMKJtTjDdLcA3STVeTlvZvcA5wJDgOnAUkkXNly4nAxMMbMpZlZtZk8AFYSDCuHk6CGpjZl9YGaJL1cldQMGAfea2UeE5HtKHNcF+C4h8awwsyozmx5nPQO43cyeiDEtNrN59dimy81srZl9BmBmt5vZGjPbQDhBSiV1lNSCkIzPi+vYZGbPx+kmA9+QtF9c5ihCM8zntazvBODRGG8VoQbYhlDwa9xoZkvM7BNCQe2VIfYfEJLBwjjt1fXY7nQHA7ub2RVm9rmZvQPcBvwwZZoXzOxhM6sGOhCO+3/H/bcU+E3a9O+Z2W1mtgm4i/AhvEc9YhpFSLJzgb8A3VNu7o0EppnZfbE8LDezWQmXey+xOU+SYsz3ApjZf+Kx2WBmywhXmAOSLFTS3kB/4EIzWx/j+SOxHEfPxfNnE+H8L82y2JmSVsbXjQm3rwq4Iu6XKYQa6/5ZynA2w4C3zWyimW00s/sIH/ZHp0xzh5m9Fc+l+/mi3FYRPhC/HtdZaWar61pZLhPuSMInyd2ZJoo75Mr4qpOZ/dnMjgR2IdSArpQ0pEGiDbWK41MKwErCZV0XM1tLSCZnAx9IelTSAfVY9ijgjZST5s/ASEmtCTW4T8xsRS3z7U34ZN1WC2v+kdRS0jWS5ktaTajNAHSOr6La1mVm6wm175NjoT6RcGxrsyfhcq5m3uoYw14p03yY8v86wo2jTMtamPL+vQzTJbEPsGfasf05WybIhWnTtyYc65rpbyHUdGts3g4zWxf/zbQttTmFUA4ws8WESsSpcdz2HPcHgX7xg/zbhIrCswCS9pD0F0mLYxm4h3Dsk9iTUE7XpAx7j7qPbVGW9vAyM9slvn6cMI7lZrYxbT07U0cZTmCLchtl27aaYz0ReBz4i6Qlkn4Vz+uMcpJwzew9QtvWUcDfskx+ByGJ/lfCZVeZ2V8JbYE9tifOFAuBiSkFYBcza2dm18R1Pm5mgwk1mXmEGhKES5VsTgG+KunD+LWl6wkF5Ki43t0k1XbTYCGh3a02a9nyRmRtN11SYxtJaA8+ktA+WRyHi9Cet76Odd1FaLM+AlhnZi9kmG4JIVmFBYca1t5kvrqpywdx3hrdtmEZNRYCC9KObXszOyplGkubfgOhna5m+g5m1j3h+uosE5IOBfYDLk4pE98kfAi3ou7jXuey4wf3VEIFYSTwF4vXvoSbxAaUmFkHwlWdEi57CaGctk8Z1o1tO7Z1WUf2cl2buspwtnN0i3IbJdq2mIvGm9lBhCu577FlrX8rufwe7hmENp61dU0UP7HGARmbCCSdJmmYpPaSWsQmiO7ASw0U6z3A0ZKGxNpgkaSBkrrGmsEx8ZsGGwiXMdVxvo+ArpJ2yhB3P0Ih6Eu4DOlF+JC4FzjFzD4AHgNulrSrpNaSvh1n/xNwuqQj4jbvlVKzngX8ME5fDhyXZfvax9iXEwr05m9vxJro7cD1kvaM299P0pfi+Bfi9v6azLVbCJdaw2K8rQltkxsIN3/q637gx3H/7wpctA3LqPEysEbShZLaxO3rIeng2iaOx2Qq8GtJHeK+/5qkRJffhDLx1TrGnwo8QWi/TS0TbQjNS38GjpT0A0mtJHWSVHMJm23ZEMsWoUzcmzK8PaHsrpK0F+F+SKK4zWwh4TheHc+NnoTz+54ssdTXLMIHT0tJQ0nY5JGlDC8jlN9M+20KodlsZNzfJxCOzSPZ1itpkKSS+F3l1YQmhuq65slZwjWz+WZWkXDy+wi1mkxWEy4D3yfcDfwVMNbMntu+KINYoI6J61hGqGVcQNg/LYCfED4JPyEUgrFx1icJd2Q/lPRxLYs+Ffi7mc0xsw9rXsBvge9J2o3Q5FBFqDkvBf47xvQycDqh/XAV4bKz5pP4F4REvgIYz5YnVm3uJlwmLQbmAi+mjT8fmEO4y/oJcC1blo27gRLqOMHM7E1CrekmQo3jaODoDO292dxGuFSbTbiLn+0qKaPYpvg9QmJbEGP7I6Gmn8kphBtscwn7+AHC1U0SvwWOk7QivW1SUhGhffqm1PJgZgsIH2anmtn7hKufnxKOxSy+aA/9E3BQbOp4OMP6JxNq0B+a2eyU4eOBMkJZepSt9+nVwKVx2efXstwTCVdGSwg3FceZ2bRsO6OeziOUm5WEq6pM21ibWstwbPL5H+DfcdsOSZ3JzJYTysdPCRWSnwHfM7Pazud0XyGUjdWEe1XTqbtSsvnOqnMZSToFGGNm3yp0LM7tyPynva5OktoC5wC3FjoW53Z0nnBdRvFbIMsI7XvZmi2cc1l4k4JzzuWJ13Cdcy5PPOE651yeNPUnSdVb586drbi4uNBhONcsVVZWfmxmuxc6jlzxhJumuLiYioqkXx92zjUkSdvzM+5Gz5sUnHMuTzzhOudcnnjCdc65PPGE65xzeeIJ1znn8sS/pZCmshKk7NM555LxH7N+wWu4zjmXJ55wnXMuTxpFwpV0mKTXJc2KT2o/Kvtczjm3Y2kUCZfwdPerzawXsD9f9JbrnHNNRs5umsU+wO4HuhL6bL+S0L3JhLjeGYSuakYRuh0ZImkYoTvmNpK+Rej240BgX0KfRN2A/wccQuj/aTGhG5cqSZcRuudoQ+h/6ay43heAC8zsaUlXA9Vmdkmutts55zLJZQ13KLDEzErNrAfwT+BO4AQzKyEk3bFm9kdCP0wXmNmJwGXAJDPrZWaT4rK+BhwODCf0q/VUXMZnhH7lAX5nZgfHdbUh9Eu0kdBV+/9KOjLGND6H2+yccxnlMuHOAQZLulbSYYQO6BaY2Vtx/F3AtzPNnOYxM6uKy2xJSN416yiO/w+S9JKkOYTk3B3AzF4ndOz2CPCj2jo1lDRGUoWkitDBgXPONbxc9tr7FqGX0DnAVcD3t2NxG+Iyq4Eq+6KbimqgVewN9WbguFjzvQ0oSpm/hNAT6JczxHqrmZWbWTk02SfDOecKLGcJV9KewDozuwe4DugHFEv6epxkFKFb4XRrgPb1XF1Ncv1Y0s7AcSlx/BewG6E2fZOkXeq5bOecaxC5/KVZCXCdpGqginCDrCPwV0k1N83+UMt8TwEXSZpFuGmWlZmtlHQb8BrwYVw2kjoD1wBHmNlCSb8Dfgucul1b5pxz28A7kUwjlRv4A8idayj1STGSKkPTXtPUWL6H65xzTZ4/vCZNnz7gPew453LBa7jOOZcnnnCdcy5PPOE651yeeMJ1zrk88YTrnHN54gnXOefyxBOuc87liSdc55zLE0+4zjmXJ55wnXMuTzzhOudcniR6loKk3YEzCb0rbJ7HzH6Um7Ccc67pSfrwmr8DzwLTgE25C8c555qupAm3rZldmNNIGonKSpAKHYVzOw5/pHZySdtwH5F0VE4jcc65Ji5pwj2PkHTXS1oTX6tzGZhzzjU1iRKumbU3sxZmVhT/b29mHbZlhZL+KOmgbZm3HuuYUltnkZIul3R+LtftnHOZJO7xQdJwQs+3AE+b2SPbskIzG70t89VzHd784ZxrdBLVcCVdQ2hWmBtf50nK2qOupHaSHpU0W9Jrkk6Q9LSk8jj+DElvSXpZ0m2xV10k3SnpfyW9KOkdSQMl3S7pDUl3piz/RElz4rKvTRn+buyxF0mXxHU8B+yffNc451zDSlrDPQroZWbVAJLuAl4BLs4y31BgiZkNi/N1JHSXjqQ9gV8AZcAa4Elgdsq8uwL9gOHAZKA/MBqYIakXsBS4FugDrACmSvq+mT1cswBJfYAfAr3its4EKhNus3PONaj6/NIstU20Y8J55gCDJV0r6TAzW5Uyri8w3cw+MbMq4K9p8/7DQh/uc4CPzGxOTPivE36AcTChaWOZmW0E/swXTR41DgMeMrN1ZraakLi3ImmMpApJFbAs4aY551z9JK3hXg28IukpQITEdlG2mczsLUllhBryVZL+VY/YNsS/1Sn/17xvBVTVY1l1MrNbgVsBpHL/VqFzLieSfkvhPuAQ4G/Ag0A/M5uUbb7YbLDOzO4BriM0H9SYAQyQtKukVsCIesb+cpy/s6SWwInA9LRpngG+L6mNpPbA0fVch3PONZg6a7iSDjCzebGWCrAo/t1T0p5mNjPL8kuA6yRVE2qkY4EJAGa2WNIvCYnzE2AesCrTgtKZ2QeSLgJqat2Pmtnf06aZKWkSoW14KSHJO+dcQcjq+F2epFvNbExsSkhnZnb4dq1c2tnMPo013IeA283soe1Z5vYKTQoVhQzBuR1KQ/60V1KlmZU33BIblzpruGY2Jv77XTNbnzpOUlEDrP9ySUcCRcBU4OEs0zvn3A4r6U2z59my/TXTsHoxM//Vl3Ou2cjWhvsVYC+gjaTehLZSgA5A2xzHVhB9+kCFtyg453IgWw13CHAa0BW4PmX4GuDnOYrJOeeapGxtuHcBd0kaYWYP5ikm55xrkhK14ZrZg5KGAd0JN7hqhl+Rq8Ccc66pSfrwmj8AJwDnEtpxjwf2yWFczjnX5CR9lsKhZnYKsMLMxhMeKvON3IXlnHNNT9KE+1n8uy7+XLcK6JKbkJxzrmlK+j3cR2IPCtcRHnFowG05i8o555qgpDfNroz/PijpEaAo7VGLzjnnskh60+xVST+X9DUz2+DJ1jnn6i9pG+7RwEbgfkkzJJ0vqVsO43LOuSYn6fNw3zOzX5lZH2Ak0BNYkNPInHOuialPr737EL6LewKwCfhZroJyzrmmKFHClfQS0JrQ79jxZvZOTqMqoMpKkLJPt70a8hmizrkdQ9Ia7ilm9mZOI3HOuSYu2+MZT479kQ2Lz1LYgpldX8tszjnnapGthtsu/m1fy7hGe1Es6VMz27nQcTjnXKpsj2e8Jf47zcz+nTpOUv+cRZWAJBH6ZKsuZBzOOZdU0u/h3pRwWE5JKpb0pqS7gdeAX8TvBb8qaXwt0w+Mv4yref87SaflMWTnnNssWxtuP+BQYHdJP0kZ1QFomcvA6rAfcGqM4TigL+GRkZMlfdvMnilQXM45V6dsNdydgJ0Jibl9yms1IdkVwntm9iLwnfh6hfBAnQMIybjeJI2RVCGpApY1XKTOOZciWxvudGC6pDvN7L08xZTN2vhXwNUp7cy12ciWHyq1du1uZrcCtwJI5Y32ZqBzbseWtA33j/HxjABI2lXS4zmKKanHgR9J2jnGtJekL6dN8x5wkKQvxfiPyHeQzjlXI+kPHzqb2cqaN2a2opbklldmNlXSgcAL4QsLfAqcDCxNmWahpPsJN9gWEJofnHOuIGQJfmMqqRI41szej++Lgb+ZWVlOoyuA0KRQkfP1+E97nduapEozKy90HLmStIZ7CfCcpOmEttPDgDE5i8o555qgpD0+/FNSOSHJvgI8zBf9nDnnnEsg6dPCRgPnAV2BWcAhwAvA4bkLrTD69IGK3LcoOOeaoaTfUjgPOJjwHdhBQG9gZd2zOOecS5U04a43s/UAkr5kZvOA/XMXlnPONT1Jb5otit9jfRh4QtIKwndcnXPOJZT0ptmx8d/LJT0FdAT+mbOonHOuCUrcp1mN+HNf55xz9ZS0Ddc559x28oTrnHN54gnXOefyxBOuc87lSb1vmjnX2FVVVbFo0SLWr19f6FBcBkVFRXTt2pXWrVsXOpS88oTrmpxFixbRvn17iouLiY/udI2ImbF8+XIWLVrEvvvuW+hw8sqbFFyTs379ejp16uTJtpGSRKdOnZrlFYjXcNNUVkIuzlN//m1+ebJt3Jrr8fEarnM5sHLlSm6++eZtmveGG25g3bp1DRyRaww84bomT2rYVxJNJeFu3Lix0CE0KY064UraRdI52zH/0/HB6c7l1UUXXcT8+fPp1asXF1xwAddddx0HH3wwPXv2ZNy4cQCsXbuWYcOGUVpaSo8ePZg0aRI33ngjS5YsYdCgQQwaNCjj8seOHUt5eTndu3ffvDyAGTNmcOihh1JaWkrfvn1Zs2YNmzZt4vzzz6dHjx707NmTm266CYDi4mI+/vhjACoqKhg4cCAAl19+OaNGjaJ///6MGjWKd999l8MOO4yysjLKysp4/vnnN6/v2muvpaSkhNLS0s3bXFb2Rc9bb7/99hbvmz0za7QvoBh4bTvmfxoor988fSy0uDbsy+XP3Llzt3hfiGO5YMEC6969u5mZPf7443bmmWdadXW1bdq0yYYNG2bTp0+3Bx54wEaPHr15npUrV5qZ2T777GPLli2rc/nLly83M7ONGzfagAEDbPbs2bZhwwbbd9997eWXXzYzs1WrVllVVZXdfPPNNmLECKuqqtpi3tT1zJgxwwYMGGBmZuPGjbOysjJbt26dmZmtXbvWPvvsMzMze+utt6xPnz5mZjZlyhTr16+frV27dovlDhw40F555RUzM7v44ovtxhtvrHUb0o+TmRlQYY0g9+Tq1ahruMA1wNckzZL0G0n/kjRT0hxJx0Do0FLSG5Juk/S6pKmS2qQs43hJL0t6S9JhhdkM15xNnTqVqVOn0rt3b8rKypg3bx5vv/02JSUlPPHEE1x44YU8++yzdOzYMfEy77//fsrKyujduzevv/46c+fO5c0336RLly4cfPDBAHTo0IFWrVoxbdo0zjrrLFq1CvfId9ttt6zLHz58OG3ahNOoqqqKM888k5KSEo4//njmzp0LwLRp0zj99NNp27btFssdPXo0d9xxB5s2bWLSpEmMHDky+c5q4hr7txQuAnqYWS9JrYC2ZrZaUmfgRUmT43T7ASea2ZmxW/QRwD1xXCsz6yvpKGAccGS+N8I1b2bGxRdfzFlnnbXVuJkzZzJlyhQuvfRSjjjiCC677LKsy1uwYAETJkxgxowZ7Lrrrpx22mnb9BWrVq1aUV1dDbDV/O3atdv8/29+8xv22GMPZs+eTXV1NUVFRXUud8SIEYwfP57DDz+cPn360KlTp3rH1lQ19hpuKgG/lPQqMA3YC9gjjltgZrPi/5WEpogaf8sw/IsFS2MkVUiqgGUNHbdrhtq3b8+aNWsAGDJkCLfffjuffvopAIsXL2bp0qUsWbKEtm3bcvLJJ3PBBRcwc+bMreatzerVq2nXrh0dO3bko48+4rHHHgNg//3354MPPmDGjBkArFmzho0bNzJ48GBuueWWzTfAPvnkEyC04VZWVgLw4IMPZlzfqlWr6NKlCy1atGDixIls2rQJgMGDB3PHHXdsvsFXs9yioiKGDBnC2LFjOf3007dh7zVdO1LCPQnYHehjZr2Aj4Caj9oNKdNtYsua+4YMwzczs1vNrNzMysMqnNs+nTp1on///vTo0YMnnniCkSNH0q9fP0pKSjjuuONYs2YNc+bMoW/fvvTq1Yvx48dz6aWXAjBmzBiGDh2a8aZZaWkpvXv35oADDmDkyJH0798fgJ122olJkyZx7rnnUlpayuDBg1m/fj2jR4+mW7du9OzZk9LSUu69914Axo0bx3nnnUd5eTktW7bMuC3nnHMOd911F6WlpcybN29z7Xfo0KEMHz6c8vJyevXqxYQJEzbPc9JJJ9GiRQu+853vNMj+bCoU2qkbJ0mdgJlmto+k84Cvm9m5kgYBTwI1vwt8xMx6xHnOB3Y2s8slPQ2cb2YVsRmiwsyK615nuUHDd9vbiHdzk/PGG29w4IEHFjqMZm3ChAmsWrWKK6+8MuM0tR0nSZWh4tM0Neo2XDNbLunfkl4DZgAHSJpDyIjzChudc642xx57LPPnz+fJJ58sdCiNTqNOuABmluQWZ4+U6Sek/D8w5f+PydCG61xj9c1vfpMNGzZsMWzixImUlJQUKKLsHnrooUKH0Gg1+oTrXHP20ksvFToE14B2pJtmzjm3Q/OEm6ZPn1z8zqzQW9X8NOabwa75Hh9PuK7JKSoqYvny5c32pG7szMIDyLP9gKIp8jZc1+R07dqVRYsWsWyZ/4ilsarpYqe58YTrmpzWrVs3u65b3I7BmxSccy5PPOE651yeeMJ1zrk8adTPUigESWuANwsdRz11Bj4udBD14PHm3o4Wc028+5hZk32ClN8029qbO9rDMyRV7Egxe7y5t6PFvKPFu628ScE55/LEE65zzuWJJ9yt3VroALbBjhazx5t7O1rMO1q828RvmjnnXJ54Ddc55/Kk2SZcSUMlvSnpP5IuqmX8lyRNiuNfklSc/yi3iCdbvD+RNFfSq7E7+X0KEWdaTHXGnDLdCEkmqaB3qZPEK+kHcT+/LunefMeYFku2MtFN0lOSXonl4qhCxJkSz+2SlsYeXGobL0k3xu15VVJZvmPMOTNrdi+gJTAf+CqwEzAbOChtmnOAP8T/fwhMauTxDiJ0Iw8wtpDxJo05TtceeAZ4EShvzPEC+wGvALvG919u5PHeCoyN/x8EvFvgMvFtoAx4LcP4o4DHCD10HwK8VMh4c/FqrjXcvsB/zOwdM/sc+AtwTNo0xwB3xf8fAI6QpDzGmCprvGb2lJmti29fBAr9KKYk+xjgSuBaYH0+g6tFknjPBH5vZisAzGxpnmNMlSReAzrE/zsCS/IY31bM7BngkzomOQa424IXgV0kdclPdPnRXBPuXsDClPeL4rBapzGzjcAqoFNeottaknhTnUGoKRRS1pjjJePeZvZoPgPLIMk+/gbwjdix6YuShuYtuq0lifdy4GRJi4ApwLn5CW2b1bec73D8l2ZNjKSTgXJgQKFjqYukFsD1wGkFDqU+WhGaFQYSriCekVRiZisLGlVmJwJ3mtmvJfUDJkrqYWbVhQ6suWquNdzFwN4p77vGYbVOI6kV4ZJseV6i21qSeJF0JHAJMNzMNqSPz7NsMbcn9Lb8tKR3CW12kwt44yzJPl4ETDazKjNbALxFSMCFkCTeM4D7AczsBaCI8MyCxipROd+RNdeEOwPYT9K+knYi3BSbnDbNZODU+P9xwJMWW/YLIGu8knoDtxCSbSHbFmvUGbOZrTKzzmZWbGbFhHbn4WZWUZhwE5WJhwm1WyR1JjQxvJPPIFMkifd94AgASQcSEm5j7gZjMnBK/LbCIcAqM/ug0EE1qELftSvUi3BH9C3Cnd5L4rArCCc9hML5V+A/wMvAVxt5vNOAj4BZ8TW5se/jtGmfpoDfUki4j0VoBpkLzAF+2MjjPQj4N+EbDLOA7xQ43vuAD4AqwtXCGcDZwNkp+/f3cXvmFLo85OLlvzRzzrk8aa5NCs45l3eecJ1zLk884TrnXJ54wnXOuTzxhOucc3niCdc55/LEE65zzuWJJ1znnMuT/w8B6G/b9D8jMQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "frames = [df[['opt','dataset','activation','test_accuracy']][0:1],df[['opt','dataset','activation','test_accuracy']][10:13]]\n",
    "d_act_rate = pd.concat(frames,ignore_index=False)#.reset_index(drop=True)\n",
    "d_act_rate = d_act_rate.sort_values(by=['test_accuracy'])\n",
    "ax = d_act_rate.plot.barh(x='activation', y='test_accuracy',figsize=(4,2),color='b',title='MNIST Test Accuracy on different Activation Functions')\n",
    "d_act_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>opt</th>\n",
       "      <th>dataset</th>\n",
       "      <th>dropout_rate</th>\n",
       "      <th>test_accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>sgd</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.9884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sgd</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.9600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>sgd</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.9852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>sgd</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.0980</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    opt dataset  dropout_rate  test_accuracy\n",
       "15  sgd   mnist           0.1         0.9884\n",
       "0   sgd   mnist           0.2         0.9600\n",
       "14  sgd   mnist           0.5         0.9852\n",
       "13  sgd   mnist           0.9         0.0980"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATMAAACrCAYAAAAKGnvEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAaf0lEQVR4nO3debgcVZ3/8fcnISGBJARMZDW5soisImQQfjKCkghBCSIgBgMEAgiIMKPwA8SNERlFQRnRYTGsSgTcJo4iwyoDsiRsQVYDBMNmwpKEkEAIfOePc5pU2u57q5Pb9970/byep5/uqjpVdep01bfrVHWdo4jAzGxV16e7M2Bm1hkczMysJTiYmVlLcDAzs5bgYGZmLcHBzMxagoOZ9RiS2iSFpNXy8LWSDi1MP0PSi5JeyMP7SpotaaGkD3ZXvq1nWKFgJmmWpCWShlWNvy/vjG15+NI8vGMhzaaSojB8i6QjCsNfkfRU3kGfkXRVHv9QHrdQ0luSXi8Mf6UqH+cXpi2R9GZh+NoV2N6Jkm4rmfZSSUslrd/oemx5ETE2Ii4DkDQC+DKwZUSsl5N8HzguIgZFxH1dmbfq/bbG9Epgrux3f5f035LGdGU+yyqzj+dtrhx3L0r6ddn9XNJukp7pnNzWtjJnZk8B4ysDkrYB1qiR7mXgjDILzL/CBwOjI2IQMAq4ESAitso77SDgf1m2Ew+KiDOLy4mIowtpzwSuKqQd2/imliNpTWA/YD4woVnrqbPu1bpyfd1gBPBSRMwpjBsJPLQiC+vC8hqa98MPANcDv5E0sZvztDKOy9uzKTCI9IPSM0REwy9gFvBVYFph3PeB04AA2vK4S4FzgBeAXfO4TdNq35nvFuCI/Pk84Icl1v/OPCXSfhP4WWF4J+DPwDzgAWC3wrSJwJPAq6Rg/TlgC+B14C1gITCvnXUdAswGTgD+UjVtHeAS4DngFeC3hWn7APcDC4AngD0L5Ty61rYAbbmsJwF/A27N46/J5T0fuBXYqjD/QOBs4Ok8/bY87vfAF6vyOwPYt852jiMFkXn5u9iiat84Mc8/H7gKGFBnOX3zfvNiLvcv5G1arfg9A6OBxcDb+TuYkt8DeA14IqffAPgVMDd/f8dXld0vgZ/lcj4CWAuYDDwPPEv60e1b2Bduy/l7JS9vbJ727bw/vJ7zcV6Nbat8P6tVjT8R+DvQp1BeJ+fyegNYrUT5ngo8nPN1SbF8gSOBmaSTiKnABvXyUyjfUvs4VccdcCzwUGH4MOAR0vHzJPD5PH7Nqu9vYf6u+gCnkPb5l4CrgXXyPAPyd/VSLodpwLrtHusrEcxGA4/lgugLPEP6pawOZmcAxwO3lQhmE/KXcBLprKxvZwYzYMNcOHvlghyTh4fnAl8AbJ7Trk8OBOQdu8S6bgTOAtYFlgI7FKb9nnRgrw30Y1lw35F00I/JedoQeH8DwezynPeBefzhwGBgdeCHwP2F+X+cy27D/J39v5zuM8BdhXQfyOXSv8Y2vo8UQMbk7fj/pIOnfyHPd5N21nVIO/fRdcrraOBR4D057c3UCGb5827AM1XzB7Bp/twHuAf4OtAf2Jh0QO1RKLs3gU/ltAOB3wAX5PJ7d8535QCcmNMfmcvqGNIPkcrsg9QPZhvn8VsUyuv+XAYDS5bvXwpldjtwRp72MdIPw/b5e/0Ry37k/iE/VeU7kQ728ar07wJuAP6rMP0TwCaAgF2BRcD27Xx/JwB3Ahvl/F4ATMnTPg/8jlTb6wvsAAxpZjD7KvDvwJ6kU+jVqB3MViedPYylnWCWhz+XC+k10gF1cicGs5OBK6qmXwccStqh55GqiQOr0pT5okeQfnm2Kyz33EJgfBtYu8Z8FwA/aK+cOwhmG7eTp6E5zVqkA3gx8IEa6QaQfuU3y8PfB35SZ5lfA64uDPchndXsVsjzhML0s4Dz6yzrJgqBDvg4Kx7MPgT8rWr6qcAlhbK7tTBtXdKZ0MDCuPHAzYXvfGZh2hp5feuV2QepH8wG5PEfLpTX4Q2Wb7HM9mLZmelk4KzCtEGkgNxWKz+sWDBbRPrxDVIQHtFO+t8CJ7Tz/T0C7F4YXj/ndzXSj/KfgW3LHOcRsdJ3M68ADsoFcXm9RBHxBvCt/GpXRPw8IkaTDsSjgW9J2mMl81kxEjhA0rzKC9gFWD8iXgMOzOt8XtLvJb2/gWUfDDwSEffn4Z8DB0nqR/oVfTkiXqkx33tIp9kranblg6S+kr4j6QlJC0g7PsCw/BpQa10R8TrprHGCpD6kg/qKOuvbgFRNrcz7ds7DhoU0LxQ+LyIdVPWWNbsw/HSddGWMBDao+m6/QgpaFbOr0vcjfdeV9BeQztAq3tmOiFiUP9bblrIq5fRynXyVKd/qMtugzrwLSScExXlX1vERsRawLamWsVFlgqSxku6U9HIuz71I+109I0nXECvl/wipqrsuaf+7DviFpOcknZWPpbpWKphFxNOkawl7Ab/uIPklpAD16ZLLfjMiriFdS9h6ZfJZMJt0Zja08FozIr6T13ldRIwh/UI8ClxUyU6JZR8CbCzphfzXgXNIX+Reeb3rSBpaJ0+b1Fnmayx/U2W9GmmKeTuIdP1tNOlsrC2PF6n68Xo767qMdFa8O7AoIu6ok+450k6YFiyJFJCfrZO+Pc/neStGrMAyKmYDT1V9t4MjYq9CmqhK/wYwrJB+SERsVXJ9ZfaJWvYF5pAu0dRaVpnyrS6z5+rMuyapOvgsaV+C+vtTQ9sTEQ+Sal0/VrI66Xrl90nXtoYCfyDte/WWP5t0HbL4nQ2IiGfz8X96RGxJuhzySdIxVldn/M9sEvCxfGZTV0QsBb5BqurVlG8Pf0LSYEl9JI0FtgLu6oR8QrqguLekPfJZzIB8y3gjSetK2ifvAG+QLlK+nef7O7CRpP518r0zKUjsCGyXX1sDVwKHRMTzwLXATyStLamfpI/k2ScDh0naPW/zhoUzwvuBz+b0o4D9O9i+wTnvL5F22nfu8uZf+IuBcyRtkLd/57wTkoPX26QbBPXOyiBdpP1Ezm8/0t8l3iBVCRp1NXB8Lv+1SReDV9TdwKuSTpY0MG/f1pL+qVbi/J38D3C2pCG57DeRtGvJ9f2ddP2rlLx/HUc6Bk7N30ctZcr3C7nM1iHddLsqj59C2pe2y9/rmaRrobMiYi4pqE3IZXM4y/+wtbuP13EZ6SxqHOk65eqkmy9L87H78arlv0vSWoVx5wPfljQSQNJwSfvkzx+VtI2kvqRr2W+y7HisaaWDWUQ8ERHTSyafQvo1rmcBqWrwN9L1q7OAYyKi1H+8OhIRs0lnLl8hFfps0s2GPvn1JdKv28ukC5jH5FlvIt1dekHSizUWfSjpQuiDEfFC5QWcC3wy73QHk76QR0m/zP+S83Q36S7QD0jXIv7Esl/Xr5F2uFeA00nBsT2Xk6oZz5Ludt1ZNf1E4EHSnaGXge+y/D5wObANKejXFBGPkW7U/Ih0trc3sHdELOkgb7VcRKpKPADcS8dn93VFxFukX+/tSLWFF4Gfks5Q6zmEdBBW7gz+knRWXsa5wP6SXpH0H+2kmyfpNVK57wUcEBEXt7MdZcr3SlIgfpJ02eCMPO8NpH3mV6TjbBPgs4X5jiTt7y+RThKKAbKjfbxWXpeQyuFrEfEq6Ubf1aSyPIh0N7WS9lHS8f9krlZukOedCvyPpFdJ++uH8izrkb6PBaTq559o/0f2nTszZkg6BDgqInbp7rxYbZJmkS7a39Ddeelp/DiTASBpDdL/hi7s7ryYrQgHMyPfLZ5Luq7RUVXWrEdyNdPMWoLPzMysJTiYmVlLaOpT+pIuJt0unxMR//DH1/yHwHNJt6wXARMj4t72ljls2LBoa2trQm7NrCP33HPPixExvLvzUUuzmxy5lNQSRr1HncYCm+XXh4D/ZNn/TGpqa2tj+vSyf2szs84kaWUeOWuqplYzI+JWln8Grdo+wOWR3AkMVTc1ajh3Lkyblt6tOVzG1kzdfc1sQ5Z/aPYZOveh2FKmTIGRI2HMmPQ+ZUpX56D1uYyt2bo7mJUi6ShJ0yVNn9vJP+tz58KkSbB4Mcyfn94nTfLZQ2dyGVtX6O5g9izLtwCwETVaX4iICyNiVESMGj68c689zpoF/asere3XL43vDK5aNb+MzaD7g9lU4JDchMhOwPzcmkGXaWuDJVWPSL/5Zhq/sly1SppZxmYVTQ1mkqYAdwCbK/W0NEnS0ZKOzkn+QHryfyapBYVjm5mfWoYPh8mTYeBAGDIkvU+enMavDFetlmlWGZsVNfWvGRExvoPpQerEoluNHw+jR6dqT1tb5xxklarV4sXLxlWqVr3xIG5GGZsVrQpdW3WJ4cM79wBz1eofdXYZmxV19zWzluWqlVnX8plZE7lqZdZ1HMyazFUrs67haqaZtQQHMzNrCQ5mZtYSHMzMrCU4mJlZS3AwM7OW4GBmZi3BwczMWoKDmZm1hA6DmaR7JH1B0tqNLlzSnpIekzRT0ik1po+QdLOk+yTNkLRXo+swM4NyZ2YHAhsA0yT9QtIeuYu4dknqC/yY1APTlsB4SVtWJfsqcHVEfBD4LPCThnJvZpZ1GMwiYmZEnAa8D7gSuBh4WtLpktZpZ9YdgZkR8WRELAF+QeqNabnFA0Py57WA5xrdADMzKHnNTNK2wNnA94BfAQcAC4Cb2pmtTM9L3wQmSHqG1OrsF0vl2sysSoetZki6B5gHTAZOiYg38qS7JH14Jdc/Hrg0Is6WtDNwhaStI+LtqjwcBRwFMGLEiJVcpZm1ojJNAB0QEU/WmhARn25nvjI9L00C9szLukPSAGAYMKdqPRcCFwKMGjUqSuTZzHqZMtXMIyQNrQxIWlvSGSXmmwZsJum9kvqTLvBPrUrzN2D3vNwtgAFAL+zyw8xWVplgNjYi5lUGIuIVoMO/UETEUuA44DrgEdJdy4ck/ZukcTnZl4EjJT0ATAEm5k5OzMwaUqaa2VfS6pVrZZIGAquXWXhE/IF0Yb847uuFzw8DK3vdzcysVDD7OXCjpEvy8GHAZc3LkplZ4zoMZhHxXUkzyNe2gG9FxHXNzZaZWWNKdWgSEdcC1zY5L2ZmK6zMs5k7SZomaaGkJZLekrSgKzJnZlZWmbuZ55H+3PpXYCBwBOmZSzOzHqPU40wRMRPoGxFvRcQl5D+6mpn1FGWumS3Kf3q9X9JZwPO4HTQz62HKBKWDc7rjgNdIjyjt18xMmZk1qt0zs9wm2ZkR8TngdeD0LsmVmVmD2j0zi4i3gJG5mmlm1mOVuWb2JHC7pKmkaiYAEXFO03JlZtagMsHsifzqAwxubnbMzFZMmceZfJ3MzHq8Mi3N3kxqq385EfGxEvPuCZwL9AV+GhHfqZHmM6TmswN4ICIO6jjbZmbLK1PNPLHweQDpbxlLO5qp0DvTGFL7/9MkTc3N/lTSbAacCnw4Il6R9O5GMm9mVlGmmnlP1ajbJd1dYtnv9M4EIKnSO9PDhTRHAj/ODT4SEXP+YSlmZiWUqWYWu5PrA+xA6hauI7V6Z/pQVZr35XXcTqqKfjMi/lhi2WZmyylTzbyHdD1LpOrlU6SOSDpr/ZsBu5E6PLlV0jbFZrrBvTOZWcfKVDPfu4LLLtM70zPAXRHxJvCUpMdJwW1aVR7cO5OZtatMe2ZfqNE707Elll2md6bfks7KkDSMVO2s2a2dmVl7yjxofmSN3pmO7Gimkr0zXQe8JOlh4GbgpIh4qdGNMDMr2zuTKl3A5b9clHpWs0TvTAF8Kb/MzFZYmWD2R+AqSRfk4c/ncWZmPUaZYHYy6U7iMXn4euCnTcuRmdkKKBPMBgIXRcT58E41c3VgUTMzZmbWiDI3AG4kBbSKgcANzcmOmdmKKRPMBkTEwspA/rxG87JkZta4MsHsNUnbVwYk7QAsbl6WzMwaV+aa2b8A10h6jvRI03rAgU3NlZlZg8o8zjRN0vuBzfOox/LjR2ZmPUaZMzNIgWxLUntm20siIi5vXrbMzBpTpgmgb5Cen9yS9G/+scBtgIOZmfUYZW4A7A/sDrwQEYcBH6Bce2ZmZl2mTDBbHBFvA0slDQHmsHzTPmZm3a7MNbPpuQmgi0gNNS4E7mhqrszMGtThmVlEHBsR8/LjTGOAQ3N1EwBJW9WbV9Kekh6TNFPSKe2k209SSBrV6AaYmUG5auY7ImJWRMyoGn1FrbSF3pnGkm4ejJe0ZY10g4ETgLsayYuZWVFDwawO1Rn/Tu9MEbEEqPTOVO1bwHeB1zshL2bWS3VGMKvXJn+t3pk2LCbIj0m9JyJ+3wn5MLNerDOC2QqR1Ac4B/hyibRHSZouafrcuXObnzkzW+V0RjBbUmd8R70zDQa2Bm6RNAvYCZha6yZARFwYEaMiYtTw4cM7Ictm1mrK9M50Y3vjImKnOrO22ztTRMyPiGER0RYRbcCdwLiImN7gNpiZ1f+fmaQBpHbLhklam2UX+odQde2rlohYKqnSO1Nf4OJK70zA9Iio7nbOzGyFtfen2c+Tmv/ZgPRn2UowWwCcV2bhHfXOVDV+tzLLNDOrpW4wi4hzgXMlfTEiftSFeTIza1iZGwAv5D+2Iumrkn5dbHnWzKwnKBPMvhYRr0raBRgNTAb+s7nZMjNrTJlg9lZ+/wRwYf6Da6kezc3MukqZYPZs7s38QOAPklYvOZ+ZWZcpE5Q+Q/p7xR4RMQ9YBzipqbkyM2tQmSaAFpEaZNwlj1oK/LWZmTIza1SZJwC+AZwMnJpH9QN+1sxMmZk1qkw1c19gHPAaQEQ8R3qu0sysxygTzJZERJCb+pG0ZnOzZGbWuDLB7Op8N3OopCOBG0j9AZiZ9RhlOjQZDvyS9Ezm5sDXSX+eNTPrMcoEszERcTJwfWWEpLNJNwXMzHqE9poAOgY4FthYUrETk8HA7c3OmJlZI9q7ZnYlsDepQcW9C68dImJCmYV31NWcpC9JeljSDEk3Shq5AttgZtZuE0DzgfnA+BVZcKGruTGkzkymSZoaEQ8Xkt0HjIqIRflM8CzSY1NmZg1p5jOWHXY1FxE35ycMIDWbvVET82NmLayZwazDruaqTAKubWJ+zKyFlbmb2XSSJgCjgF3rTD8KOApgxIgRXZgzM1tVNPPMrKOu5gCQNBo4jdQz0xu1FuSu5sysI80MZu12NQcg6YPABaRANqeJeTGzFte0YBYRS4FKV3OPAFdXupqTNC4n+x4wCLhG0v2S3P2cma2Qpl4z66iruYjwY1Fm1inc/LWZtQQHMzNrCQ5mZtYSHMzMWsjcuTBtWnrvbRzMzFrElCkwciSMGZPep0zp7hx1LQczsxYwdy5MmgSLF8P8+el90qTedYbmYGbWAmbNgv79lx/Xr18a31s4mJm1gLY2WLJk+XFvvpnG9xYOZmYtYPhwmDwZBg6EIUPS++TJaXxv0SNazTCzlTd+PIwenaqWbW29K5CBg5lZSxk+vPcFsQql/n1XHZLmAk83afHDgBdXoeWuilwWzdfMMh4ZET0yXK5ywayZJE2PiFGrynJXRS6L5uutZewbAGbWEhzMzKwlOJgt78JVbLmrIpdF8/XKMvY1MzNrCT4zM7OW0OuCmaQ9JT0maaakU2pM/4ikeyUtlbR/Jy/7S5IeljRD0o2SRq7MtvRkJcpioqS5ue+H+yUd0R35XJWVKOOReT+bIekWSa3dyXZE9JoX0Bd4AtgY6A88AGxZlaYN2Ba4HNi/k5f9UWCN/PkY4KruLpNuLOeJwHndnddV9VWyjK8BDs2fPwZc0d35buart52Z7QjMjIgnI2IJ8Atgn2KCiJgVETOAt5uw7JsjYlEevJPUl2gr6rAsbKWVKeMtgZvy55trTG8pvS2YbQjMLgw/k8d1x7InAdd20rp7mrJlsV+uAv1S0ntqTLf6ypTxA8Cn8+d9gcGS3tUFeesWvS2Y9QiSJgCjSP2G9la/A9oiYlvgeuCybs5PKzoR2FXSfcCuwLPAW92bpebpbQ+aPwsUzwA2yuO6bNmSRgOnAbtGxBudtO6epsOyiIiXCoM/Bc7qgny1kjJl/Bz5zEzSIGC/iJjXZTnsYr3tzGwasJmk90rqD3wW6Kxe1DtctqQPAhcA4yJiTiettycqUxbrFwbHkXq9t/LKlPEwSZVj/FTg4i7OY5fqVcEsIpYCxwHXkQ6eqyPiIUn/JmkcgKR/kvQMcABwgaSHOmvZpGrlIOCa/HeEzgqkPUrJsjhe0kOSHgCOJ93dtJJKlvFuwGOSHgfWBb7dLZntIn4CwMxaQq86MzOz1uVgZmYtwcHMzFqCg5mZtQQHMzNrCQ5mZtYSHMwMSd+UdGI3rLdN0kGdtKyhko7tjGXZqsnBzGqS1BWPurUBpYNZB3kaCjiY9WIOZr2UpNMkPS7pNmDzPO4WST+UNB04QdLuku6T9KCkiyWtntPNknRWHn+3pE3z+DZJNxUanxyRx19abOhS0sL88TvAP+enIf61Tj4nSpoq6SbgRkmD8rLvzevfp7CsTfKyvpfnPUnStJyf0zu9EK1H6W0PmhsgaQfSs3zbkfaBe4F78uT+ETFK0gDgr8DuEfG4pMtJDUr+MKebHxHbSDokj/sk8CPgsoi4TNLhwH8An2onK6cAJ0bEJzvI8vbAthHxcj472zciFkgaBtyZHws7Bdg6IrbL2/hxYDNSu18Cpkr6SETcWrKYbBXjM7Pe6Z+B30TEoohYwPIPKF+V3zcHnoqIx/PwZcBHCummFN53zp93Bq7Mn68Adumk/F4fES/nzwLOlDQDuIHUhte6Neb5eH7dRwrW7ycFN2tRPjOzaq+VTBd1PteylPzDmVtx6L8SefocMBzYISLelDQLGFBjHgH/HhEXNLguW0X5zKx3uhX4lKSBkgYDe9dI8xjQVrkeBhwM/Kkw/cDC+x35859J1VdIQed/8+dZwA758zigX/78KjC4wbyvBczJgeyjQKVTmOplXQccntvxQtKGkt7d4LpsFeIzs14oIu6VdBWpWeU5pLaxqtO8LukwUnNFq+U05xeSrJ2rem8A4/O4LwKXSDoJmAsclsdfBPxXbu7njyw705oBvJXHXxoRPyiR/Z8Dv5P0IDAdeDTn9yVJt0v6C3BtRJwkaQvgDkkAC4EJeXutBbkJIGtYrtqNiogXuzsvZhWuZppZS/CZmfUIkvYAvls1+qmI2Lc78mOrHgczM2sJrmaaWUtwMDOzluBgZmYtwcHMzFqCg5mZtYT/A2KCkCHUuDe0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "frames = [df[['opt','dataset','dropout_rate','test_accuracy']][0:1],df[['opt','dataset','dropout_rate','test_accuracy']][13:16]]\n",
    "d_drp_rate = pd.concat(frames,ignore_index=False)#.reset_index(drop=True)\n",
    "d_drp_rate = d_drp_rate.sort_values(by=['dropout_rate'])\n",
    "list(d_drp_rate['dropout_rate'].values)\n",
    "ax = d_drp_rate.plot.scatter(x='dropout_rate', y='test_accuracy',figsize=(4,2),color='b',xticks=list(d_drp_rate['dropout_rate'].values),xlim=[0,1],title='MNIST Test Accuracy on different Dropout Rates')\n",
    "d_drp_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "vector_cols = ['aux1_accuracy', 'aux1_loss', 'aux2_accuracy',\n",
    "       'aux2_loss', 'main_accuracy', 'main_loss', 'loss',\n",
    "         'val_aux1_accuracy', 'val_aux1_loss', 'val_aux2_accuracy',\n",
    "       'val_aux2_loss', 'val_loss', 'val_main_accuracy', 'val_main_loss']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in vector_cols:\n",
    "    df[col] = df.apply(lambda x: x[col][-1],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_name</th>\n",
       "      <th>activation</th>\n",
       "      <th>dataset</th>\n",
       "      <th>dropout_rate</th>\n",
       "      <th>epochs</th>\n",
       "      <th>input_dims</th>\n",
       "      <th>lr_rate</th>\n",
       "      <th>opt</th>\n",
       "      <th>main_accuracy</th>\n",
       "      <th>val_main_accuracy</th>\n",
       "      <th>loss</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>test_loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>inception</td>\n",
       "      <td>softmax</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>(224, 224, 3)</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.993060</td>\n",
       "      <td>0.957121</td>\n",
       "      <td>0.067948</td>\n",
       "      <td>0.9600</td>\n",
       "      <td>0.408434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>inception</td>\n",
       "      <td>softmax</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>(224, 224, 3)</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>adam</td>\n",
       "      <td>0.992512</td>\n",
       "      <td>0.985000</td>\n",
       "      <td>0.082657</td>\n",
       "      <td>0.9874</td>\n",
       "      <td>0.172555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>inception</td>\n",
       "      <td>softmax</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>(224, 224, 3)</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.997637</td>\n",
       "      <td>0.984848</td>\n",
       "      <td>0.039962</td>\n",
       "      <td>0.9859</td>\n",
       "      <td>0.130673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>inception</td>\n",
       "      <td>softmax</td>\n",
       "      <td>cifar10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>(224, 224, 3)</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.713463</td>\n",
       "      <td>0.628545</td>\n",
       "      <td>2.505806</td>\n",
       "      <td>0.6306</td>\n",
       "      <td>3.394365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>inception</td>\n",
       "      <td>softmax</td>\n",
       "      <td>cifar10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>(224, 224, 3)</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>adam</td>\n",
       "      <td>0.740657</td>\n",
       "      <td>0.644485</td>\n",
       "      <td>2.209393</td>\n",
       "      <td>0.6440</td>\n",
       "      <td>3.209218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>inception</td>\n",
       "      <td>softmax</td>\n",
       "      <td>cifar10</td>\n",
       "      <td>0.2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>(224, 224, 3)</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>adagrad</td>\n",
       "      <td>0.625672</td>\n",
       "      <td>0.581394</td>\n",
       "      <td>3.241262</td>\n",
       "      <td>0.5795</td>\n",
       "      <td>3.547598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>inception</td>\n",
       "      <td>softmax</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>(224, 224, 3)</td>\n",
       "      <td>0.0100</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.981294</td>\n",
       "      <td>0.899343</td>\n",
       "      <td>0.198402</td>\n",
       "      <td>0.9159</td>\n",
       "      <td>0.651437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>inception</td>\n",
       "      <td>softmax</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>(224, 224, 3)</td>\n",
       "      <td>0.1000</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.105050</td>\n",
       "      <td>0.098030</td>\n",
       "      <td>43.033309</td>\n",
       "      <td>0.1032</td>\n",
       "      <td>43.032091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>inception</td>\n",
       "      <td>softmax</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>(224, 224, 3)</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.099925</td>\n",
       "      <td>0.098030</td>\n",
       "      <td>43.679637</td>\n",
       "      <td>0.1032</td>\n",
       "      <td>43.683261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>inception</td>\n",
       "      <td>softmax</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>(224, 224, 3)</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.987612</td>\n",
       "      <td>0.982273</td>\n",
       "      <td>0.144570</td>\n",
       "      <td>0.9842</td>\n",
       "      <td>0.152705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>inception</td>\n",
       "      <td>relu</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>(224, 224, 3)</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.109876</td>\n",
       "      <td>0.112020</td>\n",
       "      <td>33.697362</td>\n",
       "      <td>0.1135</td>\n",
       "      <td>33.720931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>inception</td>\n",
       "      <td>tanh</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>(224, 224, 3)</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.098930</td>\n",
       "      <td>0.098283</td>\n",
       "      <td>4.602556</td>\n",
       "      <td>0.0980</td>\n",
       "      <td>4.601884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>inception</td>\n",
       "      <td>sigmoid</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>(224, 224, 3)</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.995174</td>\n",
       "      <td>0.921515</td>\n",
       "      <td>0.044295</td>\n",
       "      <td>0.9257</td>\n",
       "      <td>0.749413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>inception</td>\n",
       "      <td>softmax</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.9</td>\n",
       "      <td>10.0</td>\n",
       "      <td>(224, 224, 3)</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.098930</td>\n",
       "      <td>0.098283</td>\n",
       "      <td>6.905074</td>\n",
       "      <td>0.0980</td>\n",
       "      <td>6.904525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>inception</td>\n",
       "      <td>softmax</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.5</td>\n",
       "      <td>10.0</td>\n",
       "      <td>(224, 224, 3)</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.996866</td>\n",
       "      <td>0.986111</td>\n",
       "      <td>0.034133</td>\n",
       "      <td>0.9852</td>\n",
       "      <td>0.148186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>inception</td>\n",
       "      <td>softmax</td>\n",
       "      <td>mnist</td>\n",
       "      <td>0.1</td>\n",
       "      <td>10.0</td>\n",
       "      <td>(224, 224, 3)</td>\n",
       "      <td>0.0010</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0.996368</td>\n",
       "      <td>0.987576</td>\n",
       "      <td>0.037953</td>\n",
       "      <td>0.9884</td>\n",
       "      <td>0.113462</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   model_name activation  dataset  dropout_rate  epochs     input_dims  \\\n",
       "0   inception    softmax    mnist           0.2    10.0  (224, 224, 3)   \n",
       "1   inception    softmax    mnist           0.2    10.0  (224, 224, 3)   \n",
       "2   inception    softmax    mnist           0.2    10.0  (224, 224, 3)   \n",
       "3   inception    softmax  cifar10           0.2    10.0  (224, 224, 3)   \n",
       "4   inception    softmax  cifar10           0.2    10.0  (224, 224, 3)   \n",
       "5   inception    softmax  cifar10           0.2    10.0  (224, 224, 3)   \n",
       "6   inception    softmax    mnist           0.2    10.0  (224, 224, 3)   \n",
       "7   inception    softmax    mnist           0.2    10.0  (224, 224, 3)   \n",
       "8   inception    softmax    mnist           0.2    10.0  (224, 224, 3)   \n",
       "9   inception    softmax    mnist           0.2    10.0  (224, 224, 3)   \n",
       "10  inception       relu    mnist           0.2    10.0  (224, 224, 3)   \n",
       "11  inception       tanh    mnist           0.2    10.0  (224, 224, 3)   \n",
       "12  inception    sigmoid    mnist           0.2    10.0  (224, 224, 3)   \n",
       "13  inception    softmax    mnist           0.9    10.0  (224, 224, 3)   \n",
       "14  inception    softmax    mnist           0.5    10.0  (224, 224, 3)   \n",
       "15  inception    softmax    mnist           0.1    10.0  (224, 224, 3)   \n",
       "\n",
       "    lr_rate      opt  main_accuracy  val_main_accuracy       loss  \\\n",
       "0    0.0010      sgd       0.993060           0.957121   0.067948   \n",
       "1    0.0010     adam       0.992512           0.985000   0.082657   \n",
       "2    0.0010  adagrad       0.997637           0.984848   0.039962   \n",
       "3    0.0010      sgd       0.713463           0.628545   2.505806   \n",
       "4    0.0010     adam       0.740657           0.644485   2.209393   \n",
       "5    0.0010  adagrad       0.625672           0.581394   3.241262   \n",
       "6    0.0100      sgd       0.981294           0.899343   0.198402   \n",
       "7    0.1000      sgd       0.105050           0.098030  43.033309   \n",
       "8    0.5000      sgd       0.099925           0.098030  43.679637   \n",
       "9    0.0001      sgd       0.987612           0.982273   0.144570   \n",
       "10   0.0010      sgd       0.109876           0.112020  33.697362   \n",
       "11   0.0010      sgd       0.098930           0.098283   4.602556   \n",
       "12   0.0010      sgd       0.995174           0.921515   0.044295   \n",
       "13   0.0010      sgd       0.098930           0.098283   6.905074   \n",
       "14   0.0010      sgd       0.996866           0.986111   0.034133   \n",
       "15   0.0010      sgd       0.996368           0.987576   0.037953   \n",
       "\n",
       "    test_accuracy  test_loss  \n",
       "0          0.9600   0.408434  \n",
       "1          0.9874   0.172555  \n",
       "2          0.9859   0.130673  \n",
       "3          0.6306   3.394365  \n",
       "4          0.6440   3.209218  \n",
       "5          0.5795   3.547598  \n",
       "6          0.9159   0.651437  \n",
       "7          0.1032  43.032091  \n",
       "8          0.1032  43.683261  \n",
       "9          0.9842   0.152705  \n",
       "10         0.1135  33.720931  \n",
       "11         0.0980   4.601884  \n",
       "12         0.9257   0.749413  \n",
       "13         0.0980   6.904525  \n",
       "14         0.9852   0.148186  \n",
       "15         0.9884   0.113462  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['model_name','activation','dataset','dropout_rate','epochs','input_dims','lr_rate','opt','main_accuracy','val_main_accuracy','loss','test_accuracy','test_loss']]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}