prediction_Window_9_attention_transformer_new.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import keras\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "from keras.layers.core import Dense, Activation, Flatten, Dropout, Reshape\n",
    "from keras.layers import Conv2D, LSTM, Embedding, Bidirectional, Input, merge, multiply, concatenate, add, GlobalAveragePooling1D, Layer, TimeDistributed, Conv1D, Lambda, Add\n",
    "from keras.models import Sequential\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "from keras.utils import to_categorical\n",
    "from keras import regularizers\n",
    "from keras.models import Model\n",
    "from keras import backend as K\n",
    "import tensorflow as tf\n",
    "\n",
    "from keras.models import *\n",
    "from keras.layers import *\n",
    "from keras.callbacks import *\n",
    "from keras.initializers import *\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PELM Dataset, S positive shape:  (1554, 9)\n",
      "PELM Dataset, T positive shape:  (707, 9)\n",
      "PELM Dataset, Y positive shape:  (267, 9)\n",
      "PPA Dataset, S positive shape:  (307, 9)\n",
      "PPA Dataset, T positive shape:  (68, 9)\n",
      "PPA Dataset, Y positive shape:  (51, 9)\n",
      "\n",
      "PELM Dataset, S negative shape:  (1543, 9)\n",
      "PELM Dataset, T negative shape:  (453, 9)\n",
      "PELM Dataset, Y negative shape:  (226, 9)\n",
      "PPA Dataset, S negative shape:  (307, 9)\n",
      "PPA Dataset, T negative shape:  (68, 9)\n",
      "PPA Dataset, Y negative shape:  (51, 9)\n"
     ]
    }
   ],
   "source": [
    "# Read sample from Dataset\n",
    "\n",
    "with open('dataset/PELM/fixed_sequences_length_9/Group_Phos_S_pos.fasta', 'r') as f:\n",
    "    PELM_s_positif_txt = f.readlines()\n",
    "with open('dataset/PELM/fixed_sequences_length_9/Group_Phos_T_pos.fasta', 'r') as f:\n",
    "    PELM_t_positif_txt = f.readlines()\n",
    "with open('dataset/PELM/fixed_sequences_length_9/Group_Phos_Y_pos.fasta', 'r') as f:\n",
    "    PELM_y_positif_txt = f.readlines()\n",
    "with open('dataset/PPA/fixed_sequences_length_9/S_IDS_pos.fasta', 'r') as f:\n",
    "    PPA_s_positif_txt = f.readlines()\n",
    "with open('dataset/PPA/fixed_sequences_length_9/T_IDS_pos.fasta', 'r') as f:\n",
    "    PPA_t_positif_txt = f.readlines()\n",
    "with open('dataset/PPA/fixed_sequences_length_9/Y_IDS_pos.fasta', 'r') as f:\n",
    "    PPA_y_positif_txt = f.readlines()\n",
    "\n",
    "with open('dataset/PELM/fixed_sequences_length_9/Group_Phos_S_neg.fasta', 'r') as f:\n",
    "    PELM_s_negatif_txt = f.readlines()\n",
    "with open('dataset/PELM/fixed_sequences_length_9/Group_Phos_T_neg.fasta', 'r') as f:\n",
    "    PELM_t_negatif_txt = f.readlines()\n",
    "with open('dataset/PELM/fixed_sequences_length_9/Group_Phos_Y_neg.fasta', 'r') as f:\n",
    "    PELM_y_negatif_txt = f.readlines()\n",
    "with open('dataset/PPA/fixed_sequences_length_9/S_IDS_neg.fasta', 'r') as f:\n",
    "    PPA_s_negatif_txt = f.readlines()\n",
    "with open('dataset/PPA/fixed_sequences_length_9/T_IDS_neg.fasta', 'r') as f:\n",
    "    PPA_t_negatif_txt = f.readlines()\n",
    "with open('dataset/PPA/fixed_sequences_length_9/Y_IDS_neg.fasta', 'r') as f:\n",
    "    PPA_y_negatif_txt = f.readlines()\n",
    "\n",
    "# Pick the window 9\n",
    "\n",
    "PELM_s_positif = np.array([])\n",
    "for i in range(1,len(PELM_s_positif_txt),2):\n",
    "    temp = PELM_s_positif_txt[i]\n",
    "    temp1 = temp[0:9]\n",
    "    temp2 = list(temp1)\n",
    "    PELM_s_positif = np.append(PELM_s_positif, temp2)\n",
    "print('PELM Dataset, S positive shape: ', PELM_s_positif.reshape(int(len(PELM_s_positif)/9),9).shape)\n",
    "\n",
    "PELM_t_positif = np.array([])\n",
    "for i in range(1,len(PELM_t_positif_txt),2):\n",
    "    temp = PELM_t_positif_txt[i]\n",
    "    temp1 = temp[0:9]\n",
    "    temp2 = list(temp1)\n",
    "    PELM_t_positif = np.append(PELM_t_positif, temp2)\n",
    "print('PELM Dataset, T positive shape: ', PELM_t_positif.reshape(int(len(PELM_t_positif)/9),9).shape)\n",
    "    \n",
    "PELM_y_positif = np.array([])\n",
    "for i in range(1,len(PELM_y_positif_txt),2):\n",
    "    temp = PELM_y_positif_txt[i]\n",
    "    temp1 = temp[0:9]\n",
    "    temp2 = list(temp1)\n",
    "    PELM_y_positif = np.append(PELM_y_positif, temp2)\n",
    "print('PELM Dataset, Y positive shape: ', PELM_y_positif.reshape(int(len(PELM_y_positif)/9),9).shape)\n",
    "\n",
    "PPA_s_positif = np.array([])\n",
    "for i in range(1,len(PPA_s_positif_txt),2):\n",
    "    temp = PPA_s_positif_txt[i]\n",
    "    temp1 = temp[0:9]\n",
    "    temp2 = list(temp1)\n",
    "    PPA_s_positif = np.append(PPA_s_positif, temp2)\n",
    "print('PPA Dataset, S positive shape: ', PPA_s_positif.reshape(int(len(PPA_s_positif)/9),9).shape)\n",
    "\n",
    "PPA_t_positif = np.array([])\n",
    "for i in range(1,len(PPA_t_positif_txt),2):\n",
    "    temp = PPA_t_positif_txt[i]\n",
    "    temp1 = temp[0:9]\n",
    "    temp2 = list(temp1)\n",
    "    PPA_t_positif = np.append(PPA_t_positif, temp2)\n",
    "print('PPA Dataset, T positive shape: ', PPA_t_positif.reshape(int(len(PPA_t_positif)/9),9).shape)\n",
    "    \n",
    "PPA_y_positif = np.array([])\n",
    "for i in range(1,len(PPA_y_positif_txt),2):\n",
    "    temp = PPA_y_positif_txt[i]\n",
    "    temp1 = temp[0:9]\n",
    "    temp2 = list(temp1)\n",
    "    PPA_y_positif = np.append(PPA_y_positif, temp2)\n",
    "print('PPA Dataset, Y positive shape: ', PPA_y_positif.reshape(int(len(PPA_y_positif)/9),9).shape)\n",
    "\n",
    "print()\n",
    "\n",
    "PELM_s_negatif = np.array([])\n",
    "for i in range(1,len(PELM_s_negatif_txt),2):\n",
    "    temp = PELM_s_negatif_txt[i]\n",
    "    temp1 = temp[0:9]\n",
    "    temp2 = list(temp1)\n",
    "    PELM_s_negatif = np.append(PELM_s_negatif, temp2)\n",
    "print('PELM Dataset, S negative shape: ', PELM_s_negatif.reshape(int(len(PELM_s_negatif)/9),9).shape)\n",
    "\n",
    "PELM_t_negatif = np.array([])\n",
    "for i in range(1,len(PELM_t_negatif_txt),2):\n",
    "    temp = PELM_t_negatif_txt[i]\n",
    "    temp1 = temp[0:9]\n",
    "    temp2 = list(temp1)\n",
    "    PELM_t_negatif = np.append(PELM_t_negatif, temp2)\n",
    "print('PELM Dataset, T negative shape: ', PELM_t_negatif.reshape(int(len(PELM_t_negatif)/9),9).shape)\n",
    "    \n",
    "PELM_y_negatif = np.array([])\n",
    "for i in range(1,len(PELM_y_negatif_txt),2):\n",
    "    temp = PELM_y_negatif_txt[i]\n",
    "    temp1 = temp[0:9]\n",
    "    temp2 = list(temp1)\n",
    "    PELM_y_negatif = np.append(PELM_y_negatif, temp2)\n",
    "print('PELM Dataset, Y negative shape: ', PELM_y_negatif.reshape(int(len(PELM_y_negatif)/9),9).shape)\n",
    "\n",
    "PPA_s_negatif = np.array([])\n",
    "for i in range(1,len(PPA_s_negatif_txt),2):\n",
    "    temp = PPA_s_negatif_txt[i]\n",
    "    temp1 = temp[0:9]\n",
    "    temp2 = list(temp1)\n",
    "    PPA_s_negatif = np.append(PPA_s_negatif, temp2)\n",
    "print('PPA Dataset, S negative shape: ', PPA_s_negatif.reshape(int(len(PPA_s_negatif)/9),9).shape)\n",
    "\n",
    "PPA_t_negatif = np.array([])\n",
    "for i in range(1,len(PPA_t_negatif_txt),2):\n",
    "    temp = PPA_t_negatif_txt[i]\n",
    "    temp1 = temp[0:9]\n",
    "    temp2 = list(temp1)\n",
    "    PPA_t_negatif = np.append(PPA_t_negatif, temp2)\n",
    "print('PPA Dataset, T negative shape: ', PPA_t_negatif.reshape(int(len(PPA_t_negatif)/9),9).shape)\n",
    "    \n",
    "PPA_y_negatif = np.array([])\n",
    "for i in range(1,len(PPA_y_negatif_txt),2):\n",
    "    temp = PPA_y_negatif_txt[i]\n",
    "    temp1 = temp[0:9]\n",
    "    temp2 = list(temp1)\n",
    "    PPA_y_negatif = np.append(PPA_y_negatif, temp2)\n",
    "print('PPA Dataset, Y negative shape: ', PPA_y_negatif.reshape(int(len(PPA_y_negatif)/9),9).shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Positive Dataset shape:  (1554, 9)\n",
      "Positive Label shape:  (1554, 1)\n",
      "Negative Dataset shape:  (1543, 9)\n",
      "Negative Label shape:  (1543, 1)\n"
     ]
    }
   ],
   "source": [
    "# Choose Dataset to train, make sure correspond with negative dataset\n",
    "\n",
    "dataset_pos = PELM_s_positif\n",
    "dataset_neg = PELM_s_negatif\n",
    "string_name = 'PELM_s'\n",
    "\n",
    "# Expand dimension, Reshape and Create Label\n",
    "\n",
    "sequenceLP = int(len(dataset_pos)/9)\n",
    "dataset_pos = np.expand_dims(dataset_pos, axis=0)\n",
    "dataset_pos = dataset_pos.reshape(sequenceLP,9)\n",
    "label_pos = np.ones((sequenceLP,), dtype=int)\n",
    "label_pos = np.expand_dims(label_pos, axis=0)\n",
    "label_pos = label_pos.reshape(sequenceLP,1)\n",
    "\n",
    "sequenceLN = int(len(dataset_neg)/9)\n",
    "dataset_neg = np.expand_dims(dataset_neg, axis=0)\n",
    "dataset_neg = dataset_neg.reshape(sequenceLN,9)\n",
    "label_neg = np.zeros((sequenceLN,), dtype=int)\n",
    "label_neg = np.expand_dims(label_neg, axis=0)\n",
    "label_neg = label_neg.reshape(sequenceLN,1)\n",
    "\n",
    "# Validate\n",
    "\n",
    "print('Positive Dataset shape: ', dataset_pos.shape)\n",
    "print('Positive Label shape: ', label_pos.shape)\n",
    "print('Negative Dataset shape: ', dataset_neg.shape)\n",
    "print('Negative Label shape: ', label_neg.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "main X shape:  (3097, 9)\n",
      "main Y shape:  (3097, 2)\n",
      "train X shape:  (2477, 9)\n",
      "train Y shape:  (2477, 2)\n",
      "valid X shape:  (620, 9)\n",
      "valid Y shape:  (620, 2)\n"
     ]
    }
   ],
   "source": [
    "# Dataset preparation\n",
    "\n",
    "dataset_X = np.concatenate((dataset_pos, dataset_neg), axis=0, out=None)\n",
    "dataset_Y = np.concatenate((label_pos, label_neg), axis=0, out=None)\n",
    "\n",
    "# Tokenizing, Unique character got its own number\n",
    "\n",
    "asam = ['A','R','N','D','C','Q','E','G','H','I','L','K','M','F','P','S','T','W','Y','V']\n",
    "tokenizer = Tokenizer(char_level=True)\n",
    "tokenizer.fit_on_texts(asam)\n",
    "dataset_X_token = []\n",
    "for i in range(len(dataset_X)):\n",
    "    temp = tokenizer.texts_to_sequences(dataset_X[i])\n",
    "    dataset_X_token = np.append(dataset_X_token, temp)\n",
    "\n",
    "dataset_X_token = dataset_X_token-1\n",
    "dataset_X_token = dataset_X_token.reshape(len(dataset_X),9)\n",
    "\n",
    "# Onehot\n",
    "\n",
    "dataset_X_token_onehot = to_categorical(dataset_X_token)\n",
    "dataset_X_token_onehot = np.expand_dims(dataset_X_token_onehot, axis=3)\n",
    "dataset_X_token_onehot = dataset_X_token_onehot.reshape(len(dataset_X),9,20,1)\n",
    "\n",
    "dataset_Y_onehot = to_categorical(dataset_Y)\n",
    "\n",
    "# Shuffle Dataset, devide\n",
    "\n",
    "main_X, main_Y = shuffle(dataset_X_token, dataset_Y_onehot, random_state=13)\n",
    "train_X, valid_X, train_Y, valid_Y = train_test_split(dataset_X_token, dataset_Y_onehot, \n",
    "                                                              test_size=0.2, random_state=13)\n",
    "\n",
    "# Validation\n",
    "\n",
    "print('main X shape: ', main_X.shape)\n",
    "print('main Y shape: ', main_Y.shape)\n",
    "print('train X shape: ', train_X.shape)\n",
    "print('train Y shape: ', train_Y.shape)\n",
    "print('valid X shape: ', valid_X.shape)\n",
    "print('valid Y shape: ', valid_Y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 9)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 9, 8)         160         input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   (None, 9)            648         embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "reshape_1 (Reshape)             (None, 9, 1)         0           lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 9, 8)         0           reshape_1[0][0]                  \n",
      "                                                                 reshape_1[0][0]                  \n",
      "                                                                 reshape_1[0][0]                  \n",
      "                                                                 reshape_1[0][0]                  \n",
      "                                                                 reshape_1[0][0]                  \n",
      "                                                                 reshape_1[0][0]                  \n",
      "                                                                 reshape_1[0][0]                  \n",
      "                                                                 reshape_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_1 (Multiply)           (None, 9, 8)         0           embedding_1[0][0]                \n",
      "                                                                 concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                   (None, 9, 20)        2320        multiply_1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lstm_3 (LSTM)                   (None, 9, 20)        3280        lstm_2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "lstm_4 (LSTM)                   (None, 9, 20)        3280        lstm_3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "lstm_5 (LSTM)                   (None, 20)           3280        lstm_4[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 2)            42          lstm_5[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 13,010\n",
      "Trainable params: 13,010\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#LSTM\n",
    "\n",
    "epochs = 300\n",
    "inp = Input(shape=(9,))\n",
    "emb = Embedding(20, 8, input_length=9)(inp)\n",
    "\n",
    "att = LSTM(9, activation = 'softmax')(emb)\n",
    "att = Reshape(target_shape=(9,1))(att)\n",
    "att = concatenate([att,att,att,att,att,att,att,att], axis=2)\n",
    "emb = multiply([emb, att])\n",
    "\n",
    "i = LSTM(20, return_sequences=True)(emb)\n",
    "i = LSTM(20, return_sequences=True)(i)\n",
    "i = LSTM(20, return_sequences=True)(i)\n",
    "i = LSTM(20, return_sequences=False)(i)\n",
    "\n",
    "out = Dense(2, activation='softmax')(i)\n",
    "\n",
    "model_LSTM = Model(inputs=inp, outputs=out)\n",
    "model_LSTM.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy'])\n",
    "\n",
    "model_LSTM.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train model LSTM\n",
    "\n",
    "model_LSTM_train = model_LSTM.fit(train_X, train_Y, epochs=epochs, batch_size=32, validation_data=(valid_X, valid_Y))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot the training accuracy model LSTM\n",
    "\n",
    "accuracy = model_LSTM_train.history['acc']\n",
    "val_accuracy = model_LSTM_train.history['val_acc']\n",
    "loss = model_LSTM_train.history['loss']\n",
    "val_loss = model_LSTM_train.history['val_loss']\n",
    "epochs = range(len(accuracy))\n",
    "plt.plot(epochs, accuracy, 'bo', label='Training accuracy')\n",
    "plt.plot(epochs, val_accuracy, 'b', label='Validation accuracy')\n",
    "plt.title('LSTM Training and validation accuracy')\n",
    "plt.legend()\n",
    "plt.figure()\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('LSTM Training and validation loss')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "y_pred = np.argmax(model_LSTM.predict(valid_X), axis=1)\n",
    "y_true = np.argmax(valid_Y, axis = 1)\n",
    "\n",
    "tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()\n",
    "f1 = f1_score(y_true, y_pred, labels=None, pos_label=1, average='binary', sample_weight=None)\n",
    "auc = roc_auc_score(y_true, y_pred, average='macro', sample_weight=None, max_fpr=None)\n",
    "sensi = tp/(tp+fn)\n",
    "specificity = tn/(tn+fp)\n",
    "accu = (tn + tp)/(tn + tp + fn + fp)\n",
    "mcc = ((tp*tn)-(fp*fn))/np.sqrt((tp+fp)*(tp+fn)*(tn+fp)*(tn+fn))\n",
    "print('{} Result'.format(string_name))\n",
    "print('Accuracy :', accu)\n",
    "print('AUC :', auc)\n",
    "print('Sensitivity :', sensi)\n",
    "print('Specificity :', specificity)\n",
    "print('F1 :', f1)\n",
    "print('MCC :', mcc)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, 9)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 9, 8)         160         input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 9)            1296        embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "reshape_2 (Reshape)             (None, 9, 1)         0           bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 9, 8)         0           reshape_2[0][0]                  \n",
      "                                                                 reshape_2[0][0]                  \n",
      "                                                                 reshape_2[0][0]                  \n",
      "                                                                 reshape_2[0][0]                  \n",
      "                                                                 reshape_2[0][0]                  \n",
      "                                                                 reshape_2[0][0]                  \n",
      "                                                                 reshape_2[0][0]                  \n",
      "                                                                 reshape_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "multiply_2 (Multiply)           (None, 9, 8)         0           embedding_2[0][0]                \n",
      "                                                                 concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) (None, 9, 40)        4640        multiply_2[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 9, 40)        0           bidirectional_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_3 (Bidirectional) (None, 9, 40)        9760        dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 9, 40)        0           bidirectional_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_4 (Bidirectional) (None, 40)           9760        dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 2)            82          bidirectional_4[0][0]            \n",
      "==================================================================================================\n",
      "Total params: 25,698\n",
      "Trainable params: 25,698\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Bidirectional LSTM\n",
    "\n",
    "epochs = 300\n",
    "inp = Input(shape=(9,))\n",
    "emb = Embedding(20, 8, input_length=9)(inp)\n",
    "\n",
    "att = Bidirectional(LSTM(9, activation = 'softmax'), merge_mode='ave', weights=None)(emb)\n",
    "att = Reshape(target_shape=(9,1))(att)\n",
    "att = concatenate([att,att,att,att,att,att,att,att], axis=2)\n",
    "emb = multiply([emb, att])\n",
    "\n",
    "i = Bidirectional(LSTM(20, return_sequences=True), merge_mode='concat', weights=None)(emb)\n",
    "i = Dropout(0.6, noise_shape=None, seed=None)(i)\n",
    "i = Bidirectional(LSTM(20, return_sequences=True), merge_mode='concat', weights=None)(i)\n",
    "i = Dropout(0.6, noise_shape=None, seed=None)(i)\n",
    "i = Bidirectional(LSTM(20, return_sequences=False), merge_mode='concat', weights=None)(i)\n",
    "\n",
    "out = Dense(2, activation='softmax')(i)\n",
    "\n",
    "model_BLSTM = Model(inputs=inp, outputs=out)\n",
    "model_BLSTM.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy'])\n",
    "\n",
    "model_BLSTM.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train model Bi LSTM\n",
    "\n",
    "model_BLSTM_train = model_BLSTM.fit(train_X, train_Y, epochs=epochs, batch_size=32, validation_data=(valid_X, valid_Y))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot the training accuracy model Bi LSTM\n",
    "\n",
    "accuracy = model_BLSTM_train.history['acc']\n",
    "val_accuracy = model_BLSTM_train.history['val_acc']\n",
    "loss = model_BLSTM_train.history['loss']\n",
    "val_loss = model_BLSTM_train.history['val_loss']\n",
    "epochs = range(len(accuracy))\n",
    "plt.plot(epochs, accuracy, 'bo', label='Training accuracy')\n",
    "plt.plot(epochs, val_accuracy, 'b', label='Validation accuracy')\n",
    "plt.title('BSTM Training and validation accuracy')\n",
    "plt.legend()\n",
    "plt.figure()\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('BLSTM Training and validation loss')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "y_pred = np.argmax(model_BLSTM.predict(valid_X), axis=1)\n",
    "y_true = np.argmax(valid_Y, axis = 1)\n",
    "\n",
    "tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()\n",
    "f1 = f1_score(y_true, y_pred, labels=None, pos_label=1, average='binary', sample_weight=None)\n",
    "auc = roc_auc_score(y_true, y_pred, average='macro', sample_weight=None, max_fpr=None)\n",
    "sensi = tp/(tp+fn)\n",
    "specificity = tn/(tn+fp)\n",
    "accu = (tn + tp)/(tn + tp + fn + fp)\n",
    "mcc = ((tp*tn)-(fp*fn))/np.sqrt((tp+fp)*(tp+fn)*(tn+fp)*(tn+fn))\n",
    "print('{} Result'.format(string_name))\n",
    "print('Accuracy :', accu)\n",
    "print('AUC :', auc)\n",
    "print('Sensitivity :', sensi)\n",
    "print('Specificity :', specificity)\n",
    "print('F1 :', f1)\n",
    "print('MCC :', mcc)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_3 (InputLayer)            (None, 9)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_3 (Embedding)         (None, 9, 8)         160         input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 9, 1)         9           embedding_3[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 9, 8)         0           dense_3[0][0]                    \n",
      "                                                                 dense_3[0][0]                    \n",
      "                                                                 dense_3[0][0]                    \n",
      "                                                                 dense_3[0][0]                    \n",
      "                                                                 dense_3[0][0]                    \n",
      "                                                                 dense_3[0][0]                    \n",
      "                                                                 dense_3[0][0]                    \n",
      "                                                                 dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "multiply_3 (Multiply)           (None, 9, 8)         0           embedding_3[0][0]                \n",
      "                                                                 concatenate_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 9, 20)        180         multiply_3[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 9, 20)        420         dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 9, 20)        420         dense_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 9, 20)        420         dense_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 9, 20)        420         dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, 9, 20)        420         dense_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_10 (Dense)                (None, 9, 20)        420         dense_9[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_11 (Dense)                (None, 9, 20)        420         dense_10[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_12 (Dense)                (None, 9, 20)        420         dense_11[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_13 (Dense)                (None, 9, 20)        420         dense_12[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_14 (Dense)                (None, 9, 20)        420         dense_13[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (None, 9, 20)        420         dense_14[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_16 (Dense)                (None, 9, 20)        420         dense_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_17 (Dense)                (None, 9, 20)        420         dense_16[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_18 (Dense)                (None, 9, 20)        420         dense_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_19 (Dense)                (None, 9, 20)        420         dense_18[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_20 (Dense)                (None, 9, 20)        420         dense_19[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 180)          0           dense_20[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_21 (Dense)                (None, 2)            362         flatten_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 7,431\n",
      "Trainable params: 7,431\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#Dense\n",
    "\n",
    "epochs = 300\n",
    "inp = Input(shape=(9,))\n",
    "emb = Embedding(20, 8, input_length=9)(inp)\n",
    "\n",
    "att = Dense(1, activation = 'softmax')(emb)\n",
    "att = concatenate([att,att,att,att,att,att,att,att], axis=2)\n",
    "emb = multiply([emb, att])\n",
    "\n",
    "i = Dense(20)(emb)\n",
    "i = Dense(20)(i)\n",
    "i = Dense(20)(i)\n",
    "i = Dense(20)(i)\n",
    "i = Dense(20)(i)\n",
    "i = Dense(20)(i)\n",
    "i = Dense(20)(i)\n",
    "i = Dense(20)(i)\n",
    "i = Dense(20)(i)\n",
    "i = Dense(20)(i)\n",
    "i = Dense(20)(i)\n",
    "i = Dense(20)(i)\n",
    "i = Dense(20)(i)\n",
    "i = Dense(20)(i)\n",
    "i = Dense(20)(i)\n",
    "i = Dense(20)(i)\n",
    "i = Dense(20)(i)\n",
    "i = Flatten()(i)\n",
    "\n",
    "out = Dense(2, activation='softmax')(i)\n",
    "\n",
    "model_Dense = Model(inputs=inp, outputs=out)\n",
    "model_Dense.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy'])\n",
    "\n",
    "model_Dense.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 2477 samples, validate on 620 samples\n",
      "Epoch 1/300\n",
      "2477/2477 [==============================] - 7s 3ms/step - loss: 0.4267 - acc: 0.7909 - val_loss: 0.2572 - val_acc: 0.8823\n",
      "Epoch 2/300\n",
      "2477/2477 [==============================] - 1s 227us/step - loss: 0.2560 - acc: 0.8946 - val_loss: 0.2259 - val_acc: 0.9177\n",
      "Epoch 3/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2390 - acc: 0.9067 - val_loss: 0.2249 - val_acc: 0.9145\n",
      "Epoch 4/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2438 - acc: 0.9019 - val_loss: 0.2254 - val_acc: 0.9194\n",
      "Epoch 5/300\n",
      "2477/2477 [==============================] - 1s 228us/step - loss: 0.2364 - acc: 0.9055 - val_loss: 0.2126 - val_acc: 0.9210\n",
      "Epoch 6/300\n",
      "2477/2477 [==============================] - 1s 229us/step - loss: 0.2277 - acc: 0.9112 - val_loss: 0.2120 - val_acc: 0.9242\n",
      "Epoch 7/300\n",
      "2477/2477 [==============================] - 1s 228us/step - loss: 0.2263 - acc: 0.9043 - val_loss: 0.2197 - val_acc: 0.9161\n",
      "Epoch 8/300\n",
      "2477/2477 [==============================] - 1s 246us/step - loss: 0.2182 - acc: 0.9124 - val_loss: 0.2171 - val_acc: 0.9177\n",
      "Epoch 9/300\n",
      "2477/2477 [==============================] - 1s 240us/step - loss: 0.2224 - acc: 0.9104 - val_loss: 0.2080 - val_acc: 0.9226\n",
      "Epoch 10/300\n",
      "2477/2477 [==============================] - 1s 238us/step - loss: 0.2330 - acc: 0.9092 - val_loss: 0.2023 - val_acc: 0.9274\n",
      "Epoch 11/300\n",
      "2477/2477 [==============================] - 1s 238us/step - loss: 0.2267 - acc: 0.9096 - val_loss: 0.2153 - val_acc: 0.9194\n",
      "Epoch 12/300\n",
      "2477/2477 [==============================] - 1s 229us/step - loss: 0.2296 - acc: 0.9063 - val_loss: 0.2468 - val_acc: 0.9000\n",
      "Epoch 13/300\n",
      "2477/2477 [==============================] - 1s 257us/step - loss: 0.2229 - acc: 0.9124 - val_loss: 0.2115 - val_acc: 0.9129\n",
      "Epoch 14/300\n",
      "2477/2477 [==============================] - 1s 246us/step - loss: 0.2264 - acc: 0.9088 - val_loss: 0.2078 - val_acc: 0.9194\n",
      "Epoch 15/300\n",
      "2477/2477 [==============================] - 1s 242us/step - loss: 0.2176 - acc: 0.9184 - val_loss: 0.2013 - val_acc: 0.9210\n",
      "Epoch 16/300\n",
      "2477/2477 [==============================] - 1s 240us/step - loss: 0.2219 - acc: 0.9100 - val_loss: 0.2018 - val_acc: 0.9210\n",
      "Epoch 17/300\n",
      "2477/2477 [==============================] - 1s 240us/step - loss: 0.2291 - acc: 0.9080 - val_loss: 0.1984 - val_acc: 0.9226\n",
      "Epoch 18/300\n",
      "2477/2477 [==============================] - 1s 238us/step - loss: 0.2157 - acc: 0.9152 - val_loss: 0.2070 - val_acc: 0.9177\n",
      "Epoch 19/300\n",
      "2477/2477 [==============================] - 1s 233us/step - loss: 0.2238 - acc: 0.9140 - val_loss: 0.2249 - val_acc: 0.9290\n",
      "Epoch 20/300\n",
      "2477/2477 [==============================] - 1s 236us/step - loss: 0.2225 - acc: 0.9116 - val_loss: 0.2088 - val_acc: 0.9145\n",
      "Epoch 21/300\n",
      "2477/2477 [==============================] - 1s 239us/step - loss: 0.2221 - acc: 0.9088 - val_loss: 0.2111 - val_acc: 0.9194\n",
      "Epoch 22/300\n",
      "2477/2477 [==============================] - 1s 239us/step - loss: 0.2241 - acc: 0.9096 - val_loss: 0.2094 - val_acc: 0.9210\n",
      "Epoch 23/300\n",
      "2477/2477 [==============================] - 1s 238us/step - loss: 0.2224 - acc: 0.9108 - val_loss: 0.2103 - val_acc: 0.9258\n",
      "Epoch 24/300\n",
      "2477/2477 [==============================] - 1s 237us/step - loss: 0.2193 - acc: 0.9160 - val_loss: 0.2354 - val_acc: 0.9113\n",
      "Epoch 25/300\n",
      "2477/2477 [==============================] - 1s 233us/step - loss: 0.2225 - acc: 0.9128 - val_loss: 0.2082 - val_acc: 0.9210\n",
      "Epoch 26/300\n",
      "2477/2477 [==============================] - 1s 235us/step - loss: 0.2217 - acc: 0.9164 - val_loss: 0.2159 - val_acc: 0.9177\n",
      "Epoch 27/300\n",
      "2477/2477 [==============================] - 1s 233us/step - loss: 0.2210 - acc: 0.9092 - val_loss: 0.2095 - val_acc: 0.9258\n",
      "Epoch 28/300\n",
      "2477/2477 [==============================] - 1s 247us/step - loss: 0.2269 - acc: 0.9116 - val_loss: 0.2053 - val_acc: 0.9242\n",
      "Epoch 29/300\n",
      "2477/2477 [==============================] - 1s 238us/step - loss: 0.2162 - acc: 0.9124 - val_loss: 0.2075 - val_acc: 0.9258\n",
      "Epoch 30/300\n",
      "2477/2477 [==============================] - 1s 238us/step - loss: 0.2221 - acc: 0.9120 - val_loss: 0.2094 - val_acc: 0.9242\n",
      "Epoch 31/300\n",
      "2477/2477 [==============================] - 1s 234us/step - loss: 0.2220 - acc: 0.9100 - val_loss: 0.2310 - val_acc: 0.9065\n",
      "Epoch 32/300\n",
      "2477/2477 [==============================] - 1s 236us/step - loss: 0.2145 - acc: 0.9132 - val_loss: 0.2264 - val_acc: 0.8968\n",
      "Epoch 33/300\n",
      "2477/2477 [==============================] - 1s 237us/step - loss: 0.2157 - acc: 0.9156 - val_loss: 0.2038 - val_acc: 0.9226\n",
      "Epoch 34/300\n",
      "2477/2477 [==============================] - 1s 237us/step - loss: 0.2176 - acc: 0.9116 - val_loss: 0.2009 - val_acc: 0.9194\n",
      "Epoch 35/300\n",
      "2477/2477 [==============================] - 1s 239us/step - loss: 0.2164 - acc: 0.9156 - val_loss: 0.2096 - val_acc: 0.9210\n",
      "Epoch 36/300\n",
      "2477/2477 [==============================] - 1s 226us/step - loss: 0.2198 - acc: 0.9120 - val_loss: 0.2020 - val_acc: 0.9306\n",
      "Epoch 37/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2149 - acc: 0.9193 - val_loss: 0.2053 - val_acc: 0.9210\n",
      "Epoch 38/300\n",
      "2477/2477 [==============================] - 1s 226us/step - loss: 0.2152 - acc: 0.9140 - val_loss: 0.2004 - val_acc: 0.9242\n",
      "Epoch 39/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2148 - acc: 0.9148 - val_loss: 0.2060 - val_acc: 0.9161\n",
      "Epoch 40/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2187 - acc: 0.9096 - val_loss: 0.2082 - val_acc: 0.9161\n",
      "Epoch 41/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2184 - acc: 0.9112 - val_loss: 0.2013 - val_acc: 0.9258\n",
      "Epoch 42/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2157 - acc: 0.9144 - val_loss: 0.2133 - val_acc: 0.9274\n",
      "Epoch 43/300\n",
      "2477/2477 [==============================] - 1s 226us/step - loss: 0.2215 - acc: 0.9160 - val_loss: 0.2030 - val_acc: 0.9258\n",
      "Epoch 44/300\n",
      "2477/2477 [==============================] - 1s 225us/step - loss: 0.2172 - acc: 0.9120 - val_loss: 0.2023 - val_acc: 0.9210\n",
      "Epoch 45/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2148 - acc: 0.9193 - val_loss: 0.2132 - val_acc: 0.9258\n",
      "Epoch 46/300\n",
      "2477/2477 [==============================] - 1s 226us/step - loss: 0.2199 - acc: 0.9112 - val_loss: 0.2641 - val_acc: 0.9032\n",
      "Epoch 47/300\n",
      "2477/2477 [==============================] - 1s 226us/step - loss: 0.2210 - acc: 0.9112 - val_loss: 0.2012 - val_acc: 0.9274\n",
      "Epoch 48/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2172 - acc: 0.9197 - val_loss: 0.2144 - val_acc: 0.9242\n",
      "Epoch 49/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2176 - acc: 0.9152 - val_loss: 0.2140 - val_acc: 0.9242\n",
      "Epoch 50/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2257 - acc: 0.9168 - val_loss: 0.2079 - val_acc: 0.9258\n",
      "Epoch 51/300\n",
      "2477/2477 [==============================] - 1s 223us/step - loss: 0.2174 - acc: 0.9168 - val_loss: 0.2121 - val_acc: 0.9226\n",
      "Epoch 52/300\n",
      "2477/2477 [==============================] - 1s 225us/step - loss: 0.2155 - acc: 0.9140 - val_loss: 0.2140 - val_acc: 0.9177\n",
      "Epoch 53/300\n",
      "2477/2477 [==============================] - 1s 225us/step - loss: 0.2136 - acc: 0.9116 - val_loss: 0.2133 - val_acc: 0.9274\n",
      "Epoch 54/300\n",
      "2477/2477 [==============================] - 1s 226us/step - loss: 0.2164 - acc: 0.9144 - val_loss: 0.2218 - val_acc: 0.9129\n",
      "Epoch 55/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2162 - acc: 0.9180 - val_loss: 0.2003 - val_acc: 0.9258\n",
      "Epoch 56/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2186 - acc: 0.9156 - val_loss: 0.2103 - val_acc: 0.9258\n",
      "Epoch 57/300\n",
      "2477/2477 [==============================] - 1s 241us/step - loss: 0.2174 - acc: 0.9193 - val_loss: 0.2099 - val_acc: 0.9226\n",
      "Epoch 58/300\n",
      "2477/2477 [==============================] - 1s 226us/step - loss: 0.2161 - acc: 0.9152 - val_loss: 0.2052 - val_acc: 0.9210\n",
      "Epoch 59/300\n",
      "2477/2477 [==============================] - 1s 223us/step - loss: 0.2227 - acc: 0.9144 - val_loss: 0.2125 - val_acc: 0.9258\n",
      "Epoch 60/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2208 - acc: 0.9128 - val_loss: 0.2048 - val_acc: 0.9242\n",
      "Epoch 61/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2244 - acc: 0.9116 - val_loss: 0.1997 - val_acc: 0.9290\n",
      "Epoch 62/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2179 - acc: 0.9184 - val_loss: 0.1992 - val_acc: 0.9258\n",
      "Epoch 63/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2173 - acc: 0.9152 - val_loss: 0.2147 - val_acc: 0.9290\n",
      "Epoch 64/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2156 - acc: 0.9144 - val_loss: 0.2140 - val_acc: 0.9242\n",
      "Epoch 65/300\n",
      "2477/2477 [==============================] - 1s 240us/step - loss: 0.2158 - acc: 0.9152 - val_loss: 0.2288 - val_acc: 0.9113\n",
      "Epoch 66/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2190 - acc: 0.9116 - val_loss: 0.2061 - val_acc: 0.9290\n",
      "Epoch 67/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2185 - acc: 0.9189 - val_loss: 0.2314 - val_acc: 0.9226\n",
      "Epoch 68/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2151 - acc: 0.9172 - val_loss: 0.2008 - val_acc: 0.9242\n",
      "Epoch 69/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2191 - acc: 0.9148 - val_loss: 0.2039 - val_acc: 0.9210\n",
      "Epoch 70/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2155 - acc: 0.9132 - val_loss: 0.2044 - val_acc: 0.9210\n",
      "Epoch 71/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2202 - acc: 0.9128 - val_loss: 0.2217 - val_acc: 0.9194\n",
      "Epoch 72/300\n",
      "2477/2477 [==============================] - 1s 218us/step - loss: 0.2196 - acc: 0.9156 - val_loss: 0.2028 - val_acc: 0.9242\n",
      "Epoch 73/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2178 - acc: 0.9144 - val_loss: 0.2095 - val_acc: 0.9161\n",
      "Epoch 74/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2174 - acc: 0.9144 - val_loss: 0.2035 - val_acc: 0.9290\n",
      "Epoch 75/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2170 - acc: 0.9100 - val_loss: 0.2072 - val_acc: 0.9258\n",
      "Epoch 76/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2172 - acc: 0.9156 - val_loss: 0.2136 - val_acc: 0.9177\n",
      "Epoch 77/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2120 - acc: 0.9136 - val_loss: 0.2017 - val_acc: 0.9242\n",
      "Epoch 78/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2126 - acc: 0.9116 - val_loss: 0.2106 - val_acc: 0.9274\n",
      "Epoch 79/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2164 - acc: 0.9197 - val_loss: 0.2072 - val_acc: 0.9194\n",
      "Epoch 80/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2152 - acc: 0.9116 - val_loss: 0.2169 - val_acc: 0.9177\n",
      "Epoch 81/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2127 - acc: 0.9180 - val_loss: 0.2114 - val_acc: 0.9194\n",
      "Epoch 82/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2161 - acc: 0.9096 - val_loss: 0.2098 - val_acc: 0.9210\n",
      "Epoch 83/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2140 - acc: 0.9156 - val_loss: 0.1994 - val_acc: 0.9258\n",
      "Epoch 84/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2148 - acc: 0.9120 - val_loss: 0.2021 - val_acc: 0.9258\n",
      "Epoch 85/300\n",
      "2477/2477 [==============================] - 1s 232us/step - loss: 0.2139 - acc: 0.9180 - val_loss: 0.2041 - val_acc: 0.9194\n",
      "Epoch 86/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2157 - acc: 0.9116 - val_loss: 0.2087 - val_acc: 0.9210\n",
      "Epoch 87/300\n",
      "2477/2477 [==============================] - 1s 225us/step - loss: 0.2149 - acc: 0.9148 - val_loss: 0.2018 - val_acc: 0.9210\n",
      "Epoch 88/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2166 - acc: 0.9156 - val_loss: 0.2000 - val_acc: 0.9226\n",
      "Epoch 89/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2159 - acc: 0.9168 - val_loss: 0.1977 - val_acc: 0.9306\n",
      "Epoch 90/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2214 - acc: 0.9071 - val_loss: 0.2222 - val_acc: 0.9177\n",
      "Epoch 91/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2149 - acc: 0.9156 - val_loss: 0.1975 - val_acc: 0.9290\n",
      "Epoch 92/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2173 - acc: 0.9124 - val_loss: 0.2029 - val_acc: 0.9258\n",
      "Epoch 93/300\n",
      "2477/2477 [==============================] - 1s 235us/step - loss: 0.2134 - acc: 0.9164 - val_loss: 0.2009 - val_acc: 0.9258\n",
      "Epoch 94/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2133 - acc: 0.9164 - val_loss: 0.2043 - val_acc: 0.9226\n",
      "Epoch 95/300\n",
      "2477/2477 [==============================] - 1s 223us/step - loss: 0.2163 - acc: 0.9144 - val_loss: 0.2050 - val_acc: 0.9290\n",
      "Epoch 96/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2158 - acc: 0.9144 - val_loss: 0.2033 - val_acc: 0.9210\n",
      "Epoch 97/300\n",
      "2477/2477 [==============================] - 1s 232us/step - loss: 0.2111 - acc: 0.9144 - val_loss: 0.2054 - val_acc: 0.9242\n",
      "Epoch 98/300\n",
      "2477/2477 [==============================] - 1s 225us/step - loss: 0.2170 - acc: 0.9132 - val_loss: 0.2124 - val_acc: 0.9290\n",
      "Epoch 99/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2140 - acc: 0.9184 - val_loss: 0.2089 - val_acc: 0.9242\n",
      "Epoch 100/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2155 - acc: 0.9184 - val_loss: 0.2060 - val_acc: 0.9226\n",
      "Epoch 101/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2159 - acc: 0.9116 - val_loss: 0.2050 - val_acc: 0.9210\n",
      "Epoch 102/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2144 - acc: 0.9189 - val_loss: 0.2059 - val_acc: 0.9194\n",
      "Epoch 103/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2175 - acc: 0.9112 - val_loss: 0.2073 - val_acc: 0.9274\n",
      "Epoch 104/300\n",
      "2477/2477 [==============================] - 1s 226us/step - loss: 0.2147 - acc: 0.9164 - val_loss: 0.2050 - val_acc: 0.9274\n",
      "Epoch 105/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2167 - acc: 0.9152 - val_loss: 0.2062 - val_acc: 0.9274\n",
      "Epoch 106/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2148 - acc: 0.9180 - val_loss: 0.1996 - val_acc: 0.9274\n",
      "Epoch 107/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2143 - acc: 0.9168 - val_loss: 0.2060 - val_acc: 0.9258\n",
      "Epoch 108/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2141 - acc: 0.9176 - val_loss: 0.2224 - val_acc: 0.9226\n",
      "Epoch 109/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2167 - acc: 0.9116 - val_loss: 0.2081 - val_acc: 0.9210\n",
      "Epoch 110/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2102 - acc: 0.9136 - val_loss: 0.2047 - val_acc: 0.9258\n",
      "Epoch 111/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2157 - acc: 0.9152 - val_loss: 0.2181 - val_acc: 0.9242\n",
      "Epoch 112/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2123 - acc: 0.9156 - val_loss: 0.1997 - val_acc: 0.9274\n",
      "Epoch 113/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2184 - acc: 0.9128 - val_loss: 0.2155 - val_acc: 0.9323\n",
      "Epoch 114/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2166 - acc: 0.9136 - val_loss: 0.2015 - val_acc: 0.9194\n",
      "Epoch 115/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2184 - acc: 0.9136 - val_loss: 0.2023 - val_acc: 0.9226\n",
      "Epoch 116/300\n",
      "2477/2477 [==============================] - 1s 228us/step - loss: 0.2122 - acc: 0.9193 - val_loss: 0.2073 - val_acc: 0.9242\n",
      "Epoch 117/300\n",
      "2477/2477 [==============================] - 1s 226us/step - loss: 0.2183 - acc: 0.9144 - val_loss: 0.2004 - val_acc: 0.9226\n",
      "Epoch 118/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2148 - acc: 0.9176 - val_loss: 0.1983 - val_acc: 0.9226\n",
      "Epoch 119/300\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2157 - acc: 0.9140 - val_loss: 0.1979 - val_acc: 0.9274\n",
      "Epoch 120/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2126 - acc: 0.9148 - val_loss: 0.2009 - val_acc: 0.9242\n",
      "Epoch 121/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2146 - acc: 0.9168 - val_loss: 0.2055 - val_acc: 0.9161\n",
      "Epoch 122/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2121 - acc: 0.9140 - val_loss: 0.2106 - val_acc: 0.9290\n",
      "Epoch 123/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2136 - acc: 0.9120 - val_loss: 0.2032 - val_acc: 0.9242\n",
      "Epoch 124/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2153 - acc: 0.9152 - val_loss: 0.2011 - val_acc: 0.9258\n",
      "Epoch 125/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2138 - acc: 0.9156 - val_loss: 0.2068 - val_acc: 0.9226\n",
      "Epoch 126/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2119 - acc: 0.9176 - val_loss: 0.2061 - val_acc: 0.9194\n",
      "Epoch 127/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2159 - acc: 0.9144 - val_loss: 0.1987 - val_acc: 0.9306\n",
      "Epoch 128/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2150 - acc: 0.9193 - val_loss: 0.2063 - val_acc: 0.9194\n",
      "Epoch 129/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2147 - acc: 0.9156 - val_loss: 0.2028 - val_acc: 0.9226\n",
      "Epoch 130/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2128 - acc: 0.9124 - val_loss: 0.2109 - val_acc: 0.9210\n",
      "Epoch 131/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2130 - acc: 0.9144 - val_loss: 0.2061 - val_acc: 0.9226\n",
      "Epoch 132/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2154 - acc: 0.9132 - val_loss: 0.2067 - val_acc: 0.9210\n",
      "Epoch 133/300\n",
      "2477/2477 [==============================] - 1s 218us/step - loss: 0.2170 - acc: 0.9156 - val_loss: 0.2172 - val_acc: 0.9258\n",
      "Epoch 134/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2156 - acc: 0.9140 - val_loss: 0.2063 - val_acc: 0.9242\n",
      "Epoch 135/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2121 - acc: 0.9148 - val_loss: 0.2032 - val_acc: 0.9194\n",
      "Epoch 136/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2165 - acc: 0.9132 - val_loss: 0.2015 - val_acc: 0.9258\n",
      "Epoch 137/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2139 - acc: 0.9164 - val_loss: 0.2060 - val_acc: 0.9226\n",
      "Epoch 138/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2141 - acc: 0.9197 - val_loss: 0.2053 - val_acc: 0.9274\n",
      "Epoch 139/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2160 - acc: 0.9148 - val_loss: 0.2041 - val_acc: 0.9274\n",
      "Epoch 140/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2109 - acc: 0.9164 - val_loss: 0.2063 - val_acc: 0.9274\n",
      "Epoch 141/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2141 - acc: 0.9160 - val_loss: 0.2198 - val_acc: 0.9177\n",
      "Epoch 142/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2125 - acc: 0.9156 - val_loss: 0.2167 - val_acc: 0.9161\n",
      "Epoch 143/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2121 - acc: 0.9184 - val_loss: 0.1987 - val_acc: 0.9242\n",
      "Epoch 144/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2117 - acc: 0.9152 - val_loss: 0.2035 - val_acc: 0.9242\n",
      "Epoch 145/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2138 - acc: 0.9144 - val_loss: 0.2033 - val_acc: 0.9226\n",
      "Epoch 146/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2130 - acc: 0.9148 - val_loss: 0.2029 - val_acc: 0.9210\n",
      "Epoch 147/300\n",
      "2477/2477 [==============================] - 1s 223us/step - loss: 0.2100 - acc: 0.9184 - val_loss: 0.2046 - val_acc: 0.9210\n",
      "Epoch 148/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2106 - acc: 0.9193 - val_loss: 0.2064 - val_acc: 0.9226\n",
      "Epoch 149/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2134 - acc: 0.9180 - val_loss: 0.2004 - val_acc: 0.9274\n",
      "Epoch 150/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2099 - acc: 0.9168 - val_loss: 0.2045 - val_acc: 0.9226\n",
      "Epoch 151/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2132 - acc: 0.9168 - val_loss: 0.2036 - val_acc: 0.9226\n",
      "Epoch 152/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2204 - acc: 0.9128 - val_loss: 0.1986 - val_acc: 0.9242\n",
      "Epoch 153/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2168 - acc: 0.9136 - val_loss: 0.1983 - val_acc: 0.9258\n",
      "Epoch 154/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2126 - acc: 0.9168 - val_loss: 0.2024 - val_acc: 0.9242\n",
      "Epoch 155/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2100 - acc: 0.9160 - val_loss: 0.2063 - val_acc: 0.9258\n",
      "Epoch 156/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2187 - acc: 0.9120 - val_loss: 0.2025 - val_acc: 0.9274\n",
      "Epoch 157/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2127 - acc: 0.9168 - val_loss: 0.2041 - val_acc: 0.9194\n",
      "Epoch 158/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2123 - acc: 0.9189 - val_loss: 0.2042 - val_acc: 0.9274\n",
      "Epoch 159/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2118 - acc: 0.9193 - val_loss: 0.2209 - val_acc: 0.9210\n",
      "Epoch 160/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2133 - acc: 0.9180 - val_loss: 0.2030 - val_acc: 0.9242\n",
      "Epoch 161/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2119 - acc: 0.9156 - val_loss: 0.2015 - val_acc: 0.9274\n",
      "Epoch 162/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2130 - acc: 0.9229 - val_loss: 0.2117 - val_acc: 0.9210\n",
      "Epoch 163/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2142 - acc: 0.9176 - val_loss: 0.2090 - val_acc: 0.9258\n",
      "Epoch 164/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2125 - acc: 0.9144 - val_loss: 0.1984 - val_acc: 0.9242\n",
      "Epoch 165/300\n",
      "2477/2477 [==============================] - 1s 223us/step - loss: 0.2142 - acc: 0.9128 - val_loss: 0.2006 - val_acc: 0.9242\n",
      "Epoch 166/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2150 - acc: 0.9144 - val_loss: 0.2013 - val_acc: 0.9242\n",
      "Epoch 167/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2122 - acc: 0.9160 - val_loss: 0.2039 - val_acc: 0.9290\n",
      "Epoch 168/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2105 - acc: 0.9164 - val_loss: 0.2174 - val_acc: 0.9177\n",
      "Epoch 169/300\n",
      "2477/2477 [==============================] - 1s 223us/step - loss: 0.2132 - acc: 0.9164 - val_loss: 0.2119 - val_acc: 0.9194\n",
      "Epoch 170/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2149 - acc: 0.9144 - val_loss: 0.2040 - val_acc: 0.9242\n",
      "Epoch 171/300\n",
      "2477/2477 [==============================] - 1s 229us/step - loss: 0.2112 - acc: 0.9156 - val_loss: 0.2041 - val_acc: 0.9194\n",
      "Epoch 172/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2103 - acc: 0.9184 - val_loss: 0.2021 - val_acc: 0.9258\n",
      "Epoch 173/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2158 - acc: 0.9156 - val_loss: 0.2084 - val_acc: 0.9242\n",
      "Epoch 174/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2103 - acc: 0.9209 - val_loss: 0.1989 - val_acc: 0.9210\n",
      "Epoch 175/300\n",
      "2477/2477 [==============================] - 1s 240us/step - loss: 0.2139 - acc: 0.9140 - val_loss: 0.1993 - val_acc: 0.9226\n",
      "Epoch 176/300\n",
      "2477/2477 [==============================] - 1s 225us/step - loss: 0.2139 - acc: 0.9217 - val_loss: 0.2018 - val_acc: 0.9258\n",
      "Epoch 177/300\n",
      "2477/2477 [==============================] - 1s 223us/step - loss: 0.2135 - acc: 0.9197 - val_loss: 0.2014 - val_acc: 0.9242\n",
      "Epoch 178/300\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2477/2477 [==============================] - 1s 223us/step - loss: 0.2141 - acc: 0.9152 - val_loss: 0.2063 - val_acc: 0.9226\n",
      "Epoch 179/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2140 - acc: 0.9156 - val_loss: 0.2004 - val_acc: 0.9274\n",
      "Epoch 180/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2140 - acc: 0.9128 - val_loss: 0.1979 - val_acc: 0.9274\n",
      "Epoch 181/300\n",
      "2477/2477 [==============================] - 1s 226us/step - loss: 0.2129 - acc: 0.9193 - val_loss: 0.2061 - val_acc: 0.9274\n",
      "Epoch 182/300\n",
      "2477/2477 [==============================] - 1s 225us/step - loss: 0.2135 - acc: 0.9168 - val_loss: 0.2120 - val_acc: 0.9242\n",
      "Epoch 183/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2117 - acc: 0.9184 - val_loss: 0.1993 - val_acc: 0.9242\n",
      "Epoch 184/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2133 - acc: 0.9180 - val_loss: 0.2214 - val_acc: 0.9194\n",
      "Epoch 185/300\n",
      "2477/2477 [==============================] - 1s 239us/step - loss: 0.2132 - acc: 0.9197 - val_loss: 0.2192 - val_acc: 0.9242\n",
      "Epoch 186/300\n",
      "2477/2477 [==============================] - 1s 252us/step - loss: 0.2132 - acc: 0.9164 - val_loss: 0.2000 - val_acc: 0.9242\n",
      "Epoch 187/300\n",
      "2477/2477 [==============================] - 1s 229us/step - loss: 0.2143 - acc: 0.9168 - val_loss: 0.2063 - val_acc: 0.9177\n",
      "Epoch 188/300\n",
      "2477/2477 [==============================] - 1s 225us/step - loss: 0.2140 - acc: 0.9156 - val_loss: 0.2029 - val_acc: 0.9226\n",
      "Epoch 189/300\n",
      "2477/2477 [==============================] - 1s 254us/step - loss: 0.2112 - acc: 0.9193 - val_loss: 0.2009 - val_acc: 0.9242\n",
      "Epoch 190/300\n",
      "2477/2477 [==============================] - 1s 239us/step - loss: 0.2121 - acc: 0.9184 - val_loss: 0.1997 - val_acc: 0.9242\n",
      "Epoch 191/300\n",
      "2477/2477 [==============================] - 1s 239us/step - loss: 0.2128 - acc: 0.9160 - val_loss: 0.2118 - val_acc: 0.9145\n",
      "Epoch 192/300\n",
      "2477/2477 [==============================] - 1s 238us/step - loss: 0.2134 - acc: 0.9140 - val_loss: 0.2054 - val_acc: 0.9210\n",
      "Epoch 193/300\n",
      "2477/2477 [==============================] - 1s 229us/step - loss: 0.2108 - acc: 0.9140 - val_loss: 0.2039 - val_acc: 0.9274\n",
      "Epoch 194/300\n",
      "2477/2477 [==============================] - 1s 230us/step - loss: 0.2129 - acc: 0.9172 - val_loss: 0.2057 - val_acc: 0.9242\n",
      "Epoch 195/300\n",
      "2477/2477 [==============================] - 1s 233us/step - loss: 0.2147 - acc: 0.9132 - val_loss: 0.2080 - val_acc: 0.9226\n",
      "Epoch 196/300\n",
      "2477/2477 [==============================] - 1s 242us/step - loss: 0.2122 - acc: 0.9160 - val_loss: 0.1972 - val_acc: 0.9290\n",
      "Epoch 197/300\n",
      "2477/2477 [==============================] - 1s 242us/step - loss: 0.2131 - acc: 0.9168 - val_loss: 0.2003 - val_acc: 0.9258\n",
      "Epoch 198/300\n",
      "2477/2477 [==============================] - 1s 241us/step - loss: 0.2137 - acc: 0.9136 - val_loss: 0.2075 - val_acc: 0.9242\n",
      "Epoch 199/300\n",
      "2477/2477 [==============================] - 1s 240us/step - loss: 0.2172 - acc: 0.9128 - val_loss: 0.2045 - val_acc: 0.9242\n",
      "Epoch 200/300\n",
      "2477/2477 [==============================] - 1s 235us/step - loss: 0.2127 - acc: 0.9213 - val_loss: 0.2076 - val_acc: 0.9226\n",
      "Epoch 201/300\n",
      "2477/2477 [==============================] - 1s 240us/step - loss: 0.2117 - acc: 0.9184 - val_loss: 0.2047 - val_acc: 0.9274\n",
      "Epoch 202/300\n",
      "2477/2477 [==============================] - 1s 245us/step - loss: 0.2151 - acc: 0.9152 - val_loss: 0.2067 - val_acc: 0.9258\n",
      "Epoch 203/300\n",
      "2477/2477 [==============================] - 1s 246us/step - loss: 0.2128 - acc: 0.9168 - val_loss: 0.2042 - val_acc: 0.9258\n",
      "Epoch 204/300\n",
      "2477/2477 [==============================] - 1s 238us/step - loss: 0.2162 - acc: 0.9176 - val_loss: 0.2041 - val_acc: 0.9194\n",
      "Epoch 205/300\n",
      "2477/2477 [==============================] - 1s 255us/step - loss: 0.2090 - acc: 0.9164 - val_loss: 0.2009 - val_acc: 0.9210\n",
      "Epoch 206/300\n",
      "2477/2477 [==============================] - 1s 243us/step - loss: 0.2112 - acc: 0.9160 - val_loss: 0.1992 - val_acc: 0.9258\n",
      "Epoch 207/300\n",
      "2477/2477 [==============================] - 1s 238us/step - loss: 0.2108 - acc: 0.9176 - val_loss: 0.2244 - val_acc: 0.9274\n",
      "Epoch 208/300\n",
      "2477/2477 [==============================] - 1s 235us/step - loss: 0.2127 - acc: 0.9148 - val_loss: 0.1995 - val_acc: 0.9210\n",
      "Epoch 209/300\n",
      "2477/2477 [==============================] - 1s 233us/step - loss: 0.2089 - acc: 0.9168 - val_loss: 0.2063 - val_acc: 0.9242\n",
      "Epoch 210/300\n",
      "2477/2477 [==============================] - 1s 233us/step - loss: 0.2123 - acc: 0.9152 - val_loss: 0.2126 - val_acc: 0.9145\n",
      "Epoch 211/300\n",
      "2477/2477 [==============================] - 1s 243us/step - loss: 0.2185 - acc: 0.9124 - val_loss: 0.2096 - val_acc: 0.9242\n",
      "Epoch 212/300\n",
      "2477/2477 [==============================] - 1s 248us/step - loss: 0.2119 - acc: 0.9180 - val_loss: 0.2080 - val_acc: 0.9290\n",
      "Epoch 213/300\n",
      "2477/2477 [==============================] - 1s 232us/step - loss: 0.2118 - acc: 0.9189 - val_loss: 0.1990 - val_acc: 0.9226\n",
      "Epoch 214/300\n",
      "2477/2477 [==============================] - 1s 237us/step - loss: 0.2098 - acc: 0.9180 - val_loss: 0.2031 - val_acc: 0.9210\n",
      "Epoch 215/300\n",
      "2477/2477 [==============================] - 1s 235us/step - loss: 0.2118 - acc: 0.9184 - val_loss: 0.2048 - val_acc: 0.9258\n",
      "Epoch 216/300\n",
      "2477/2477 [==============================] - 1s 230us/step - loss: 0.2084 - acc: 0.9217 - val_loss: 0.2094 - val_acc: 0.9242\n",
      "Epoch 217/300\n",
      "2477/2477 [==============================] - 1s 239us/step - loss: 0.2132 - acc: 0.9184 - val_loss: 0.2023 - val_acc: 0.9210\n",
      "Epoch 218/300\n",
      "2477/2477 [==============================] - 1s 234us/step - loss: 0.2097 - acc: 0.9164 - val_loss: 0.2082 - val_acc: 0.9258\n",
      "Epoch 219/300\n",
      "2477/2477 [==============================] - 1s 236us/step - loss: 0.2110 - acc: 0.9160 - val_loss: 0.1979 - val_acc: 0.9274\n",
      "Epoch 220/300\n",
      "2477/2477 [==============================] - 1s 230us/step - loss: 0.2105 - acc: 0.9172 - val_loss: 0.2105 - val_acc: 0.9226\n",
      "Epoch 221/300\n",
      "2477/2477 [==============================] - 1s 232us/step - loss: 0.2089 - acc: 0.9136 - val_loss: 0.2036 - val_acc: 0.9323\n",
      "Epoch 222/300\n",
      "2477/2477 [==============================] - 1s 230us/step - loss: 0.2111 - acc: 0.9144 - val_loss: 0.2028 - val_acc: 0.9274\n",
      "Epoch 223/300\n",
      "2477/2477 [==============================] - 1s 229us/step - loss: 0.2119 - acc: 0.9160 - val_loss: 0.2018 - val_acc: 0.9258\n",
      "Epoch 224/300\n",
      "2477/2477 [==============================] - 1s 230us/step - loss: 0.2094 - acc: 0.9197 - val_loss: 0.2056 - val_acc: 0.9210\n",
      "Epoch 225/300\n",
      "2477/2477 [==============================] - 1s 229us/step - loss: 0.2116 - acc: 0.9168 - val_loss: 0.2073 - val_acc: 0.9258\n",
      "Epoch 226/300\n",
      "2477/2477 [==============================] - 1s 229us/step - loss: 0.2132 - acc: 0.9172 - val_loss: 0.2071 - val_acc: 0.9242\n",
      "Epoch 227/300\n",
      "2477/2477 [==============================] - 1s 237us/step - loss: 0.2118 - acc: 0.9164 - val_loss: 0.2024 - val_acc: 0.9258\n",
      "Epoch 228/300\n",
      "2477/2477 [==============================] - 1s 240us/step - loss: 0.2090 - acc: 0.9152 - val_loss: 0.2022 - val_acc: 0.9226\n",
      "Epoch 229/300\n",
      "2477/2477 [==============================] - 1s 237us/step - loss: 0.2112 - acc: 0.9160 - val_loss: 0.2023 - val_acc: 0.9242\n",
      "Epoch 230/300\n",
      "2477/2477 [==============================] - 1s 232us/step - loss: 0.2136 - acc: 0.9120 - val_loss: 0.2068 - val_acc: 0.9290\n",
      "Epoch 231/300\n",
      "2477/2477 [==============================] - 1s 231us/step - loss: 0.2100 - acc: 0.9197 - val_loss: 0.2009 - val_acc: 0.9258\n",
      "Epoch 232/300\n",
      "2477/2477 [==============================] - 1s 226us/step - loss: 0.2101 - acc: 0.9176 - val_loss: 0.2044 - val_acc: 0.9226\n",
      "Epoch 233/300\n",
      "2477/2477 [==============================] - 1s 231us/step - loss: 0.2114 - acc: 0.9152 - val_loss: 0.2056 - val_acc: 0.9226\n",
      "Epoch 234/300\n",
      "2477/2477 [==============================] - 1s 230us/step - loss: 0.2091 - acc: 0.9168 - val_loss: 0.2122 - val_acc: 0.9145\n",
      "Epoch 235/300\n",
      "2477/2477 [==============================] - 1s 237us/step - loss: 0.2112 - acc: 0.9201 - val_loss: 0.2084 - val_acc: 0.9242\n",
      "Epoch 236/300\n",
      "2477/2477 [==============================] - 1s 232us/step - loss: 0.2096 - acc: 0.9164 - val_loss: 0.2007 - val_acc: 0.9226\n",
      "Epoch 237/300\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2477/2477 [==============================] - 1s 223us/step - loss: 0.2120 - acc: 0.9201 - val_loss: 0.2028 - val_acc: 0.9226\n",
      "Epoch 238/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2116 - acc: 0.9176 - val_loss: 0.1974 - val_acc: 0.9226\n",
      "Epoch 239/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2081 - acc: 0.9180 - val_loss: 0.2090 - val_acc: 0.9194\n",
      "Epoch 240/300\n",
      "2477/2477 [==============================] - 1s 223us/step - loss: 0.2133 - acc: 0.9164 - val_loss: 0.2024 - val_acc: 0.9194\n",
      "Epoch 241/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2082 - acc: 0.9189 - val_loss: 0.2007 - val_acc: 0.9258\n",
      "Epoch 242/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2109 - acc: 0.9180 - val_loss: 0.2030 - val_acc: 0.9274\n",
      "Epoch 243/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2103 - acc: 0.9189 - val_loss: 0.2072 - val_acc: 0.9226\n",
      "Epoch 244/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2154 - acc: 0.9156 - val_loss: 0.2020 - val_acc: 0.9242\n",
      "Epoch 245/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2146 - acc: 0.9164 - val_loss: 0.2028 - val_acc: 0.9258\n",
      "Epoch 246/300\n",
      "2477/2477 [==============================] - 1s 227us/step - loss: 0.2122 - acc: 0.9140 - val_loss: 0.1986 - val_acc: 0.9306\n",
      "Epoch 247/300\n",
      "2477/2477 [==============================] - 1s 224us/step - loss: 0.2151 - acc: 0.9112 - val_loss: 0.2016 - val_acc: 0.9226\n",
      "Epoch 248/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2127 - acc: 0.9140 - val_loss: 0.2179 - val_acc: 0.9258\n",
      "Epoch 249/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2089 - acc: 0.9164 - val_loss: 0.1995 - val_acc: 0.9242\n",
      "Epoch 250/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2105 - acc: 0.9156 - val_loss: 0.1959 - val_acc: 0.9242\n",
      "Epoch 251/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2122 - acc: 0.9152 - val_loss: 0.2081 - val_acc: 0.9274\n",
      "Epoch 252/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2100 - acc: 0.9164 - val_loss: 0.2022 - val_acc: 0.9226\n",
      "Epoch 253/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2111 - acc: 0.9136 - val_loss: 0.2045 - val_acc: 0.9210\n",
      "Epoch 254/300\n",
      "2477/2477 [==============================] - 1s 218us/step - loss: 0.2141 - acc: 0.9152 - val_loss: 0.2007 - val_acc: 0.9258\n",
      "Epoch 255/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2087 - acc: 0.9176 - val_loss: 0.2066 - val_acc: 0.9258\n",
      "Epoch 256/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2122 - acc: 0.9176 - val_loss: 0.2064 - val_acc: 0.9226\n",
      "Epoch 257/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2191 - acc: 0.9108 - val_loss: 0.2028 - val_acc: 0.9242\n",
      "Epoch 258/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2099 - acc: 0.9197 - val_loss: 0.2018 - val_acc: 0.9258\n",
      "Epoch 259/300\n",
      "2477/2477 [==============================] - 1s 228us/step - loss: 0.2076 - acc: 0.9193 - val_loss: 0.2000 - val_acc: 0.9226\n",
      "Epoch 260/300\n",
      "2477/2477 [==============================] - 1s 223us/step - loss: 0.2095 - acc: 0.9132 - val_loss: 0.2117 - val_acc: 0.9258\n",
      "Epoch 261/300\n",
      "2477/2477 [==============================] - 1s 218us/step - loss: 0.2170 - acc: 0.9160 - val_loss: 0.2115 - val_acc: 0.9242\n",
      "Epoch 262/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2109 - acc: 0.9168 - val_loss: 0.2021 - val_acc: 0.9242\n",
      "Epoch 263/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2119 - acc: 0.9164 - val_loss: 0.2045 - val_acc: 0.9210\n",
      "Epoch 264/300\n",
      "2477/2477 [==============================] - 1s 223us/step - loss: 0.2096 - acc: 0.9172 - val_loss: 0.2028 - val_acc: 0.9210\n",
      "Epoch 265/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2118 - acc: 0.9156 - val_loss: 0.2054 - val_acc: 0.9210\n",
      "Epoch 266/300\n",
      "2477/2477 [==============================] - 1s 218us/step - loss: 0.2116 - acc: 0.9180 - val_loss: 0.2004 - val_acc: 0.9274\n",
      "Epoch 267/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2140 - acc: 0.9176 - val_loss: 0.2038 - val_acc: 0.9274\n",
      "Epoch 268/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2095 - acc: 0.9144 - val_loss: 0.2000 - val_acc: 0.9226\n",
      "Epoch 269/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2115 - acc: 0.9180 - val_loss: 0.2076 - val_acc: 0.9242\n",
      "Epoch 270/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2153 - acc: 0.9144 - val_loss: 0.2089 - val_acc: 0.9258\n",
      "Epoch 271/300\n",
      "2477/2477 [==============================] - 1s 218us/step - loss: 0.2115 - acc: 0.9164 - val_loss: 0.1960 - val_acc: 0.9274\n",
      "Epoch 272/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2096 - acc: 0.9152 - val_loss: 0.1995 - val_acc: 0.9242\n",
      "Epoch 273/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2097 - acc: 0.9160 - val_loss: 0.1968 - val_acc: 0.9226\n",
      "Epoch 274/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2093 - acc: 0.9172 - val_loss: 0.1997 - val_acc: 0.9226\n",
      "Epoch 275/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2110 - acc: 0.9164 - val_loss: 0.1986 - val_acc: 0.9226\n",
      "Epoch 276/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2096 - acc: 0.9176 - val_loss: 0.2048 - val_acc: 0.9242\n",
      "Epoch 277/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2109 - acc: 0.9164 - val_loss: 0.2013 - val_acc: 0.9306\n",
      "Epoch 278/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2103 - acc: 0.9180 - val_loss: 0.2047 - val_acc: 0.9258\n",
      "Epoch 279/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2098 - acc: 0.9164 - val_loss: 0.2077 - val_acc: 0.9210\n",
      "Epoch 280/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2104 - acc: 0.9148 - val_loss: 0.2019 - val_acc: 0.9258\n",
      "Epoch 281/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2086 - acc: 0.9176 - val_loss: 0.2055 - val_acc: 0.9242\n",
      "Epoch 282/300\n",
      "2477/2477 [==============================] - 1s 223us/step - loss: 0.2111 - acc: 0.9168 - val_loss: 0.2069 - val_acc: 0.9194\n",
      "Epoch 283/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2114 - acc: 0.9189 - val_loss: 0.2083 - val_acc: 0.9258\n",
      "Epoch 284/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2107 - acc: 0.9176 - val_loss: 0.2123 - val_acc: 0.9210\n",
      "Epoch 285/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2106 - acc: 0.9189 - val_loss: 0.1998 - val_acc: 0.9226\n",
      "Epoch 286/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2072 - acc: 0.9148 - val_loss: 0.2053 - val_acc: 0.9258\n",
      "Epoch 287/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2105 - acc: 0.9168 - val_loss: 0.1983 - val_acc: 0.9274\n",
      "Epoch 288/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2128 - acc: 0.9140 - val_loss: 0.2166 - val_acc: 0.9274\n",
      "Epoch 289/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2084 - acc: 0.9193 - val_loss: 0.1998 - val_acc: 0.9242\n",
      "Epoch 290/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2124 - acc: 0.9124 - val_loss: 0.2018 - val_acc: 0.9258\n",
      "Epoch 291/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2103 - acc: 0.9189 - val_loss: 0.1995 - val_acc: 0.9258\n",
      "Epoch 292/300\n",
      "2477/2477 [==============================] - 1s 222us/step - loss: 0.2105 - acc: 0.9156 - val_loss: 0.2031 - val_acc: 0.9258\n",
      "Epoch 293/300\n",
      "2477/2477 [==============================] - 1s 223us/step - loss: 0.2099 - acc: 0.9197 - val_loss: 0.1998 - val_acc: 0.9242\n",
      "Epoch 294/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2092 - acc: 0.9197 - val_loss: 0.2029 - val_acc: 0.9290\n",
      "Epoch 295/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2101 - acc: 0.9205 - val_loss: 0.2014 - val_acc: 0.9242\n",
      "Epoch 296/300\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2062 - acc: 0.9168 - val_loss: 0.2068 - val_acc: 0.9306\n",
      "Epoch 297/300\n",
      "2477/2477 [==============================] - 1s 219us/step - loss: 0.2155 - acc: 0.9172 - val_loss: 0.2111 - val_acc: 0.9242\n",
      "Epoch 298/300\n",
      "2477/2477 [==============================] - 1s 220us/step - loss: 0.2116 - acc: 0.9189 - val_loss: 0.1985 - val_acc: 0.9274\n",
      "Epoch 299/300\n",
      "2477/2477 [==============================] - 1s 218us/step - loss: 0.2129 - acc: 0.9120 - val_loss: 0.2052 - val_acc: 0.9274\n",
      "Epoch 300/300\n",
      "2477/2477 [==============================] - 1s 221us/step - loss: 0.2105 - acc: 0.9164 - val_loss: 0.2011 - val_acc: 0.9210\n"
     ]
    }
   ],
   "source": [
    "# Train model Dense\n",
    "\n",
    "model_Dense_train = model_Dense.fit(train_X, train_Y, epochs=epochs, batch_size=32, validation_data=(valid_X, valid_Y))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEICAYAAACzliQjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJztnXeYFUX2v9/DEIfsgImsksMgjJgwgbiIgVUxIAZM7Lqia9r9Ylizm8yrrisqGGAJawQTJvxhQCWDoAQRYQBJIqCgpPr9cbqn+965aWbuxHve5+mnu6urq091V3/6VHVVtzjnMAzDMDKDauVtgGEYhlF2mOgbhmFkECb6hmEYGYSJvmEYRgZhom8YhpFBmOgbhmFkECb6GYiIZInITyLSMp1xyxMROURE0t7/WEROFJEVofXFInJMKnGLcaynReTm4u5vGKlQvbwNMJIjIj+FVrOBX4E93vrvnHNji5Kec24PUC/dcTMB51z7dKQjIpcDFzjnjg+lfXk60jaMRJjoVwKccwWi63mSlzvn3osXX0SqO+d2l4VthpEMK48VC2veqQKIyD0iMkFExonINuACETlSRD4TkR9FZK2I/EtEanjxq4uIE5HW3voYb/tbIrJNRKaLSJuixvW2nywiS0Rki4g8KiKfiMjQOHanYuPvRGSZiGwWkX+F9s0SkYdEZJOILAf6Jzg/t4jI+Kiwx0XkQW/5chH5ysvPN54XHi+tfBE53lvOFpEXPNsWAj2j4t4qIsu9dBeKyOleeFfgMeAYr+lsY+jc3hHa//de3jeJyKsickAq56Yo59m3R0TeE5EfROR7Eflz6Dh/8c7JVhGZKSIHxmpKE5GP/evsnc9p3nF+AG4VkbYiMtU7xkbvvDUM7d/Ky+MGb/sjIlLbs7ljKN4BIrJdRHLi5ddIgnPOpko0ASuAE6PC7gF2AqehD/I6wGHA4Wht7iBgCTDci18dcEBrb30MsBHIA2oAE4AxxYi7L7ANGOhtux7YBQyNk5dUbHwNaAi0Bn7w8w4MBxYCzYEcYJoW55jHOQj4CagbSns9kOetn+bFEaAPsAPo5m07EVgRSisfON5bvh/4EGgMtAIWRcU9BzjAuybnezbs5227HPgwys4xwB3e8kmejd2B2sC/gQ9SOTdFPM8NgXXAH4FaQAOgl7ftJmAe0NbLQ3dgH+CQ6HMNfOxfZy9vu4ErgSy0PLYD+gI1vXLyCXB/KD9feuezrhf/aG/bSODe0HFuAF4p7/uwMk/lboBNRbxg8UX/gyT73Qj8z1uOJeT/CcU9HfiyGHEvBT4KbRNgLXFEP0Ubjwhtfxm40VuehjZz+dsGRAtRVNqfAed7yycDixPEfR24yltOJPorw9cC+EM4box0vwRO8ZaTif5zwF9D2xqg73GaJzs3RTzPFwIz4sT7xrc3KjwV0V+exIZB/nGBY4DvgawY8Y4GvgXEW58LnJnu+yqTJmveqTqsCq+ISAcRecOrrm8F7gKaJNj/+9DydhK/vI0X98CwHU7v0vx4iaRoY0rHAr5LYC/Af4HB3vL53rpvx6ki8rnX9PAj6mUnOlc+BySyQUSGisg8r4niR6BDiumC5q8gPefcVmAz0CwUJ6VrluQ8t0DFPRaJtiUjujzuLyITRWS1Z8OzUTascNppIALn3CdoraG3iHQBWgJvFNMmA2vTr0pEd1d8EvUsD3HONQBuQz3v0mQt6okCICJCpEhFUxIb16Ji4ZOsS+lE4EQRaYY2P/3Xs7EO8CLwN7TppRHwTop2fB/PBhE5CHgCbeLI8dL9OpRusu6la9AmIz+9+mgz0uoU7Iom0XleBRwcZ7942372bMoOhe0fFSc6f/9Ae5119WwYGmVDKxHJimPH88AFaK1konPu1zjxjBQw0a+61Ae2AD97L8J+VwbHfB3oISKniUh1tJ24aSnZOBG4VkSaeS/1/i9RZOfc92gTxLNo085Sb1MttJ15A7BHRE5F255TteFmEWkkOo5heGhbPVT4NqDPvytQT99nHdA8/EI1inHAZSLSTURqoQ+lj5xzcWtOCUh0nicBLUVkuIjUEpEGItLL2/Y0cI+IHCxKdxHZB33YfY92GMgSkWGEHlAJbPgZ2CIiLdAmJp/pwCbgr6Ivx+uIyNGh7S+gzUHnow8AowSY6FddbgAuRl+sPom+cC1VnHPrgHOBB9Gb+GBgDurhpdvGJ4D3gQXADNRbT8Z/0Tb6gqYd59yPwHXAK+jL0EHowysVbkdrHCuAtwgJknNuPvAo8IUXpz3weWjfd4GlwDoRCTfT+Pu/jTbDvOLt3xIYkqJd0cQ9z865LUA/4Cz0QbQEOM7bfB/wKnqet6IvVWt7zXZXADejL/UPicpbLG4HeqEPn0nASyEbdgOnAh1Rr38leh387SvQ6/yrc+7TIubdiMJ/OWIYacerrq8BBjnnPipve4zKi4g8j74cvqO8bans2OAsI62ISH+0p8wOtMvfLtTbNYxi4b0fGQh0LW9bqgLWvGOkm97AcrQt+zfAGfbizSguIvI3dKzAX51zK8vbnqqANe8YhmFkEObpG4ZhZBAVrk2/SZMmrnXr1uVthmEYRqVi1qxZG51zibpIAxVQ9Fu3bs3MmTPL2wzDMIxKhYgkG5UOWPOOYRhGRmGibxiGkUGY6BuGYWQQJvqGYRgZhIm+YRhGBmGibxiGkUGY6BuGYWQQJvpGhWP8eNi0qbytMIqCc/Dcc/DTT+VtiZEME32jQrF4MQweDMOGlbclRlFYuhSGDoUxY8rbEiMZJvqlxK5dsGpV8niVjV9/hdXF+WFfinz7rc43by76vhs3wtat6bXHSI3163X+9dele5wtW/Q6lxfLl8PevelJa9Mm+PHH9KRVFEz0S4lRo6BDB9i+vbwtSS//+hd06ZK+gh/NsmU63z/6j6spMHAgXHNNeu0xUsNvjluypHSP87vfwW9/W7rHiMfq1dCuHbyYyj/aUuCss+CKK9KTVlEw0S8lli9Xwd+wobwtSS9Ll6p3Ulptt4sX67xOnaLv+913sNK+uF4u+N63f/1KizlzAsegrFm6FPbsURvSwVdf6VTWmOjH4Jpr4IYbSpaGfxOUZlV0zx449FBtR+3VC0aP1vDWreH/EvwmfNAg9YqLw7p1On/iCahRA375Rdc//FBrNj/9BDNmQJs2xcu77ykma6Zp1w7+/OfIsM2bi9csVFb88AOIwOTJZXvcxx6DPn2Kv/+XX0KrVrBmTfw4vqe/YoU2AcbCOTj8cK0FF4ddu9SZ2rBBy35Z4zfXlrQ2c/75en+uX18+TcAm+jGYNg0+LeHvl/2boDR7oaxcCXPnwl//qkL7zjsa/t138M9/xt/vpZdg0iS9CYuKL/ojRsDu3YHX89FH6uV9+y188IHe/LNmFT1931NMJvpLl8J99wXrO3dqzao82khT5XPv1+GPPFK2x506Vafivov58EMta4k8XL+c790L33wTO87338MXXxS/eeTbb7XM7d1bPr27fIEuSW3GOX3oP/OMrm/dWvbvoTJO9O++G957L3GcH3+EbdtKdpx4or90KVx0UWT6mzdrz4cffoiM++uv2uYXzxvwC59fRVyypGhC7t+cY8eq554Kvuj7be7Tp+vct3HdusCusEe0cSOcfrp6nH36wMkn64MhzI4dQfPMli2Fj33TTfpA3rWr8DZf7FP19F99NbDlllsSx124EP7wh9je5cqVej1XrYJLLlG7t2zR6+m/3PRZu1bn++6bmo0At90W2DlxYur7TZoUPPj9a+NfK4DPPoOrry78bmbRIrj88shz7F/HWOXQz2v4gRCvycJPZ/r01N8J/fvfQc0gLLZ+OQR46y24557U0otmxQoYMADOOCO4J9evhwsvLNw06+d/2bLENQ3nYPhw6NsX3n47ctvatVobDuvCzTfr9U31HiwxzrkKNfXs2dOVFr/+6lxWlnMXXZQ4XoMGzrVsWbJjdezoHDj36KOR4eefr+EPPBCE/ec/Gvb730fGnT1bw0eNin2MRx7R7f5Ur55z27cH6/Hwtz/3XOR6Mvbuda5OHY27zz46HzRIt/Xvr+tjxjh39NG6fNVVwb6vvaZhPXo4d8wxztWs6dzvfheZ/qefapysLD1/YbZs0W1nn+3cpk2Bzb/8otsXL9Z1Eef27Emel5NP1uvcvLmet0Tcequm/dVXhbcNG6bbrr1W55MnOzdunC6PHRsZd8QIDb/yyuT2Oaf5qF1by2JOjnNHHJHafs45d8opztWqpWV+//31uNddF2w/4ggNmzQpcr8aNTR83rwg7KSTNOzmmwsf57rrgmvRtq1zjRs7d+aZsW168skg7sKFyfPw/feRZfP++4P1d98N4p14opan3buTpxnNU08Fab7zjob9+c+6/tRTkXFPOSWIu3x5/DSnTg3inXZa/G3RU5cuRbc/DDDTpaCxGeXpL1+uT+iwlxDNnj1a3Qp74l9+qZ6Rz5IlWltYuhTeeCN2OrHa9FetggkToFo1ePhhbY54/vnA85s5U6uvo0drW7m/77Zt8NRT8I9/BGH/+582qYT56afIqvXOnYXtck7blUE9rnie8ddfq/f0+uuR6e/Yoct+reTDD+HZZ2N7+rE8s1deUW/94ot1MM/69WrTCy+oxwbQv39hT3/p0sDmcHX4ySfVVj8fzhXed86coLlu0SL4+GO9hiefDL//veYrXjt0OB/h9w2jR2tzxXPPadi8eTrPz4+s/Tz7LPz8c+T+0bXIl18OagGgZWr5cq1F/PIL3HorXHopzJ4dvENJRn6+5unzz4NzH/b0a9fW+a23Bn3rly4NPPxwU1DY0/fL56+/avkaNy6I16wZXHmlXuNly7Ssb9yozTnffx9ZHsK2xOPxx4PlvXsj9//gA5327NE87typXvu338K99+r9AXrcF19UO8aPL1wTDtdeFi/Wa/Pkk7FtXLUK9tkniDtuXOCxv/ZaULN74AFo2hTOO0/L3Wuv6b10zz2BXbHIz09+TtJCKk+GspxK09P3vc3u3ePH8b3IGjWCsGhP+JBDdD03V73S776LTGPvXg0H54YPD8KfeUbD7rhD58cfr/MLL9R5tWpBnOeeCzxG35sE5+6917n164P1nj2d69bNucGDdf3554Nt335bOH8//RRsP+oo5958M1jfsSOId955GtaoURC2dGlh70QksB2cu+KKYFu4tnT33ZHHmD9f10eOdG7BgmCfgw5S7zHa+x47NogzeXKkDaed5tzbb8f3wvzwn392buBA55o0UXv/8hfnnnhCt61eHbdIuNxcjfPPf+q67wn26hWknZMTeMN5ebp86KGuoHbinHOdO+v6KacEaW/bFnmef/7ZuerVnevb17kpU3Tbhx8698oruvzJJ/HtDOPXxK6+OrgW1aoF56Zly8hzuGlTpCf95JMab8eO4BqfcIJzL73kCmo2H30UmcagQc6tWqXLN9wQue3GG5079VTnunZV2y69NHkeuncP9l+2zLnWrZ077LAgLDs70nN+802tRfnl8scfg3J82WU6nzYt8hiXXOLcAQdoebv6auceeigohx06FD6nZ52l26+/XufduulxwLlbbtEaabVqWqvzaxH++YueGjYMtp19ts63bUvt+sYC8/SVtWsDj9f3FBJ5+n7b8K5d8b0/3xuaN089jegXc1u2BG1+GzYE3oTfRnjqqTr/8EOd796t87171UsB9RB8LyLcDfHTTyMHwNSvr3b8/e+6Hm5bnTUr8DJBvfOwB7duXaQ3E26D9o8Rfr/x/feR+Tz77OB4fhvttGk6P+wwtdsfp7BuHTRsGHiYXbpATo4eP+zBHXUUNGig3veePXrudu2KjBPdTjp9euT7kHi1l2ef1ZfcGzeqve3bqw2g5/rXXzUtf4CYny/f0128WL38//xH17/4Ak45BbKygmu1eHFwTvz5//4Hf/lL0NXQt2/16uBYP/6oyzNmaHl4/331lEHtPPJIXZ44MRig9MknQVqbN+v6hg16zv3z4b8HuOMOrWHeeafWdlauhLvuCravWqW2N26s8Vat0jjLlqlE1aqlYd95P+R76ilNB7TnGOi5bN5c55MmRZ77Tz/V9Nu312sc7iixfn3s8SwrV0KnTrr8t7+pJ3/zzVDd+8nr9u2RI7cXLw7KrXOaN9+zfuEFnd99d2RZWrUKWrRQuxYu1Br4Mcfoe42vvw6uq39Oe/bUcuznb/78oMPC119rrWPvXm2j96+ZiNainn46OG7t2trL7oADtHuyrwll0ZunSov+jh1w4IFw1VW67t+869fHf5EUFoxt2yJF06d+fZ03bqxNBNFDz8MvaSZMgJYttcBs2qQ3T9euKhQ+4WaY5cs1/enTg3R8oW7VqrBI5uXpvHlzTXv+/GDboEFw7bXB+mmnwQUX6PL++6sQz5gRbPcfhs7pudpvP10PN92EadxYxbtBgyDMt+2MM3Tu91hZty5ywJWI3vzTp0e+8D3+eL2pQF8ItmungrlkSZDH99+PtGPjRm0a8wn34Alfz1GjIm+qdu0iRf/vf1ebunQJHoCrVwdNWkuWaK+LrVu1uQW022jT0K+o33479ku+e+4JnIgff9Q02rXTF7U+o0cHD+HsbLW3fn29Dvvtp11iH3lEByideSb07q0vHJ1T0ejdW8+7n8e6dYNrdsQRMGSINkd17qxh7dtr2QTdZ8kSFdkDD1R7W7WCKVN0+1FHafODf31//lmbX2rWhBNP1DD/XLZrFzTHgZaPmTO1bHfsqGl9/XXwYNpvP7UljC+yftrPPANt22pnAN9Jql9fj9OsGTRqpPYvWaKDt0Tgxht1XrOm3mP168O772r+/XvKF/127bS56LvvdL/evYPrCcEDvE0btTU8VsDvRbZkiT7MRLRraseO2nQ7aBAcckhw7x15JOTm6vXs0EEfMm3aBPaUNlVa9P0nsN/u7hfYPXsK95TxCQvGtm2xB4KsW6ffh5k7F7p3L+xZ+u3uYWFfs0aFJSdHC+FBBwXb/P0nTVJP+YYbYMGCwAv0C2j//mr35MkqfkuWqOcC6p01bly4W97UqTrfu1cLrl9427ZVb3rRIhVTCDz51av1puvbV9cTiX5WlgoK6I3nz6+5RvP68MPBvv5DxOfII/Xmnz5dhebrr7Wt33+I3HOPtmH/+9/6cOrcWeP51+Tdd4N3LeF3K+Hr4YtPmzbqyYUfyO3aQZMmurxxo4pSjRqa93//W8P9MuPv//DDcOyx6ukuXKjL4XzFchJ89ttP87d5s+Z1+/ZAVHJy9F3DxInqyfqeb5MmwTuYKVPguOO0Zue/Q/CFxvecV64MrteNNwbHbtFC++uH2+DbtdNwCDz9du2C8gBaDkGdi19+0eP4+8yfrw8c/3yHRR/0wfX11zBypIru3r0qfEcdpdvD78ny8yPvNT8PeXlBebj+ei3nPh98oN2UP/pIjzlrlpbdww5Tx2rrVm1X92sijz6qD9I9e/RcOxfp6fu2n3oqHH20Lj/0kMZ7+GEt16ecEuSvfn1t4/dr7EuXarq+I1StmpZbv3tmrVpq35tvajv/E0/oe4b//jfyOpQ2VVr0/RvBf4ouWRIUoFhNPKeeqi/2fLZti/Sqf/1VvYyNG1U0W7bUgr1rl06jR2sh8JtxwsK+bl0g+hAUHAgeNCecoE/9o47SguYLgv+AOvlknU+apJ5D27ZBcwmohxx+IQj6YnfdOr2pduwIvCT/+CtX6k0SPie+5x0t+t99pwLke+y+yPs3sV/rOPJI9TKvukofUIsX6wMlWvT9/SZP1puufXsVPN/TnzBBH6rbtmk+2rfXNPw8dOmitjdqFHmdBg3SY19yid68oLUc/yWon4eGDSM9/XXr9HinnaZe7I4dQc3p9NP1OqxcqQ/latWCpofofMVj/XqtFfz4Y3COf/lF7eneXR/Qc+YE+QM93tixutyypZ5bv5mpZk29No89pg/gK6/UfPjX66KLgnTq1dMpXAs5/XStNVWvrg//77/Xc+wLEARC3L27zhcsiByktWtX0KXSr/GEBbR9e31QgQ4IbNdOr1n16nr88FiL9u01T7fcEuTBF+QmTSLzA9rU0q9f4H1/8UVwXL9s3XBDsHzccfrQyc5Wbdi8WR+uYdH3HyzVqunyrFlq04svqjbUrx/E9cusX65++UU7ePjNOv41q1cvWD/wQC2v++2n16xpUy2DzZppOagwoi8i/UVksYgsE5ERMba3EpH3RWS+iHwoIs298O4iMl1EFnrbzk13BhLhi/7evdoOum5dIALR7dOzZqm3GPY2tm2LbHrYtk0F37ngRq9bV+fbt2tPiM2bg1412dnBvuvW6b6+yISrs35vE1/Ac3Nj29irl3owoIIfTYMGQVp3360eFhRuEorev2fPwEYI4p5wQlAQf/1VexqdeGLQg6FxY50PH64PPD9N3/O/6iq9YR56KL6n36yZLocfguHmor/8Rb3qW27R2kM4Dd+b8o8XPt//+Y+24fsC6j8wQb02v9dNtOjvt58KxcaNuv/IkdCjh44RuO02ePDBoP3Vx7epVi2d+2UimpYt9YYPP0xAHz7RPbF89uyJHEfQrl3QA6Vv30Bo+vRR8dq+PSizzZqpSPvtz2PHRraBr1qlQtawYdBkFvb+QctCnTpaywzbFObXX7VM+E16/rX0y/j++2ubuv/uKztby1KjRoEXDJqvXbt0fIHfFt+ihV6vl14Kru+cOZpnvwYE+gDxad9eR7yOHasPq2uv1eO3bq01ucMOU23we8u0aKEPpMcfV0fBZ+hQHfj4pz/B7bcHI8D9/PkPtWjOTaJyY8eqLdWq6XzsWLVr//3LaIRusje9QBbwDXAQUBOYB3SKivM/4GJvuQ/wgrfcDmjrLR8IrAUaJTpeunrv7NnjXNOm+ka8dWvnPv9cl//2N52H+1D/9a/OHXxw4bfrb73l3JAhwfqjj2qvA3DuxRd1X7+P/erV2jsB9K2/3wPI3/f88zXc79fup+P3fBFxrlUrnbdsqT0Tou0ZNSroJVKnjvaJD3PiiUHcpUu150WNGs61a+dc796Rab38crA8fHjQi6BVK+2XnZ2tvZAOOMC5Aw8M+nW/845zRx6py+PHRx7f74EwYUIQdsUV2tcctAdPNDffHPTu8Pnss8C25s2DczNmjPbtB+3hsnevxr/rLg3z+6P7U/j85+cHy/ffH5zrVq20P/v11+v+l12m6eblabi/j3/8WNx4o8aJVYbCPU3GjHHuscd0/bjjgm01a8bfL3x8Eef22y/29gYNguXcXOf23bewna1axd43nM+FC4MeLOFr4Jz23olnn0hwHP8ei3XexowJ8lK9evz0/G3+OIxk7N4d7Lt9e3As/34BXR4zxrmbbtLedX4vq+nTUzuGz9y5ut+ddwZ5bd06OI5fLsN5bdVKexbVrRu/bPTq5Vy/fkWzJQwp9t5JHgGOBKaE1m8CboqKsxBo4S0LsDVOWvP8h0C8KV2iP2mS5q5ZMxVIvyvjJ5/o/MEHNd4336joHnJI0MXOn555Rm+m8E3hTx99pPu/8IKuL1mig6vCN0F0IW7QQEVrzJhACONNfhdIf8rKCgZGRRcWnzPPDLY1a6Y2NGig3QGj0w93k4zOn4h2rXMu6CLXpIl2b9u717nf/EbDpkyJPOeLF+tD0r/pnNNujn66++xTWDi3bHHu3HP1/PksWhT7nGRnO3fGGUFaPu+9F/uc1aihA/Eefljt9m+46PMoogPKsrKCAUg33VQ4vejz7eN3l00k2P5+ftfT2rVTE/viTCLOtWlT2M54XQf9ad99ndu5U6+FP4gQ1AmJ98AI59E5zWe8cjpmTGxnJtnkd39O9OB1Tu95fwDamDGRD31/qllThfrYY9WROeUU7SZbFHbu1HL15ZdBF1Y/X36Zyckp+vWtX1+dgeKSTtEfBDwdWr8QeCwqzn+BP3rLZwIOyImK0wv4CqgW4xjDgJnAzJYlHQrrcdxxzrVoof3aQfsVV6umnkONGs793/9pPH+0IRT2orp21fk99xS+QL5I+Rd97lwVxWQX9uabk99ARZlycgJvIpYXAbEfWuGRu7GmatXUM/GFokmT4IY75xwN+/zz2Oc+2sOKFoArr4z0gKJv5LBXHj35/c9btw7ib90aP74vRs7pQz1ayP0p7HW2aqX9tuOd7+i8Jrq5o+O/8UawzR/FnEyMizOFx5n4xCt3vjjeeWdk/Hi1iuhJJBhlHO8YrVqlp9zHe/CmmtdwmQh74jk5OsUrk+HrHS67556b3uuWav5iUdaifyDwMjAHeATIDzfjAAcAi4Ejkh0vHZ6+P8Dqrruce/ZZXe7RQ6vfzmk1/vLL9eQmu+Hy8pz7+OPC4Vu2aFr+oKBPPon0juJNDzxQOjd5sikry7k+fXTZH/jUuHHxCuTll+t62Dv3iedhRYtEdJhf9XZOH8zJbMnNjTxmomP5/OlP6TmX4ZsyVSHzhcT/1ARoU1JploVoEUt2bRo3jtynKOWjLMt0+EEeq/zFcziiz02i7eHaiX+NyyqPifKXiDJt3omKXw/ID603AGYDg1IxqKSiv3ZtINKTJ2u7vH8yBwzQOAcfrAIdPSox1nTFFfodkujwli21QEybpuvvvquj9Tp1SiwEzz2XXk+/KNOjj+r8wAP1PHTsGFSdU53C8YcOLdxmmsoNl+xGcy61c+S3kyZqMoi+gUpiX3jyxbGo+fNHiJb1FD1yurJP4dpiSctdKuetrKfiePvpFP3qwHKgTehFbueoOE38ZhvgXuAub7km8D5wbSrGOFcy0fc/MeCLwOLFzs2ZE5zI66/XeF26BG3Dyab77tP2/1jbsrODzwu8+qq2D/bokbjt8oYbSq+AJptattRC7A8vP+UU/TRAcdpZS2sKV71TsSvRTRldVU6lFmKTTRVhKk4zT9pEX9NiALDE68Vzixd2F3C6tzwIWOrFeRqo5YVfAOwC5oam7omOVRLRX7cuOGnVq+sLlzVrgrC5czVer176MjK6t0es6frrC7+YCk8HHqjzP/wheDnre8P+PNyzIlb7OgTfgylKwSiu11azpnpKzZvrek5OxfMAc3KC76iUNA3fIyxqraa8prp1y88xsKniTEVt5klV9MUT5wpP5m68AAAgAElEQVRDXl6emxkeU18EVq8ORhO2b6+jAXfv1j6woKcStP/5nj3ah/fRRxOn2ahRaj/m8Id6R1OjRjDKMxGHHFJ+v4EDtVEkdh7Ki5o1dSqtXzNWVPyR3OXxdyij4iBStH9Ri8gs51xesnhVakRuWLD8ARTVq+u3ccLfAqlTRwfItG6t6+Fh59EkE3x/33hiuWtXaj9HL4/fpoXZtUuFP/zpiHSRkxMMgioKO3dmnuCDin1FFvzSKCNGYfzvIqWbKiX64b/9hEfKDRminrRPdrYK8apVurxyZeTovlTJztYvFaaDRN9zLyt+/jn9YpOTo6Nby/oXgUb6EdHa8u7d6kgV554pD8KfQagsZGcHX9xNN1VS9GvX1u+v+EQPe163Tr+PM3Kkin+bNsGnBYpCtWrB1xZLSp066UmnpKTbi9u0Sc//kCHF8/YrE8Wt0RTnOFdeGfnxsbIgfI8MGaKfcEi38LdqpXlr1UrTLml5zMlJr0OVznOelaV5jM5zq1aqTUOGpO9YEaTS8F+WU0le5Po9dV5+OQiL1Qsk1kvL8u7VccQRFacXTWl0U0ule2VFnpK9BPbHGRR31GlRz2W8sl3cqUaN5C/za9Ys3KMk0RgJ0JfSqdgYr7dKSfOYrhfiqQxAK0pHgXB+owd8lfvgrLKeSiL6/vd1Xn9d18eMKdqFKO8eLJnQa8PPo9+/urzt8adED7pUh9PHGtBTWrb6hAWjuL2TwoO3kglsrB4l8fIqEqQb61qHv/eUSOiKeh/7U7L7Odmgq3j2xTpP/kjzVB5QWVmRgh8rrXLtp1/WU0lE3x+UNWVK2XhcpTH53SnjFcTSHIhSVpNfqNMtjCUVPefif6SrKCNvw8S7VnXrFv4YV3R5jVcG4nXlK844hOi0kuU1/MAJ7xPL9vAP4BN9niEVEuUtK6vo37kJHzec51S/8xPPO09lRHD4HJb0vITJSNH3/5f5wQflN+o1HVOymy5Zlbq8ppyc1B+0/g2TStx69SL72yc6N0W57kXxqIoyWCzZfomaMpI9CJLZnOpnCJKlVVQxStZEkcibTpV4+Qo/mFOp8ZTk+zapkqh2Ej6H6TgvPhkp+u+8ozn66KPyGz6djimRePkFpijefvhcpKMJK5a4F7VpI1Ees7ISt28mOjfxxDl6oFZx2k5TEZZYoliSNtt07luU/Kez2cG59Hi0qQpkonu/JG3mRSWVc2ieviuZ6L/+uubo888rv6efrMDE2l6jRuFqbnQhK+nDMFrc4wlIMs84lTzGI5Vzk44XYyWxoSqQzvOYjvOVqkCmU0hLSnHuE2vTLwKvvKI5mj27/Nv0i9u+HO4hkUqBid6ebJ+SPAyL4y3GqpGkQ6DLQtgrgw2ViZKer1QFsrI9kK33TglEf+JEzdEBB5T+1/fiTTVqpFZ989shw9/1CX9euLRIdEMk85BK4pmbOBrpINWylIllLiNF/w9/KCxYviiVpviHm0wuuSTSpmRC6f+g5bHHip3tIpOo50EiWytStdkwjEhSFf0qNSJ3woTCYdu3w8UXwznnRP44u6iI6I+oo9PIztZwn2OOidw+ZIiOros32s7/Gbo/LwuGDIEVK/RjTitWBLYks3XlytjpxQs3DKPiUaVEf9Om2OF79sATTxRtyHiNGjqE2xe/F16A996LLYo9ewb71apVOK14IgvlI/qJSGRrvA9AldaHoQzDSD9VSvSTfT/n55/jC39OTqSYjx6tHwqL5Q1Hi2L4uEUVb/+bOxVF9BNx772xazql9WEowzDST5US/QEDksdxrnBYdrZ+BTKeh5uM8Ee2Ynn6ifDFvqj7lQfJmn8Mw6j4VC9vA9JJt246r1Yt9Z8PtGqlnmpJhKtJk2C5uKJfGTx90PNkIm8YlZcq5en7n1a+/PLE7ff+j0+aNSu6Vx+Lknj6fnNJZRF9wzAqN1VS9F94IXYzDkT++KRhw/QcNyz6RRXv44+Hf/wDDj88PbYYhmEkIiXRF5H+IrJYRJaJyIgY21uJyPsiMl9EPhSR5qFtF4vIUm+6OJ3GR+P/snDHjtjb/TbooUO1JtCgQXqOW9I2/T//OfiPr2EYRmmStE1fRLKAx4F+QD4wQ0QmOecWhaLdDzzvnHtORPoAfwMuFJF9gNuBPMABs7x9N6c7IxD5u8TC+dCmHJ86ddLn6Yd771SGF7KGYWQuqXj6vYBlzrnlzrmdwHhgYFScTsAH3vLU0PbfAO86537whP5doH/JzY7Nrl3x2/Kj+5JnZ6fP068eenSa6BuGUZFJRfSbAatC6/leWJh5wJne8hlAfRHJSXFfRGSYiMwUkZkbNmxI1fZC7NoFdeum1pe8XTvo1KnYh4qLib5hGBWZdL3IvRE4TkTmAMcBq4E9qe7snBvpnMtzzuU1bdq02Ebs3Kl/vk+lL/knn8Dttxf7UHGxXjiGYVRkUumnvxpoEVpv7oUV4Jxbg+fpi0g94Czn3I8isho4PmrfD0tgb0J27dIXoqn2JS/KZxlSxTx9wzAqMql4+jOAtiLSRkRqAucBk8IRRKSJiPhp3QSM8panACeJSGMRaQyc5IWVCr7olycm+oZhVGSSir5zbjcwHBXrr4CJzrmFInKXiJzuRTseWCwiS4D9gHu9fX8A7kYfHDOAu7ywUqE8Rd9/mZuVVT7HNwzDSIWUPsPgnHsTeDMq7LbQ8ovAi3H2HUXg+Zcq33wDy5frZxhatiz55xWKwoIF8OmnZXMswzCM4lJlvr0zdizMmRN8c+e772DYMF0uC+Hv0EEnwzCMikyV+QzDLbcU/sja9u0abhiGYShVRvTtr06GYRjJqTKib391MgzDSE6VEf1779UXuGHsr06GYRiRVBnRHzIEWrTQD6nZX50MwzBiU2V67wDUr68/KX/ppfK2xDAMo2JSZTx90G/vlPeIXMMwjIpMlRL9ivAZBsMwjIqMib5hGEYGUeVEv2bN8rbCMAyj4lKlRN/a9A3DMBJTpUTfmncMwzASY6JvGIaRQVQp0d+509r0DcMwElFlRH/PHnDOPH3DMIxEVBnR37VL5yb6hmEY8THRNwzDyCCqjOjv3Klza9M3DMOIT0qiLyL9RWSxiCwTkRExtrcUkakiMkdE5ovIAC+8hog8JyILROQrEbkp3Rnw2bsXDjgAGjQorSMYhmFUfpJ+ZVNEsoDHgX5APjBDRCY55xaFot0KTHTOPSEindCfqLcGzgZqOee6ikg2sEhExjnnVqQ5HzRtCmvWpDtVwzCMqkUqnn4vYJlzbrlzbicwHhgYFccBvo/dEFgTCq8rItWBOsBOYGuJrTYMwzCKRSqi3wxYFVrP98LC3AFcICL5qJd/tRf+IvAzsBZYCdzvnPsh+gAiMkxEZorIzA0bNhQtB4ZhGEbKpOtF7mDgWedcc2AA8IKIVENrCXuAA4E2wA0iclD0zs65kc65POdcXtOmTdNkkmEYhhFNKqK/GmgRWm/uhYW5DJgI4JybDtQGmgDnA28753Y559YDnwB5JTXaMAzDKB6piP4MoK2ItBGRmsB5wKSoOCuBvgAi0hEV/Q1eeB8vvC5wBPB1ekw3DMMwikpS0XfO7QaGA1OAr9BeOgtF5C4ROd2LdgNwhYjMA8YBQ51zDu31U09EFqIPj9HOufmlkRHDMAwjOaLaXHHIy8tzM2fOLG8zDMMwKhUiMss5l7T5vMqMyDUMwzCSY6JvGIaRQZjoG4ZhZBAm+oZhGBmEib5hGEYGYaJvGIaRQZjoG4ZhZBAm+oZhGBmEib5hGEYGYaJvGIaRQZjoG4ZhZBAm+oZhGBmEib5hGEYGYaJvGIaRQZjoG4ZhZBAm+oZhGBmEib5hGEYGYaJvGIaRQZjoG4ZhZBApib6I9BeRxSKyTERGxNjeUkSmisgcEZkvIgNC27qJyHQRWSgiC0SkdjozYBiGYaRO9WQRRCQLeBzoB+QDM0RkknNuUSjarcBE59wTItIJeBNoLSLVgTHAhc65eSKSA+xKey4MwzCMlEjF0+8FLHPOLXfO7QTGAwOj4jiggbfcEFjjLZ8EzHfOzQNwzm1yzu0pudmGYRhGcUhF9JsBq0Lr+V5YmDuAC0QkH/Xyr/bC2wFORKaIyGwR+XOsA4jIMBGZKSIzN2zYUKQMGIZhGKmTrhe5g4FnnXPNgQHACyJSDW0+6g0M8eZniEjf6J2dcyOdc3nOubymTZumySTDMAwjmlREfzXQIrTe3AsLcxkwEcA5Nx2oDTRBawXTnHMbnXPb0VpAj5IabRiGYRSPVER/BtBWRNqISE3gPGBSVJyVQF8AEemIiv4GYArQVUSyvZe6xwGLMAzDMMqFpL13nHO7RWQ4KuBZwCjn3EIRuQuY6ZybBNwAPCUi16EvdYc65xywWUQeRB8cDnjTOfdGaWXGMAzDSIyoNlcc8vLy3MyZM8vbDMMwjEqFiMxyzuUli2cjcg3DMDIIE33DMIwMwkTfMAwjgzDRNwzDyCBM9A3DMDIIE33DMIwMwkTfMAwjgzDRNwzDyCBM9A3DMDIIE33DMIwMwkTfMAwjgzDRNwzDyCBM9A3DMDIIE33DMIwMwkTfMAwjgzDRNwzDyCBM9A3DMDIIE33DMIwMwkTfMAwjg0hJ9EWkv4gsFpFlIjIixvaWIjJVROaIyHwRGRBj+08icmO6DDcMwzCKTlLRF5Es4HHgZKATMFhEOkVFuxWY6Jw7FDgP+HfU9geBt0purmEYhlESUvH0ewHLnHPLnXM7gfHAwKg4DmjgLTcE1vgbROS3wLfAwpKbaxiGYZSEVES/GbAqtJ7vhYW5A7hARPKBN4GrAUSkHvB/wJ2JDiAiw0RkpojM3LBhQ4qmG4ZhGEUlXS9yBwPPOueaAwOAF0SkGvoweMg591OinZ1zI51zec65vKZNm6bJJMMwDCOa6inEWQ20CK0398LCXAb0B3DOTReR2kAT4HBgkIj8E2gE7BWRX5xzj5XYcsMwDKPIpCL6M4C2ItIGFfvzgPOj4qwE+gLPikhHoDawwTl3jB9BRO4AfjLBNwzDKD+SNu8453YDw4EpwFdoL52FInKXiJzuRbsBuEJE5gHjgKHOOVdaRhuGYRjFQyqaNufl5bmZM2eWtxmGYRiVChGZ5ZzLSxbPRuQahmFkECb6hmEYGYSJvmEYRgZhom8YhpFBmOgbhmFkECb6hmEYGYSJvmEYRgZhom8YhpFBmOgbhmFkECb6hmEYGYSJvmEYRgZhom8YhpFBmOgbhmFkECb6hmEYGYSJvmEYRgZhom8YhpFBmOgbhmFkECb6hmEYGYSJvmEYRgaRkuiLSH8RWSwiy0RkRIztLUVkqojMEZH5IjLAC+8nIrNEZIE375PuDBiGYRipUz1ZBBHJAh4H+gH5wAwRmeScWxSKdisw0Tn3hIh0At4EWgMbgdOcc2tEpAswBWiW5jwYhmEYKZKKp98LWOacW+6c2wmMBwZGxXFAA2+5IbAGwDk3xzm3xgtfCNQRkVolN9swDMMoDqmIfjNgVWg9n8Le+h3ABSKSj3r5V8dI5yxgtnPu1+gNIjJMRGaKyMwNGzakZLhhGIZRdNL1Incw8KxzrjkwAHhBRArSFpHOwD+A38Xa2Tk30jmX55zLa9q0aZpMMgzDMKJJRfRXAy1C6829sDCXARMBnHPTgdpAEwARaQ68AlzknPumpAYbhmEYxScV0Z8BtBWRNiJSEzgPmBQVZyXQF0BEOqKiv0FEGgFvACOcc5+kz2zDMAyjOCQVfefcbmA42vPmK7SXzkIRuUtETvei3QBcISLzgHHAUOec8/Y7BLhNROZ6076lkhPDMAwjKaLaXHHIy8tzM2fOLG8zDMMwKhUiMss5l5csno3INQzDyCBM9A3DMDIIE33DMIwMwkTfMAwjgzDRNwzDyCBM9A3DMDIIE33DMIwMIumnlQ3DKD927dpFfn4+v/zyS3mbYlQQateuTfPmzalRo0ax9jfRN4wKTH5+PvXr16d169aISHmbY5Qzzjk2bdpEfn4+bdq0KVYa1rxjGBWYX375hZycHBN8AwARIScnp0Q1PxN9w6jgmOAbYUpaHkz0DcMwMggTfcOoQowdC61bQ7VqOh87tmTpbdq0ie7du9O9e3f2339/mjVrVrC+c+fOlNK45JJLWLx4ccI4jz/+OGNLaqyREvYi1zCqCGPHwrBhsH27rn/3na4DDBlSvDRzcnKYO3cuAHfccQf16tXjxhtvjIjjnMM5R7VqsX3I0aNHJz3OVVddVTwDy5Hdu3dTvXrlk1Dz9A2jinDLLYHg+2zfruHpZtmyZXTq1IkhQ4bQuXNn1q5dy7Bhw8jLy6Nz587cddddBXF79+7N3Llz2b17N40aNWLEiBHk5uZy5JFHsn79egBuvfVWHn744YL4I0aMoFevXrRv355PP/0UgJ9//pmzzjqLTp06MWjQIPLy8goeSGFuv/12DjvsMLp06cLvf/97/M/HL1myhD59+pCbm0uPHj1YsWIFAH/961/p2rUrubm53OKdLN9mgO+//55DDjkEgKeffprf/va3nHDCCfzmN79h69at9OnThx49etCtWzdef/31AjtGjx5Nt27dyM3N5ZJLLmHLli0cdNBB7N69G4DNmzdHrJcVJvqGUUVYubJo4SXl66+/5rrrrmPRokU0a9aMv//978ycOZN58+bx7rvvsmjRokL7bNmyheOOO4558+Zx5JFHMmrUqJhpO+f44osvuO+++woeII8++ij7778/ixYt4i9/+Qtz5syJue8f//hHZsyYwYIFC9iyZQtvv/02AIMHD+a6665j3rx5fPrpp+y7775MnjyZt956iy+++IJ58+Zxww03JM33nDlzePnll3n//fepU6cOr776KrNnz+a9997juuuuA2DevHn84x//4MMPP2TevHk88MADNGzYkKOPPrrAnnHjxnH22WeXeW3BRN8wqggtWxYtvKQcfPDB5OUF/+wYN24cPXr0oEePHnz11VcxRb9OnTqcfPLJAPTs2bPA247mzDPPLBTn448/5rzzzgMgNzeXzp07x9z3/fffp1evXuTm5vL//t//Y+HChWzevJmNGzdy2mmnATrAKTs7m/fee49LL72UOnXqALDPPvskzfdJJ51E48aNAX04jRgxgm7dunHSSSexatUqNm7cyAcffMC5555bkJ4/v/zyywuau0aPHs0ll1yS9HjpxkTfMKoI994L2dmRYdnZGl4a1K1bt2B56dKlPPLII3zwwQfMnz+f/v37x+xLXrNmzYLlrKysuE0btWrVShonFtu3b2f48OG88sorzJ8/n0svvbRYfdqrV6/O3r17AQrtH873888/z5YtW5g9ezZz586lSZMmCY933HHHsWTJEqZOnUqNGjXo0KFDkW0rKSb6hlFFGDIERo6EVq1AROcjRxb/JW5R2Lp1K/Xr16dBgwasXbuWKVOmpP0YRx99NBMnTgRgwYIFMWsSO3bsoFq1ajRp0oRt27bx0ksvAdC4cWOaNm3K5MmTARXy7du3069fP0aNGsWOHTsA+OGHHwBo3bo1s2bNAuDFF1+Ma9OWLVvYd999qV69Ou+++y6rV68GoE+fPkyYMKEgPX8OcMEFFzBkyJBy8fIhRdEXkf4islhElonIiBjbW4rIVBGZIyLzRWRAaNtN3n6LReQ36TTeMIxIhgyBFStg716dl4XgA/To0YNOnTrRoUMHLrroIo4++ui0H+Pqq69m9erVdOrUiTvvvJNOnTrRsGHDiDg5OTlcfPHFdOrUiZNPPpnDDz+8YNvYsWN54IEH6NatG71792bDhg2ceuqp9O/fn7y8PLp3785DDz0EwJ/+9CceeeQRevTowebNm+PadOGFF/Lpp5/StWtXxo8fT9u2bQFtfvrzn//MscceS/fu3fnTn/5UsM+QIUPYsmUL5557bjpPT8ok/TG6iGQBS4B+QD4wAxjsnFsUijMSmOOce0JEOgFvOudae8vjgF7AgcB7QDvn3J54x7MfoxtGwFdffUXHjh3L24wKwe7du9m9eze1a9dm6dKlnHTSSSxdurTSdZscP348U6ZMSakrazxilYtUf4yeytnqBSxzzi33Eh4PDATCdSsHNPCWGwJrvOWBwHjn3K/AtyKyzEtvegrHNQzDKOCnn36ib9++7N69G+ccTz75ZKUT/CuvvJL33nuvoAdPeZDKGWsGrAqt5wOHR8W5A3hHRK4G6gInhvb9LGrfZtEHEJFhwDCAlqXV1cAwjEpNo0aNCtrZKytPPPFEeZuQthe5g4FnnXPNgQHACyKSctrOuZHOuTznXF7Tpk3TZJJhGIYRTSqe/mqgRWi9uRcW5jKgP4BzbrqI1AaapLivYRiGUUak4o3PANqKSBsRqQmcB0yKirMS6AsgIh2B2sAGL955IlJLRNoAbYEv0mW8YRiGUTSSevrOud0iMhyYAmQBo5xzC0XkLmCmc24ScAPwlIhch77UHeq0W9BCEZmIvvTdDVyVqOeOYRiGUbqk1O7unHvTOdfOOXewc+5eL+w2T/Bxzi1yzh3tnMt1znV3zr0T2vdeb7/2zrm3SicbhmGUBieccEKhgVYPP/wwV155ZcL96tWrB8CaNWsYNGhQzDjHH388ybpnP/zww2wPfUVuwIAB/Pjjj6mYbsTBRuQahhGXwYMHM378+Iiw8ePHM3jw4JT2P/DAAxOOaE1GtOi/+eabNGrUqNjplTXOuYLPOVQUTPQNo5Jw7bVw/PHpna69NvExBw0axBtvvFHww5QVK1awZs0ajjnmmIJ+8z169KBr16689tprhfZfsWIFXbp0AfQTCeeddx4dO3bkjDPOKPj0AWj/df+zzLfffjsA//rXv1izZg0nnHACJ5xwAqCfR9i4cSMADz74IF26dKFLly4Fn2VesWIFHTt25IorrqBz586cdNJJEcfxmTx5MocffjiHHnooJ554IuvWrQN0LMAll1xC165d6datW8FnHN5++2169OhBbm4uffv2BfT/Avfff39Bml26dGHFihWsWLGC9u3bc9FFF9GlSxdWrVoVM38AM2bM4KijjiI3N5devXqxbds2jj322IhPRvfu3Zt58+YlvlBFoHKNbDAMo0zZZ5996NWrF2+99RYDBw5k/PjxnHPOOYgItWvX5pVXXqFBgwZs3LiRI444gtNPPz3uP1yfeOIJsrOz+eqrr5g/fz49evQo2Hbvvfeyzz77sGfPHvr27cv8+fO55pprePDBB5k6dSpNmjSJSGvWrFmMHj2azz//HOcchx9+OMcddxyNGzdm6dKljBs3jqeeeopzzjmHl156iQsuuCBi/969e/PZZ58hIjz99NP885//5IEHHuDuu++mYcOGLFiwANBv3m/YsIErrriCadOm0aZNm4jv6MRj6dKlPPfccxxxxBFx89ehQwfOPfdcJkyYwGGHHcbWrVupU6cOl112Gc8++ywPP/wwS5Ys4ZdffiE3N7dI1y0RJvqGUUnwnNkyx2/i8UX/mWeeAbTp4uabb2batGlUq1aN1atXs27dOvbff/+Y6UybNo1rrrkGgG7dutGtW7eCbRMnTmTkyJHs3r2btWvXsmjRoojt0Xz88cecccYZBV+8PPPMM/noo484/fTTadOmDd27dwfif745Pz+fc889l7Vr17Jz507atGkDwHvvvRfRnNW4cWMmT57MscceWxAnlc8vt2rVqkDw4+VPRDjggAM47LDDAGjQQD9qcPbZZ3P33Xdz3333MWrUKIYOHZr0eEWhyjTvpPvfoIZhKAMHDuT9999n9uzZbN++nZ49ewL6AbMNGzYwa9Ys5s6dy3777Veszxh/++233H///bz//vvMnz+fU045pVjp+PifZYb4n2a++uqrGT58OAsWLODJJ58s8eeXIfITzOHPLxc1f9nZ2fTr14/XXnuNiRMnMiTNX82rEqLv/xv0u+/AueDfoCb8hlFy6tWrxwknnMCll14a8QLX/6xwjRo1mDp1Kt99913CdI499lj++9//AvDll18yf/58QD/LXLduXRo2bMi6det4662gk1/9+vXZtm1bobSOOeYYXn31VbZv387PP//MK6+8wjHHHJNynrZs2UKzZvpFmOeee64gvF+/fjz++OMF65s3b+aII45g2rRpfPvtt0Dk55dnz54NwOzZswu2RxMvf+3bt2ft2rXMmDEDgG3bthU8oC6//HKuueYaDjvssIIftqSLKiH6ZflvUMPIRAYPHsy8efMiRH/IkCHMnDmTrl278vzzzyf9IciVV17JTz/9RMeOHbntttsKagy5ubkceuihdOjQgfPPPz/is8zDhg2jf//+BS9yfXr06MHQoUPp1asXhx9+OJdffjmHHnpoyvm54447OPvss+nZs2fE+4Jbb72VzZs306VLF3Jzc5k6dSpNmzZl5MiRnHnmmeTm5hZ8Evmss87ihx9+oHPnzjz22GO0a9cu5rHi5a9mzZpMmDCBq6++mtzcXPr161dQA+jZsycNGjQolW/uJ/20cllTnE8rV6umHn40IvpdccOorNinlTOTNWvWcPzxx/P1119TrVph37wkn1auEp5+Wf8b1DAMo7R4/vnnOfzww7n33ntjCn5JqRKiX9b/BjUMwygtLrroIlatWsXZZ59dKulXCdEvz3+DGkZpU9GaYI3ypaTlocr00x8yxETeqHrUrl2bTZs2kZOTE3fQk5E5OOfYtGkTtWvXLnYaVUb0DaMq0rx5c/Lz89mwYUN5m2JUEGrXrk3z5s2Lvb+JvmFUYGrUqFEwEtQw0kGVaNM3DMMwUsNE3zAMI4Mw0TcMw8ggKtyIXBHZACT+iEdimgAb02ROeVNV8lJV8gGWl4qK5QVaOeeaJotU4US/pIjIzFSGIlcGqkpeqko+wPJSUbG8pI417xiGYWQQJvqGYRgZRFUU/ZHlbUAaqSp5qSr5AMtLRcXykiJVrk3fMAzDiE9V9PQNwzCMOJjoG4ZhZAf7bSwAAAQASURBVBBVRvRFpL+ILBaRZSIyorztKSoiskJEFojIXBGZ6YXtIyLvishSb57en2WmCREZJSLrReTLUFhM20X5l3ed5otIj/KzvDBx8nKHiKz2rs1cERkQ2naTl5fFIvKb8rE6NiLSQkSmisgiEVkoIn/0wivVtUmQj0p3XUSktoh8ISLzvLzc6YW3EZHPPZsniEhNL7yWt77M2966xEY45yr9BGQB3wAHATWBeUCn8rariHlYATSJCvsnMMJbHgH8o7ztjGP7sUAP4MtktgMDgLcAAY4APi9v+1PIyx3AjTHidvLKWi2gjVcGs8o7DyH7DgB6eMv1gSWezZXq2iTIR6W7Lt65rect1wA+9871ROA8L/w/wJXe8h+A/3jL5wETSmpDVfH0ewHLnHPLnXM7gfHAwHK2KR0MBJ7zlp8DfluOtsTFOTcN+CEqOJ7tA4HnnfIZ0EhEDigbS5MTJy/xGAiMd8796pz7FliGlsUKgXNurXNutre8DfgKaEYluzYJ8hGPCntdvHP7k7daw5sc0Ad40QuPvib+tXoR6Csl/LFCVRH9ZsCq0Ho+iQtFRcQB74jILBEZ5oXt55xb6y1/D+xXPqYVi3i2V9ZrNdxr8hgVamarNHnxmgUORT3LSnttovIBlfC6iEiWiMwF1gPvojWRH51zu70oYXsL8uJt3wLklOT4VUX0qwK9nXM9gJOBq0Tk2PBGp/W7Stm/tjLb7vEEcDDQHVgLPFC+5hQNEakHvARc65zbGt5Wma5NjHxUyuvinNvjnOsONEdrIB3K8vhVRfRXAy1C6829sEqDc261N18PvIIWhnV+9dqbry8/C4tMPNsr3bVyzq3zbtS9wFMETQUVPi8iUgMVyrHOuZe94Ep3bWLlozJfFwDn3I/AVOBItCnN/6lV2N6CvHjbGwKbSnLcqiL6M4C23hvwmugLj0nlbFPKiEhdEanvLwMnAV+iebjYi3Yx8Fr5WFgs4tk+CbjI6ylyBLAl1NRQIYlq1z4DvTageTnP62HRBmgLfFHW9sXDa/t9BvjKOfdgaFOlujbx8lEZr4uINBWRRt5yHaAf+o5iKjDIixZ9TfxrNQj4wKudFZ/yfpudrgntebAEbR+7pbztKaLtB6G9DeYBC3370ba794GlwHvAPuVtaxz7x6HV611oe+Rl8WxHey887l2nBUBeedufQl5e8Gyd792EB4Ti3+LlZTFwcnnbH5WX3mjTzXxgrjcNqGzXJkE+Kt11AboBczybvwRu88IPQh9My4D/AbW88Nre+jJv+0EltcE+w2AYhpFBVJXmHcMwDCMFTPQNwzAyCBN9wzCMDMJE3zAMI4Mw0TcMw8ggTPQNwzAyCBN9wzCMDOL/A/ymw8sQOihcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEICAYAAACzliQjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8FeW5wPHfQ1gDYQsIlSXBpUoQBIygFxFZqqhVRKkXhLqLotattiIuVay9bhdX3NpqtaBopSpucKtgkVoQsAgiIiggCCIgIJtKkuf+8c5k5iTnJCfJyTrP9/OZz+wz78yc88w778y8I6qKMcaYaKhX3QkwxhhTdSzoG2NMhFjQN8aYCLGgb4wxEWJB3xhjIsSCvjHGRIgFfVMmIpImIrtFpHMqp61OInKIiKT82WURGSIia0P9K0WkfzLTlmNdfxKRCeWdv4Tl/l5E/pLq5ZrqU7+6E2Aql4jsDvWmAz8A+V7/pao6tSzLU9V8oFmqp40CVT0sFcsRkYuBMap6QmjZF6di2abus6Bfx6lqYdD1cpIXq+rbiaYXkfqqmlcVaTPGVD0r3ok47/L9BRF5XkR2AWNE5FgRmS8iO0Rkk4g8JCINvOnri4iKSLbXP8Ub/5aI7BKRf4tIl7JO640/WUQ+E5GdIvKwiPxLRM5PkO5k0nipiKwWke0i8lBo3jQRuV9EtonIF8DQEvbPTSIyrciwySIyyeu+WERWeNvzuZcLT7SsDSJygtedLiJ/9dK2HDiqyLQ3i8gX3nKXi8jp3vDuwCNAf6/obGto394Wmv8yb9u3icgrIvKTZPZNaURkuJeeHSIyW0QOC42bICIbReQ7Efk0tK3HiMiH3vDNInJvsuszlUBVrYlIA6wFhhQZ9nvgR+A0XCagCXA00Bd3JXgQ8BlwpTd9fUCBbK9/CrAVyAUaAC8AU8ox7QHALmCYN+46YD9wfoJtSSaNrwItgGzgW3/bgSuB5UBHIBOY6/4KcddzELAbaBpa9jdArtd/mjeNAIOAfUAPb9wQYG1oWRuAE7zu+4B3gVZAFvBJkWnPBn7iHZNzvDS088ZdDLxbJJ1TgNu87hO9NPYEGgOPArOT2Tdxtv/3wF+87q5eOgZ5x2gCsNLr7gasA9p703YBDvK6FwKjvO4MoG91/xei3FhO3wDMU9XXVLVAVfep6kJVXaCqear6BfAkMKCE+V9S1UWquh+Yigs2ZZ3258ASVX3VG3c/7gQRV5Jp/B9V3amqa3EB1l/X2cD9qrpBVbcBd5Wwni+Aj3EnI4CfAdtVdZE3/jVV/UKd2cA7QNybtUWcDfxeVber6jpc7j283hdVdZN3TJ7DnbBzk1guwGjgT6q6RFW/B8YDA0SkY2iaRPumJCOBGao62ztGd+FOHH2BPNwJpptXRLjG23fgTt6Hikimqu5S1QVJboepBBb0DcD6cI+IHC4ib4jI1yLyHTARaFPC/F+HuvdS8s3bRNMeGE6HqiouZxxXkmlMal24HGpJngNGed3neP1+On4uIgtE5FsR2YHLZZe0r3w/KSkNInK+iHzkFaPsAA5Pcrngtq9wear6HbAd6BCapizHLNFyC3DHqIOqrgR+jTsO33jFhe29SS8AcoCVIvKBiJyS5HaYSmBB34C73A97Ape7PURVmwO34oovKtMmXHELACIixAapoiqSxk1Ap1B/aY+UvggMEZEOuBz/c14amwAvAf+DK3ppCfxfkun4OlEaROQg4DFgHJDpLffT0HJLe7x0I67IyF9eBq4Y6ask0lWW5dbDHbOvAFR1iqr2wxXtpOH2C6q6UlVH4orw/heYLiKNK5gWU04W9E08GcBOYI+IdAUurYJ1vg70FpHTRKQ+cDXQtpLS+CJwjYh0EJFM4IaSJlbVr4F5wF+Alaq6yhvVCGgIbAHyReTnwOAypGGCiLQU9x7DlaFxzXCBfQvu/HcJLqfv2wx09G9cx/E8cJGI9BCRRrjg+56qJrxyKkOaTxeRE7x1/wZ3H2aBiHQVkYHe+vZ5TQFuA34pIm28K4Od3rYVVDAtppws6Jt4fg2ch/tDP4G74VqpVHUz8N/AJGAbcDDwH9x7BalO42O4svdluJuMLyUxz3O4G7OFRTuqugO4FngZdzN0BO7klYzf4a441gJvAc+GlrsUeBj4wJvmMCBcDv4PYBWwWUTCxTT+/DNxxSwve/N3xpXzV4iqLsft88dwJ6ShwOle+X4j4B7cfZivcVcWN3mzngKsEPd02H3Af6vqjxVNjykfcUWnxtQsIpKGK04YoarvVXd6jKkrLKdvagwRGeoVdzQCbsE99fFBNSfLmDrFgr6pSY4DvsAVHZwEDFfVRMU7xphysOIdY4yJEMvpG2NMhNS4CtfatGmj2dnZ1Z0MY4ypVRYvXrxVVUt6zBmogUE/OzubRYsWVXcyjDGmVhGR0t4sB6x4xxhjIsWCvjHGRIgFfWOMiZAaV6ZvjKla+/fvZ8OGDXz//ffVnRSThMaNG9OxY0caNEhU9VLJLOgbE3EbNmwgIyOD7OxsXOWmpqZSVbZt28aGDRvo0qVL6TPEUWeKd6ZOhexsqFfPtaeW6XPfxkTX999/T2ZmpgX8WkBEyMzMrNBVWZ3I6U+dCmPHwt69rn/dOtcPMLrCdQsaU/dZwK89Knqs6kRO/6abgoDv27vXDTfGGBOoE0H/yy/LNtwYU3Ns27aNnj170rNnT9q3b0+HDh0K+3/8Mblq9y+44AJWrlxZ4jSTJ09maorKfY877jiWLFmSkmVVtTpRvNO5syvSiTfcGJNaU6e6q+gvv3T/sTvvrFgxamZmZmEAve2222jWrBnXX399zDSqiqpSr178fOrTTz9d6nquuOKK8ieyDqkTOf0774T09Nhh6eluuDEmdfz7Z+vWgWpw/6wyHpxYvXo1OTk5jB49mm7durFp0ybGjh1Lbm4u3bp1Y+LEiYXT+jnvvLw8WrZsyfjx4znyyCM59thj+eabbwC4+eabeeCBBwqnHz9+PH369OGwww7j/fffB2DPnj2cddZZ5OTkMGLECHJzc0vN0U+ZMoXu3btzxBFHMGHCBADy8vL45S9/WTj8oYceAuD+++8nJyeHHj16MGbMmJTvs2TUiZy+n8tIZe7DGFNcSffPKuP/9umnn/Lss8+Sm5sLwF133UXr1q3Jy8tj4MCBjBgxgpycnJh5du7cyYABA7jrrru47rrreOqppxg/fnyxZasqH3zwATNmzGDixInMnDmThx9+mPbt2zN9+nQ++ugjevfuXWL6NmzYwM0338yiRYto0aIFQ4YM4fXXX6dt27Zs3bqVZcuWAbBjxw4A7rnnHtatW0fDhg0Lh1W1OpHTB/eDW7sWCgpc2wK+MalX1ffPDj744MKAD/D888/Tu3dvevfuzYoVK/jkk0+KzdOkSRNOPvlkAI466ijWrl0bd9lnnnlmsWnmzZvHyJEjATjyyCPp1q1bielbsGABgwYNok2bNjRo0IBzzjmHuXPncsghh7By5UquuuoqZs2aRYsWLQDo1q0bY8aMYerUqeV+uaqi6kzQN8ZUvkT3ySrr/lnTpk0Lu1etWsWDDz7I7NmzWbp0KUOHDo37vHrDhg0Lu9PS0sjLy4u77EaNGpU6TXllZmaydOlS+vfvz+TJk7n00ksBmDVrFpdddhkLFy6kT58+5Ofnp3S9ybCgb4xJWnXeP/vuu+/IyMigefPmbNq0iVmzZqV8Hf369ePFF18EYNmyZXGvJML69u3LnDlz2LZtG3l5eUybNo0BAwawZcsWVJVf/OIXTJw4kQ8//JD8/Hw2bNjAoEGDuOeee9i6dSt7i5aVVYE6UaZvjKka1Xn/rHfv3uTk5HD44YeTlZVFv379Ur6OX/3qV5x77rnk5OQUNn7RTDwdO3bkjjvu4IQTTkBVOe200zj11FP58MMPueiii1BVRIS7776bvLw8zjnnHHbt2kVBQQHXX389GRkZKd+G0tS4b+Tm5uaqfUTFmKqzYsUKunbtWt3JqBHy8vLIy8ujcePGrFq1ihNPPJFVq1ZRv37Nyh/HO2YislhVcxPMUqhmbYkxxlSj3bt3M3jwYPLy8lBVnnjiiRoX8Cuqbm2NMcZUQMuWLVm8eHF1J6NS2Y1cY4yJEAv6xhgTIRb0jTEmQizoG2NMhFjQN8ZUq4EDBxZ70eqBBx5g3LhxJc7XrFkzADZu3MiIESPiTnPCCSdQ2iPgDzzwQMxLUqecckpK6sW57bbbuO+++yq8nFSzoG+MqVajRo1i2rRpMcOmTZvGqFGjkpr/wAMP5KWXXir3+osG/TfffJOWLVuWe3k1nQV9Y0y1GjFiBG+88UbhB1PWrl3Lxo0b6d+/f+Fz871796Z79+68+uqrxeZfu3YtRxxxBAD79u1j5MiRdO3aleHDh7Nv377C6caNG1dYLfPvfvc7AB566CE2btzIwIEDGThwIADZ2dls3boVgEmTJnHEEUdwxBFHFFbLvHbtWrp27coll1xCt27dOPHEE2PWE8+SJUs45phj6NGjB8OHD2f79u2F6/erWvYrevvnP/9Z+BGZXr16sWvXrnLv23jsOX1jTKFrroFUfxCqZ0/w4mVcrVu3pk+fPrz11lsMGzaMadOmcfbZZyMiNG7cmJdffpnmzZuzdetWjjnmGE4//fSE34l97LHHSE9PZ8WKFSxdujSmauQ777yT1q1bk5+fz+DBg1m6dClXXXUVkyZNYs6cObRp0yZmWYsXL+bpp59mwYIFqCp9+/ZlwIABtGrVilWrVvH888/zxz/+kbPPPpvp06eXWD/+ueeey8MPP8yAAQO49dZbuf3223nggQe46667WLNmDY0aNSosUrrvvvuYPHky/fr1Y/fu3TRu3LgMe7t0ltM3xlS7cBFPuGhHVZkwYQI9evRgyJAhfPXVV2zevDnhcubOnVsYfHv06EGPHj0Kx7344ov07t2bXr16sXz58lIrU5s3bx7Dhw+nadOmNGvWjDPPPJP33nsPgC5dutCzZ0+g5OqbwdXvv2PHDgYMGADAeeedx9y5cwvTOHr0aKZMmVL45m+/fv247rrreOihh9ixY0fK3whOamkiMhR4EEgD/qSqdyWY7izgJeBoVV3kDbsRuAjIB65S1dRXjWeMSYmScuSVadiwYVx77bV8+OGH7N27l6OOOgqAqVOnsmXLFhYvXkyDBg3Izs6OW51yadasWcN9993HwoULadWqFeeff365luPzq2UGVzVzacU7ibzxxhvMnTuX1157jTvvvJNly5Yxfvx4Tj31VN5880369evHrFmzOPzww8ud1qJKzemLSBowGTgZyAFGiUhOnOkygKuBBaFhOcBIoBswFHjUW54xxhRq1qwZAwcO5MILL4y5gbtz504OOOAAGjRowJw5c1gX72PYIccffzzPPfccAB9//DFLly4FXLXMTZs2pUWLFmzevJm33nqrcJ6MjIy45eb9+/fnlVdeYe/evezZs4eXX36Z/v37l3nbWrRoQatWrQqvEv76178yYMAACgoKWL9+PQMHDuTuu+9m586d7N69m88//5zu3btzww03cPTRR/Ppp5+WeZ0lSSan3wdYrapfAIjINGAYUPTa6A7gbuA3oWHDgGmq+gOwRkRWe8v7d0UTboypW0aNGsXw4cNjnuQZPXo0p512Gt27dyc3N7fUHO+4ceO44IIL6Nq1K127di28YjjyyCPp1asXhx9+OJ06dYqplnns2LEMHTqUAw88kDlz5hQO7927N+effz59+vQB4OKLL6ZXr14lFuUk8swzz3DZZZexd+9eDjroIJ5++mny8/MZM2YMO3fuRFW56qqraNmyJbfccgtz5syhXr16dOvWrfArYKlSatXKIjICGKqqF3v9vwT6quqVoWl6Azep6lki8i5wvaouEpFHgPmqOsWb7s/AW6r6UpF1jAXGAnTu3Pmo0s7mxpjUsaqVa5+KVK1c4Ru5IlIPmAT8urzLUNUnVTVXVXPbtm1b0SQZY4xJIJnina+ATqH+jt4wXwZwBPCu9xhVe2CGiJyexLzGGGOqUDI5/YXAoSLSRUQa4m7MzvBHqupOVW2jqtmqmg3MB073nt6ZAYwUkUYi0gU4FPgg5VthjKmQmvYFPZNYRY9VqUFfVfOAK4FZwArgRVVdLiITvdx8SfMuB17E3fSdCVyhqlX/+XdjTEKNGzdm27ZtFvhrAVVl27ZtFXphy76Ra0zE7d+/nw0bNlTouXVTdRo3bkzHjh1p0KBBzHD7Rq4xJikNGjSgS5cu1Z0MU0WsGgZjjIkQC/rGGBMhFvSNMSZCLOgbY0yEWNA3xpgIsaBvjDERYkHfGGMixIK+McZEiAV9Y4yJEAv6xhgTIRb0jTEmQizoG2NMhFjQN8aYCLGgb4wxEWJB3xhjIsSCvjHGRIgFfWOMiRAL+sYYEyEW9I0xJkIs6BtjTIRY0DfGmAixoG+MMRFiQd8YYyLEgr4xxkSIBX1jjIkQC/rGGBMhFvSNMSZCLOgbY0yEWNA3xpgIsaBvjDERYkHfGGMixIK+McZEiAV9Y4yJkKSCvogMFZGVIrJaRMbHGX+ZiCwTkSUiMk9Ecrzh2SKyzxu+REQeT/UGGGOMSV790iYQkTRgMvAzYAOwUERmqOonocmeU9XHvelPByYBQ71xn6tqz9Qm2xhjTHkkk9PvA6xW1S9U9UdgGjAsPIGqfhfqbQpo6pJojDEmVZIJ+h2A9aH+Dd6wGCJyhYh8DtwDXBUa1UVE/iMi/xSR/vFWICJjRWSRiCzasmVLGZJvjDGmLFJ2I1dVJ6vqwcANwM3e4E1AZ1XtBVwHPCcizePM+6Sq5qpqbtu2bVOVJGOMMUUkE/S/AjqF+jt6wxKZBpwBoKo/qOo2r3sx8Dnw0/Il1RhjTEUlE/QXAoeKSBcRaQiMBGaEJxCRQ0O9pwKrvOFtvRvBiMhBwKHAF6lIuDHGmLIr9ekdVc0TkSuBWUAa8JSqLheRicAiVZ0BXCkiQ4D9wHbgPG/244GJIrIfKAAuU9VvK2NDjDHGlE5Ua9aDNrm5ubpo0aLqToYxxtQqIrJYVXNLm87eyDXGmAixoG+MMRFiQd8YYyLEgr4xxkSIBX1jjIkQC/rGGBMhFvSNMSZCLOgbY0yEWNA3xpgIsaBvjDERYkHfGGMixIK+McZEiAV9Y4yJEAv6xhgTIRb0jTEmQizoG2NMhFjQN8aYCLGgb4wxEWJB3xhjIsSCvjHGRIgFfWOMiRAL+sYYEyEW9I0xJkIs6BtjTIRY0DfGmAixoG+MMRFiQd8YYyLEgr4xxkSIBX1jjIkQC/rGGBMhFvSNMSZCLOgbY0yEWNA3xpgIsaBvjDERklTQF5GhIrJSRFaLyPg44y8TkWUiskRE5olITmjcjd58K0XkpFQm3hhjTNmUGvRFJA2YDJwM5ACjwkHd85yqdlfVnsA9wCRv3hxgJNANGAo86i3PGGNMNUgmp98HWK2qX6jqj8A0YFh4AlX9LtTbFFCvexgwTVV/UNU1wGpvecYYY6pB/SSm6QCsD/VvAPoWnUhErgCuAxoCg0Lzzi8yb4c4844FxgJ07tw5mXQbY4wph5TdyFXVyap6MHADcHMZ531SVXNVNbdt27apSpIxxpgikgn6XwGdQv0dvWGJTAPOKOe8xhhjKlEyQX8hcKiIdBGRhrgbszPCE4jIoaHeU4FVXvcMYKSINBKRLsChwAcVT7YxxpjyKLVMX1XzRORKYBaQBjylqstFZCKwSFVnAFeKyBBgP7AdOM+bd7mIvAh8AuQBV6hqfiVtizHGmFKIqpY+VRXKzc3VRYsWVXcyjDGmVhGRxaqaW9p09kauMcZEiAV9Y4yJEAv6xhgTIRb0jTEmQizoG2NMhFjQN8aYCLGgb4wxEWJB3xhjIsSCvjHGRIgFfWOMiRAL+sYYEyEW9I0xJkIs6BtjTIRY0DfGmAixoG+MMRFiQd8YYyLEgr4xxkSIBX1jjIkQC/rGGBMhFvSNMSZCLOgbY0yEWNA3xpgIsaBvjDERYkG/Bti4sbpTYIyJCgv61WzxYujQAT79tLpTYoyJAgv61WzTJtfevLl602GMiQYL+tXsxx9dOy+vetNhjIkGC/rVzA/6+/dXbzqMMdFQp4L+1KmQnQ316rn21KnVnaLSWdA3xlSl+tWdgFR5+mm47LIgiK5bB2PHuu7Ro6svXaWxoG+MqUp1Jqd/661BAPXt3Qs33VQ96UmWBX1jTFWqM0F/w4b4w7/8smrTUVYW9I0xVanOBP3Oncs2vKawp3eMMVUpqaAvIkNFZKWIrBaR8XHGXycin4jIUhF5R0SyQuPyRWSJ18xIZeLD/vCH4sPS0+HOOytrjalhOX1jTFUqNeiLSBowGTgZyAFGiUhOkcn+A+Sqag/gJeCe0Lh9qtrTa05PUbqLGT0amjaFjAwQgawsePLJmn0TFyzoG2OqVjI5/T7AalX9QlV/BKYBw8ITqOocVd3r9c4HOqY2mclp2RLOPhsKCmDt2pof8MGCvjGmaiUT9DsA60P9G7xhiVwEvBXqbywii0RkvoicUY40Jq1JE/fETm1iQd8YU5VS+py+iIwBcoEBocFZqvqViBwEzBaRZar6eZH5xgJjATpX4M5reroFfWOMKUkyOf2vgE6h/o7esBgiMgS4CThdVX/wh6vqV177C+BdoFfReVX1SVXNVdXctm3blmkDwtLTYd++5KZdtgy+/rrcq0oZe3rHGFOVkgn6C4FDRaSLiDQERgIxT+GISC/gCVzA/yY0vJWINPK62wD9gE9Slfii/Jx+MtUxDB8OEydWVkqSZzl9Y0xVKrV4R1XzRORKYBaQBjylqstFZCKwSFVnAPcCzYC/iQjAl96TOl2BJ0SkAHeCuUtVKy3oN2kCX3zhql/wi3kSVcfw7bewfXtlpSR5FvSNMVUpqTJ9VX0TeLPIsFtD3UMSzPc+0L0iCSyL9HT3Zm7RohK/OoZw0N+7F77/vqpSlpgFfWNMVaozb+SCC/qJysbD1TEUFMAPPyRf/l+ZLOgbY6pSnQr6TZq4F7PiCT8U5Af7mhT07UauMaYq1Kmgv24dqBYf3rBhbHUMfnm/Fe8YY6KmTgX999+PP7xoLrom5vQt6BtjqkKdCvo7d8YfXlDgnuDxH930c/oW9I0xUVOngn6rVonHhT+oYsU7xpioqlNBf/jwksevW+de1jrqKNe/Y0fyy/7nP+Fvfyt30hKyoG+MqUp1KugPHOja9RJslYgL/L7du5P/ePqkSXDzzRVLXzz29I4xpirVqaCfnu7av/990O0Tif9kz4QJyS17927Ys6di6YvHcvrGmKpUp4J+kyauPXCg+4BKVlbwQZV4AR/cS1vJ5PZrW9Dftav4h+KNMaZOBX0/d79vn6tyYe3a4IMqWVmJ5xs7Fi6/vORK2koK+q+9Bueck/jEUpLKCvr9+sHtt6d2mcaY2q9OBn3/6ZxwbZu7d7uXtOLZuxcefzx4ucuvpC0c+HfvdoH5xx+L1+I5YgQ8/zx8/nn85ZeksoL+2rWxVU8YYwzUsaDvF+/41SuPHRsE8m3bSi7uKJpLDz/iCS7oA/zlL7HLXbcuuAn7zjtlT3PRoJ9MtdAlGTIEnnjCpbe2fVDGGFP56lTQDxfv3HRTxYNeOKfsB/2JE4svt6DAtcsa9FWDYJ+XV/xEFe+KoyT5+TB7Nsyd6+a3oG+MKapOBv1du2IfzayINm3czWA/R/5VsW+GBd59t2zLDhfp7N8f/0RV9IojrOhVwR//6IL9xo3BvMYYE1angn7r1tC+Pdx4Y2qW5xcLJWvLlrI9bx8ubtq/P3EZfNHhU6e6k9GYMbFXBdde68Zv2uTaFvRNWRUUuG9SmLqrTgX9hg3hvfeCl7Sqw6WXljw+nDvv2jUY/tlniV8qC1cL7RcBxTsZ+dVKVGbQr+g9B1OzvfwyHHwwbN1a3SkxlaVOBX2AQw6BV1+Fgw6CQw9NXL9+ZXnqqcSPfxYtsy+ao8rPL7689PTYaqGTuVfx3XeuHe8ppooE6orecyhKFSZPDu6X1HRROOGtX++uQLdsqe6UmEqjqjWqOeqoozQV/uu/VAcPVp0yRdWFl6prRGL7GzRQzcxMfv60NLeMrCzVyZPdsP/7P7ddRZddWjqaNo0/fNy4su/TrKz468nKKt8xWrLEzf/ss+WbvypNmaKanh7/uPrHasoU1a+/rvy0PPWU6rRplbPsO+5w2/bBB5Wz/KoyZYo7JuFjU9fhvlleaoytczl9X7t28PXX7iWtli2rdt1FH//cv79s9wby812Rzp13QmamG3bjja4cvywvgKnGf6FMFR57zF2RlEWy9xyKSpRD9osQvvmm+LQiUL9+bLuyctc7d8Lq1SVPE+8Kyz+u/lXPxRe7e0offZT6NIZdeCGMHFk5uXH/qmvXrtp7ZZPqK9I6J5kzQ1U2qcrpjxvncmHPPlv1Of1UNg0bVv46mjYtnmMtasoUdwWS6IrCnz8zM3ZZ48YVzyH7Vytt2rh28+bBvKVtb3p62XJt4Rxf0bT5y/n5z92yv/su8XLKcoX1978nn77y8Nfzm9+Ubb5kcr9XXOGWfe21xY9bWfd9dUl0RRq+gq4N21FWJJnTTzoYV1WTqqB/++1u6w48MP4PoF69yg+mtb3xg3ZZiqaqoklLc3/a0gL6uHElB2t/nH8ye/rpxL+nRIEkXvPnP5f821y7VjU/P+gva1FEy5ZuPQMGlP4/CK8j0cnX3/6sLNX+/V23f0KO95uoavH2z+zZqs88E3/6ZE7Q8U5gVV0klOr1RT7oP/FExYNLo0Y1L+BZU3lN/fqJ/3hTpiSf27/vvmAe/2RRdN4ePRKPS0+Pf7LNzFT961+DDEvnzrHTZGbGpj8cVBJdpRVtSptOJCV/z6TFO1mF0xpve5M93n6gLek4VCQQz5ihumpV8ttV0fVFPui/+mqwI+Md8Hg3OOM1mZnuhl11ByRrqqapXz/2t+EXfSU7L7jiqurejoo2rVrFH+7n9JPJpRadZty45HK2ZTlZ+YGypJNDafOXNL68VzbhTEK8bS2pCKq8gT/yQX/+/JJ/vIMHux3s/1ETNSKqZ51V/X9Ca6ypCU2jRvEzTOHgmyjnnGge1bLNV7QpragbG32rAAAWrklEQVS2IkW55bmymTJFtUmT4ssKX4mVtI3lzfFHPuivXRvsxJ/+VLVZM9fdubNr33aby5GddFLJl4RZWarXX+8OYnZ2av441lhjTdCUNciXtWnQoPwPRKSlBVcofj8EufcpU8peBOzffyptmrJKNujX6Uc2/aqU+/WD8eNd99tvu3b79q5WzuxsVw3xlCnFv7ZVv757bHL3bsjIiK0rJ1E1zanQoEHy06anu+1LVv360LRp2dNkTGVRrdzl+1Wil0d+vnu82a/Ly3+Bct06Vw3KmDFlexwb3PSlzbNtW+U9Ylpng37jxvD3v0P//nDWWdClixu+YIFrt2vngp//rPjo0e5rW82bu/569Vw9Oh9/7IJ+s2axla2dcELJH2apX7986e7YsfQTSosWrt2smUvzo48G/fGE30q++eba8wasMVGWqKLFiqqzQR/g1FNdNcOnnhoE/X//27XbtYPjj3c5fz8XMHq0+wJWmzZBdcl33QWvvx68JAXQti388IO7QpgwwZ0g/PpxmjSBhx6CSy4JTgppaa6dlRXUCzR6tLt6AFdRnO/4490LVU2bBt8HCEtLg8suc91HH+2W45+4XnsteFGqcWPXnjEDbrstmL+gAJYsgQMOSGYPFte6deI6ghLJzHQnnszMYD9WdfUYxtQ2lfURpDod9MP8oD9/vmu3b++uAHbudHXQ+7Zvh1atgv6+fV0RyoMPBsPOPBMWLXKBf/ZsF/DXrXNXFX37ujeBH3sMFi50l655ee4Sc/r0IDd+wAFw0klw+OHuxALu6uCoo1z3z3/uqkouejWRnw/33ee6P/3Utf2g36ZNcCJp18619+2Db78N5p84Ea64wn1spTQNGgRBOiPDFYFt3BicEE8+OfZkGE+HDi59BQWuvXWr2yf5+eW/GipNZS23OmRmwrhx0KhRdafEVLVwRYupFJmg366dC94ffhj0/+xnLpi98kow3fbtLjc7c6aruG3uXFizBo49NgiUp5zicuOXX+5OIrfe6oY3b+4qO/Nf6feDMrgceW5uULz05ZeuwrWOHYMy/NatoU8f133qqcF3fu+4I1hOhw5BueKmTe6k5ZcPhoO+n5Pfu9dtUzhoLFmSuBw1HMQfe8wF6Z/9zL2Wf9ppru3r1CkI4vHuiYDbZnAnnvvui33NPy8vcZEUuCsa/wScleWCX9GTYOfOcNFFri3ixt99dzC+cWN3rNu2dVdP/jTjxsVPb6qEr2TS011/We+lZGW5/fvoo3DNNbHj/ON77bXJb0e7dtCzZ8n73NQMRStaTKXIBH0ROPJI152R4XZqo0ZwzDEuR+7bssUFvpNOgtNPd+Xrfhn7zJkudz9okBv21FMuZ37eecFyd+0KvpW7YoVrr18PL77ouv16ZtavLx70MzPdTdmZM10xk8+/zwBw/vmu7Rf9rFwZ5PRbt3bFP+npsUH/22+DKx1/2KuvulpIwwYNcsuaMsX1/9d/ue2dN8/1L18eez8gfALw74kceGCwLb16ueK0bdvgjDPgN7+B66+H3/42ODGOGpW4qOeyy9wJ69pr3cnv0Udd+xe/CKZ5/3131bV5s1vXLbcE+x1cddNZWe6K6qij3BXH7NkwfLj7rKR/kvRzVbfemvgE5mvQoPh9F7847ZBDXHvIELfOhg3hV79y69292+2XZIJ/erpL71tvuf5Fi2LH+1enn34K555b/GRYdB1+EeaSJcG2nXJKUPRYW6Sn192TVlpakCl58kn3n6oMkQn6EOSiw7nZI490N2v373c51s8/d9Uyx5OW5v7EzZq5ohxw5fd+GXdGhsvp+0H/0Ufdn/8vfym+rDVrXE69aE5fxJ1wwn/GcNAfPty1Tz7ZtT/91AXV5s2DQHTiiTB4sOv2g74fjH179wb3BsB99cv/3GOnTq69fr27ktm3z/X7N7V94e65c912T5zo+mfNgv/9X3eS69DBnTi6dXOB9t573YfowZ0MTjml+P4BdyMe3MnGp+qW1aGD6583z1Vw9sMPcPvt7v7Fn/7kxl14oTuRPfKIOwEtWOBOImPHun30zDPu5DlkiCuea9HCjfdPYB07Buv1b55nZcHTT7sTvj8sMzO42vOD/kcfuWk7dHD70U/77t3uCvHNN92wW26JLU7013H//a5u+0ceccOWLQsCef36QfHdW2/BCy+47zH4V7Fpae54h78r8fXXwfx+xuPNN91VY2lPi/kZjPJUXOhXlpcKrVq543LWWRVbTlpacGwbNIALLih+0hw+vHjRZWXfh8rPd5mDtWsrL+ADlPpMZ1U3qXpOP56pU90zsBkZwTC/6uVly1S3bHHd999f+rLmz1d98snYYb/+dfxnbg88UPWQQ1QPOsj1H3BAMO7xx1X/9jfXfdpp8dc1fXowfX6+6m9/q/rRR+5FmeuuUx09WrVLl9h58vLc9E2auOn++7+DZaSnu+eNt25140D1yy+Deb/4wg3705/cuurVc8u56irV998PljNwoJveT7//THTXrqr797txM2a4aq5nzVL9/HNXXwwE9cesX6+6fLnr9is+85+FPuKIYP/5VqxwwyZOjJ2nfXvVtm1j93tBQTDf4sXBsW3Y0K3DX49fjXB2tuopp7g0b9igOmpU8Az5cce59h13BMvs3dsN69kzOLZXXhms/5JLVI8/3jWqqjt2uOH33qu6erXrzs4Ontlu0yZY9htvuGHt2qnu3Om6Tz7ZtVu3dvs3vK2vvBLMc8MNwfAzznDt995Tbdw4+E3ccINbTv/+bj/4vwOIffmqc+fgWfRLLw1+4+HpizatW7v2f/6jOmGC+/106hSMb9bMLbd1a9UOHYLhkyYFxxWKP7//+utu3/j1avkvXYXf2s3IiH0Zq+iLWf609eqpHnyw637ySdW773b7NyfHHYfLLov9P/3nP27ayy8P1hNeblaW6gknJN4n4ef6S6q40P/flAepfDkLGAqsBFYD4+OMvw74BFgKvANkhcadB6zymvNKW1dlBv1Vq4Id7Fu2zPVPmaL673+77hkzyrd8v+77eM3o0arDhrnu00+P/SH/+c+u+7zz4i/3H/8Ifmhhxx2n2reve8Hs6KOLz+evY8iQIFiC6oknqp56qpumSxcXBPPygvl++MH9AC+6yJ0gRoxw6xk0yNXrH/7Rn3GGS8fBB6uec05y+88Plq1bB4F5zx43H7iX6Yruv2+/ddM98ojr//zz2Letr7+++DxhBQWuvht/3Kuvqs6cqfroo8E0vXoVX8bvfhe82Oc3n3yiunFjbIA88EDVm29WfeutYLo77nD7xD8h+zW+zpzpTt4XXhgbDEF11y437fjxwbBXXnHtP/xBC08U4eMLqsOHu5M0qK5Zo/rggy5AffSRG/Y//+PaDz2k+tlnbn/s3etOuued58b96leunZvr2iecEOybo492J1b/WwxPPRW7/uxsdyJo3961s7PdOvzfy6RJxffvSy+5Zb/3nusfPFj14ouD8ddc436n4UoTH3rIZT4yMlQPPVT1F7+I/8btAw+4DEGrVu7ECS6YDx5cfNrw8T3zTJfZOOOM2N/PH/8YG5z9/dmxY1B5nn/SLXrS8iuGGzxY9cYbXZoTvZA2d27J/52SpCzoA2nA58BBQEPgIyCnyDQDgXSvexzwgtfdGvjCa7fyuluVtL7KDPoFBW6LwznqH38MdnjPnsGfujy+/TZYlh/Y/NoKH3zQBQVwgeHww4Pgdc89rvvaa+Mvd8ECN75Tp9jh48e7nHVOjsulFOWnZeVK13/XXS4w7Nnj/vCqLhd+2GHF5/3JT9y8jRu7IHLRRS5I+1dLRQPzzTer7tunOm9ebA47nvPPd/MMGhQ73P/z+ydHcCc0cONUXXDzg174yip8NXTeeaqPPVZ8va+9FkyzY0fx8YMGuXEHHOBynJde6n4ffs2T/lXF448HAXbyZHdifPddt4w9e4I/9J//7HLUDRu6/X3EEa4J17BZtLqQJUtUhw6NDUYjRwb7AFS7d489vn7Q9vfV998Hy/d/k34u9N//Lr7deXkux+3nQP3tvOqqYBr/SrFePXdFvHy56/Z/B8OHqy5a5H6L4Xl3746ti+i001w7IyP4DX7zTfHg99OfBv+dm25yV4/+uLPPdr+Bo48O/rN+45+E581Tfeed2Ddx331X9ZZbXPfEie6EfMwxsfNPmOCC87HHxu6jceOCacKZqKuvDqb58MPi2+B3//a3rt28ecnVQowfX/z4JCuVQf9YYFao/0bgxhKm7wX8y+seBTwRGvcEMKqk9VVm0Fd1RRr79sUOO+OM2ErVio4vCz+Yb9ig+vbbQf3k8+er/vOfLphu2+YC49atbp6333bTzJoVf5mLFsX+2X3hIDZmTPH5/HElmT7d1dxYlF9c4X9ha/bsICCCK7II/1iXLSt5PWGTJrl5rrsudrh/1XXTTe6PNWFCUNQ0ebI7Li1auBOQqur27UHA+eQT112/vgvUiUyY4HLY8Qwc6Jbx+9/HDr/6ajd85kyXkx092q2zY0d3HMNXSf529O3rriwffjg2qL/wQvH1zp/vjj2o/vKXwfR33hlbFLBnjzsJ9+vn5vOH794de8UQVlAQW1wT72SnGlskdMklwUnL51959O0bDPvmm6Coxf+d7NzpAupXXwXTbdvmTljz57u0pqfHXtX6mTG/qVfPZXTS0tzx/vZbd3z9NNav7zIGJ54YBPVnnlFdt87Vk1WvnluPqgv8/nI/+8xdSc2aFWRM/C+FtWjh2s8+W7y4tKDAnRwOOcSlffp0N2zyZHfF5/Ov/vzG/w/Fa/r0cSeCgoKgqKxTJ9Ujj4x/fJKRyqA/AvhTqP+XwCMlTP8IcLPXfb3f7fXfAlwfZ56xwCJgUefOncu/1RUQvjSriO3bYy/RFi92ObUffih5Pr/4Ip7du92l7Jw5scO3bQuCQjjH4Zs1y+XIysP/IYbn94MBuLJvcJfgfllrsvyTXNH60L/+2g1/4olgWEGB2/acnKB45O23g/GbN7v98+OPLhjEu2pJlp+z9K8qfP/6lytm2LfPXZq3b+9y4UXLfePxi2bA/cYS8bfdzyH6JxL/Hki7dq4/Ozu4qmvZ0gUh1aBoslmz4ss+7LBguYls3hys/9ZbXXvhwmC8HxwvuSR2vqef1sKcc7I+/LD47336dPdRGAju4bz8cmxGaOfOoILEGTNi71O9/76b5rPPip9Y/fsae/YUT8vq1e4E8txzbpqPP3aZEXBX69OnByfsq68u+X/s32fx0+jfQwF39Zeb6zJN/j0eX1aWu8+yYEFwVV4e1RL0gTHAfKCRliHoh5vKzuknEi7mqU1efdUVB/zjH6ld7syZLqcZtmlTsI+6d9fC3G9Z5eW5snT/8j7snXeK/zn//vcgoHXpEls8Epab6y7Zy2v0aLeecPFIUY8+GuyD114rfZn+VVqPHiVPV1Dgtq1Nm9hA9/rrsb/LSZNcgFJ1Jzs/R6vqcsOLFxdftl9Z2B/+UHIaZs9295u+/tqVnYf388KFbhlFl++fwIs+1FAeGza4ZcW7P+UbNMgVRe3f775b4O+bNWsSz1NQ4K5Kklm/alDcCrHFQ/5+L8mSJcEJvHlzdxLaudPdA5k3z51U1q+PnadfP5fzr6gqL94BhgArgANCw2pc8U5J3n3X/UlNYqee6n41/lMqfhFVZSoocPcMOnRwN3IT2bLF/cHKa9++2GKJePbvdwHxmmtKPjn48vNdoE5mP+XnF78fUlDgcob33lv6/In4xQylbVt5bN3qytU//jg1y8vKcg8OJLJhQ/BhkjVrgoBckSLZop55JlguuIzEv/6VOLMRz2OPuZvoyfjoI9WlS8uX1rBkg764aRMTkfrAZ8Bg4CtgIXCOqi4PTdMLeAkYqqqrQsNbA4uB3t6gD4GjVDVUMUCs3NxcXVT0TRRTYxQUwI4d7pnvTZtcLaWmZvvkE9eMGFHdKSndsmXunZOSKjMM85+dLyWMlclLL7kXAE8+2b178vzz0Lt36fNVNxFZrKq5pU1Xai0lqponIlcCs3BP8jylqstFZCLuzDIDuBdoBvxN3FH4UlVPV9VvReQO3IkCYGJJAd/UfPXqBRXEWcCvHXJyXFMbdO9etulnz3ZvpafS6ae7Fwgvv7xyq+qoLqXm9Kua5fSNMabsks3pR6oaBmOMiToL+sYYEyEW9I0xJkIs6BtjTIRY0DfGmAixoG+MMRFiQd8YYyLEgr4xxkRIjXs5S0S2AOsqsIg2wNYUJae61ZVtqSvbAbYtNZVti/t4VdvSJqpxQb+iRGRRMm+l1QZ1ZVvqynaAbUtNZduSPCveMcaYCLGgb4wxEVIXg/6T1Z2AFKor21JXtgNsW2oq25Yk1bkyfWOMMYnVxZy+McaYBCzoG2NMhNSZoC8iQ0VkpYisFpHx1Z2eshKRtSKyTESWiMgib1hrEfmHiKzy2q2qO53xiMhTIvKNiHwcGhY37eI85B2npSJSoz5El2BbbhORr7xjs0RETgmNu9HblpUiclL1pDo+EekkInNE5BMRWS4iV3vDa9WxKWE7at1xEZHGIvKBiHzkbcvt3vAuIrLAS/MLItLQG97I61/tjc+ucCKS+ZBuTW9wn3H8HDgIaAh8BORUd7rKuA1rgTZFht0DjPe6xwN3V3c6E6T9eNx3kD8uLe3AKcBbgADHAAuqO/1JbMttwPVxps3xfmuNgC7ebzCturchlL6fAL297gzct65zatuxKWE7at1x8fZtM6+7AbDA29cvAiO94Y8D47zuy4HHve6RwAsVTUNdyen3AVar6heq+iMwDRhWzWlKhWHAM173M8AZ1ZiWhFR1LlD028eJ0j4MeFad+UBLEflJ1aS0dAm2JZFhwDRV/UFV1wCrcb/FGkFVN6nqh173LmAF0IFadmxK2I5Eauxx8fbtbq+3gdcoMAh4yRte9Jj4x+olYLCI/zn48qkrQb8DsD7Uv4GSfxQ1kQL/JyKLRWSsN6ydqm7yur8G2lVP0solUdpr67G60ivyeCpUzFZrtsUrFuiFy1nW2mNTZDugFh4XEUkTkSXAN8A/cFciO1Q1z5sknN7CbfHG7wQyK7L+uhL064LjVLU3cDJwhYgcHx6p7vquVj5fW5vT7nkMOBjoCWwC/rd6k1M2ItIMmA5co6rfhcfVpmMTZztq5XFR1XxV7Ql0xF2BHF6V668rQf8roFOov6M3rNZQ1a+89jfAy7gfw2b/8tprf1N9KSyzRGmvdcdKVTd7f9QC4I8ERQU1fltEpAEuUE5V1b97g2vdsYm3HbX5uACo6g5gDnAsriitvjcqnN7CbfHGtwC2VWS9dSXoLwQO9e6AN8Td8JhRzWlKmog0FZEMvxs4EfgYtw3neZOdB7xaPSksl0RpnwGc6z0pcgywM1TUUCMVKdcejjs24LZlpPeERRfgUOCDqk5fIl7Z75+BFao6KTSqVh2bRNtRG4+LiLQVkZZedxPgZ7h7FHOAEd5kRY+Jf6xGALO9q7Pyq+672alqcE8efIYrH7uputNTxrQfhHva4CNguZ9+XNndO8Aq4G2gdXWnNUH6n8ddXu/HlUdelCjtuKcXJnvHaRmQW93pT2Jb/uqldan3J/xJaPqbvG1ZCZxc3ekvsi3H4YpulgJLvOaU2nZsStiOWndcgB7Af7w0fwzc6g0/CHdiWg38DWjkDW/s9a/2xh9U0TRYNQzGGBMhdaV4xxhjTBIs6BtjTIRY0DfGmAixoG+MMRFiQd8YYyLEgr4xxkSIBX1jjImQ/wcM3wNWKBQxQwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PELM_s Result\n",
      "Accuracy : 0.9209677419354839\n",
      "AUC : 0.9201613239401393\n",
      "Sensitivity : 0.9409937888198758\n",
      "Specificity : 0.8993288590604027\n",
      "F1 : 0.9251908396946565\n",
      "MCC : 0.8420135324127758\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Plot the training accuracy model Dense\n",
    "\n",
    "accuracy = model_Dense_train.history['acc']\n",
    "val_accuracy = model_Dense_train.history['val_acc']\n",
    "loss = model_Dense_train.history['loss']\n",
    "val_loss = model_Dense_train.history['val_loss']\n",
    "epochs = range(len(accuracy))\n",
    "plt.plot(epochs, accuracy, 'bo', label='Training accuracy')\n",
    "plt.plot(epochs, val_accuracy, 'b', label='Validation accuracy')\n",
    "plt.title('Dense Training and validation accuracy')\n",
    "plt.legend()\n",
    "plt.figure()\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Dense Training and validation loss')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "y_pred = np.argmax(model_Dense.predict(valid_X), axis=1)\n",
    "y_true = np.argmax(valid_Y, axis = 1)\n",
    "\n",
    "tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()\n",
    "f1 = f1_score(y_true, y_pred, labels=None, pos_label=1, average='binary', sample_weight=None)\n",
    "auc = roc_auc_score(y_true, y_pred, average='macro', sample_weight=None, max_fpr=None)\n",
    "sensi = tp/(tp+fn)\n",
    "specificity = tn/(tn+fp)\n",
    "accu = (tn + tp)/(tn + tp + fn + fp)\n",
    "mcc = ((tp*tn)-(fp*fn))/np.sqrt((tp+fp)*(tp+fn)*(tn+fp)*(tn+fn))\n",
    "print('{} Result'.format(string_name))\n",
    "print('Accuracy :', accu)\n",
    "print('AUC :', auc)\n",
    "print('Sensitivity :', sensi)\n",
    "print('Specificity :', specificity)\n",
    "print('F1 :', f1)\n",
    "print('MCC :', mcc)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# library Transformer\n",
    "\n",
    "def GetPosEncodingMatrix(max_len, d_emb):\n",
    "    pos_enc = np.array([\n",
    "        [pos / np.power(10000, 2 * (j // 2) / d_emb) for j in range(d_emb)] \n",
    "        if pos != 0 else np.zeros(d_emb) \n",
    "            for pos in range(max_len)\n",
    "            ])\n",
    "    pos_enc[1:, 0::2] = np.sin(pos_enc[1:, 0::2]) # dim 2i\n",
    "    pos_enc[1:, 1::2] = np.cos(pos_enc[1:, 1::2]) # dim 2i+1\n",
    "    return pos_enc\n",
    "\n",
    "class LayerNormalization(Layer):\n",
    "\tdef __init__(self, eps=1e-6, **kwargs):\n",
    "\t\tself.eps = eps\n",
    "\t\tsuper(LayerNormalization, self).__init__(**kwargs)\n",
    "\tdef build(self, input_shape):\n",
    "\t\tself.gamma = self.add_weight(name='gamma', shape=input_shape[-1:],\n",
    "\t\t\t\t\t\t\t\t\t initializer=Ones(), trainable=True)\n",
    "\t\tself.beta = self.add_weight(name='beta', shape=input_shape[-1:],\n",
    "\t\t\t\t\t\t\t\t\tinitializer=Zeros(), trainable=True)\n",
    "\t\tsuper(LayerNormalization, self).build(input_shape)\n",
    "\tdef call(self, x):\n",
    "\t\tmean = K.mean(x, axis=-1, keepdims=True)\n",
    "\t\tstd = K.std(x, axis=-1, keepdims=True)\n",
    "\t\treturn self.gamma * (x - mean) / (std + self.eps) + self.beta\n",
    "\tdef compute_output_shape(self, input_shape):\n",
    "\t\treturn input_shape\n",
    "    \n",
    "class ScaledDotProductAttention():\n",
    "\tdef __init__(self, d_model, attn_dropout=0.1):\n",
    "\t\tself.temper = np.sqrt(d_model)\n",
    "\t\tself.dropout = Dropout(attn_dropout)\n",
    "\tdef __call__(self, q, k, v, mask):\n",
    "\t\tattn = Lambda(lambda x:K.batch_dot(x[0],x[1],axes=[2,2])/self.temper)([q, k])\n",
    "\t\tif mask is not None:\n",
    "\t\t\tmmask = Lambda(lambda x:(-1e+10)*(1-x))(mask)\n",
    "\t\t\tattn = Add()([attn, mmask])\n",
    "\t\tattn = Activation('softmax')(attn)\n",
    "\t\tattn = self.dropout(attn)\n",
    "\t\toutput = Lambda(lambda x:K.batch_dot(x[0], x[1]))([attn, v])\n",
    "\t\treturn output, attn\n",
    "    \n",
    "class MultiHeadAttention():\n",
    "\t# mode 0 - big martixes, faster; mode 1 - more clear implementation\n",
    "\tdef __init__(self, n_head, d_model, d_k, d_v, dropout, mode=0, use_norm=True):\n",
    "\t\tself.mode = mode\n",
    "\t\tself.n_head = n_head\n",
    "\t\tself.d_k = d_k\n",
    "\t\tself.d_v = d_v\n",
    "\t\tself.dropout = dropout\n",
    "\t\tif mode == 0:\n",
    "\t\t\tself.qs_layer = Dense(n_head*d_k, use_bias=False)\n",
    "\t\t\tself.ks_layer = Dense(n_head*d_k, use_bias=False)\n",
    "\t\t\tself.vs_layer = Dense(n_head*d_v, use_bias=False)\n",
    "\t\telif mode == 1:\n",
    "\t\t\tself.qs_layers = []\n",
    "\t\t\tself.ks_layers = []\n",
    "\t\t\tself.vs_layers = []\n",
    "\t\t\tfor _ in range(n_head):\n",
    "\t\t\t\tself.qs_layers.append(TimeDistributed(Dense(d_k, use_bias=False)))\n",
    "\t\t\t\tself.ks_layers.append(TimeDistributed(Dense(d_k, use_bias=False)))\n",
    "\t\t\t\tself.vs_layers.append(TimeDistributed(Dense(d_v, use_bias=False)))\n",
    "\t\tself.attention = ScaledDotProductAttention(d_model)\n",
    "\t\tself.layer_norm = LayerNormalization() if use_norm else None\n",
    "\t\tself.w_o = TimeDistributed(Dense(d_model))\n",
    "\n",
    "\tdef __call__(self, q, k, v, mask=None):\n",
    "\t\td_k, d_v = self.d_k, self.d_v\n",
    "\t\tn_head = self.n_head\n",
    "\n",
    "\t\tif self.mode == 0:\n",
    "\t\t\tqs = self.qs_layer(q)  # [batch_size, len_q, n_head*d_k]\n",
    "\t\t\tks = self.ks_layer(k)\n",
    "\t\t\tvs = self.vs_layer(v)\n",
    "\n",
    "\t\t\tdef reshape1(x):\n",
    "\t\t\t\ts = tf.shape(x)   # [batch_size, len_q, n_head * d_k]\n",
    "\t\t\t\tx = tf.reshape(x, [s[0], s[1], n_head, s[2]//n_head])\n",
    "\t\t\t\tx = tf.transpose(x, [2, 0, 1, 3])  \n",
    "\t\t\t\tx = tf.reshape(x, [-1, s[1], s[2]//n_head])  # [n_head * batch_size, len_q, d_k]\n",
    "\t\t\t\treturn x\n",
    "\t\t\tqs = Lambda(reshape1)(qs)\n",
    "\t\t\tks = Lambda(reshape1)(ks)\n",
    "\t\t\tvs = Lambda(reshape1)(vs)\n",
    "\n",
    "\t\t\tif mask is not None:\n",
    "\t\t\t\tmask = Lambda(lambda x:K.repeat_elements(x, n_head, 0))(mask)\n",
    "\t\t\thead, attn = self.attention(qs, ks, vs, mask=mask)  \n",
    "\t\t\t\t\n",
    "\t\t\tdef reshape2(x):\n",
    "\t\t\t\ts = tf.shape(x)   # [n_head * batch_size, len_v, d_v]\n",
    "\t\t\t\tx = tf.reshape(x, [n_head, -1, s[1], s[2]]) \n",
    "\t\t\t\tx = tf.transpose(x, [1, 2, 0, 3])\n",
    "\t\t\t\tx = tf.reshape(x, [-1, s[1], n_head*d_v])  # [batch_size, len_v, n_head * d_v]\n",
    "\t\t\t\treturn x\n",
    "\t\t\thead = Lambda(reshape2)(head)\n",
    "\t\telif self.mode == 1:\n",
    "\t\t\theads = []; attns = []\n",
    "\t\t\tfor i in range(n_head):\n",
    "\t\t\t\tqs = self.qs_layers[i](q)   \n",
    "\t\t\t\tks = self.ks_layers[i](k) \n",
    "\t\t\t\tvs = self.vs_layers[i](v) \n",
    "\t\t\t\thead, attn = self.attention(qs, ks, vs, mask)\n",
    "\t\t\t\theads.append(head); attns.append(attn)\n",
    "\t\t\thead = Concatenate()(heads) if n_head > 1 else heads[0]\n",
    "\t\t\tattn = Concatenate()(attns) if n_head > 1 else attns[0]\n",
    "\n",
    "\t\toutputs = self.w_o(head)\n",
    "\t\toutputs = Dropout(self.dropout)(outputs)\n",
    "\t\tif not self.layer_norm: return outputs, attn\n",
    "\t\toutputs = Add()([outputs, q])\n",
    "\t\treturn self.layer_norm(outputs), attn\n",
    "\n",
    "class PositionwiseFeedForward():\n",
    "    def __init__(self, d_hid, d_inner_hid, dropout=0.1):\n",
    "        self.w_1 = Conv1D(d_inner_hid, 1, activation='relu')\n",
    "        self.w_2 = Conv1D(d_hid, 1)\n",
    "        self.layer_norm = LayerNormalization()\n",
    "        self.dropout = Dropout(dropout)\n",
    "    def __call__(self, x):\n",
    "        output = self.w_1(x) \n",
    "        output = self.w_2(output)\n",
    "        output = self.dropout(output)\n",
    "        output = Add()([output, x])\n",
    "        output = self.layer_norm(output)\n",
    "        return output\n",
    "    \n",
    "class EncoderLayer():\n",
    "    def __init__(self, d_model, d_inner_hid, n_head, d_k, d_v, dropout=0.1):\n",
    "        self.self_att_layer = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)\n",
    "        self.pos_ffn_layer  = PositionwiseFeedForward(d_model, d_inner_hid, dropout=dropout)\n",
    "    def __call__(self, enc_input, mask=None):\n",
    "        output, slf_attn = self.self_att_layer(enc_input, enc_input, enc_input, mask=mask)\n",
    "        output = self.pos_ffn_layer(output)\n",
    "        return output, slf_attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Transformer\n",
    "\n",
    "epochs = 100\n",
    "d_model = 16\n",
    "d_emb = d_model\n",
    "d_inner_hid = 2*d_model\n",
    "n_head = 4\n",
    "d_k = d_model//4\n",
    "d_v = d_model//4\n",
    "dropout = 0.5\n",
    "\n",
    "inp = Input(shape=(9,))\n",
    "\n",
    "emb = Embedding(20, d_emb, input_length=9)(inp)\n",
    "\n",
    "out, slf_attn = EncoderLayer(d_emb, d_inner_hid, n_head, d_k, d_v, dropout)(emb)\n",
    "out, slf_attn = EncoderLayer(d_emb, d_inner_hid, n_head, d_k, d_v, dropout)(out)\n",
    "out, slf_attn = EncoderLayer(d_emb, d_inner_hid, n_head, d_k, d_v, dropout)(out)\n",
    "out, slf_attn = EncoderLayer(d_emb, d_inner_hid, n_head, d_k, d_v, dropout)(out)\n",
    "out, slf_attn = EncoderLayer(d_emb, d_inner_hid, n_head, d_k, d_v, dropout)(out)\n",
    "out, slf_attn = EncoderLayer(d_emb, d_inner_hid, n_head, d_k, d_v, dropout)(out)\n",
    "out = GlobalAveragePooling1D()(out)\n",
    "\n",
    "out = Dense(2, activation='softmax')(out)\n",
    "\n",
    "model_transformer = Model(inputs=inp, outputs=out)\n",
    "model_transformer.compile(loss='categorical_crossentropy', optimizer='sgd',metrics=['accuracy'])\n",
    "\n",
    "# model.summary()\n",
    "# Callback\n",
    "\n",
    "def step_decay(epoch):\n",
    "   if (0 <= epoch <= 30):\n",
    "    lrate = 1e-1\n",
    "   elif (30 < epoch <= 60):\n",
    "    lrate = 1e-2\n",
    "   elif (60 < epoch):\n",
    "    lrate = 1e-3\n",
    "\n",
    "   return lrate\n",
    "lrate = LearningRateScheduler(step_decay, verbose=1)\n",
    "callbacks_list = [lrate]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 2477 samples, validate on 620 samples\n",
      "Epoch 1/100\n",
      "\n",
      "Epoch 00001: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 61s 25ms/step - loss: 0.7293 - acc: 0.5188 - val_loss: 0.6966 - val_acc: 0.4806\n",
      "Epoch 2/100\n",
      "\n",
      "Epoch 00002: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.7072 - acc: 0.5176 - val_loss: 0.7320 - val_acc: 0.5194\n",
      "Epoch 3/100\n",
      "\n",
      "Epoch 00003: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.7071 - acc: 0.5079 - val_loss: 0.6961 - val_acc: 0.4806\n",
      "Epoch 4/100\n",
      "\n",
      "Epoch 00004: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.6453 - acc: 0.6193 - val_loss: 0.4909 - val_acc: 0.8339\n",
      "Epoch 5/100\n",
      "\n",
      "Epoch 00005: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.4238 - acc: 0.8280 - val_loss: 0.3407 - val_acc: 0.8855\n",
      "Epoch 6/100\n",
      "\n",
      "Epoch 00006: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.3431 - acc: 0.8652 - val_loss: 0.3603 - val_acc: 0.8823\n",
      "Epoch 7/100\n",
      "\n",
      "Epoch 00007: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.3269 - acc: 0.8736 - val_loss: 0.3531 - val_acc: 0.8887\n",
      "Epoch 8/100\n",
      "\n",
      "Epoch 00008: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.3211 - acc: 0.8724 - val_loss: 0.5993 - val_acc: 0.7468\n",
      "Epoch 9/100\n",
      "\n",
      "Epoch 00009: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.3061 - acc: 0.8833 - val_loss: 0.4382 - val_acc: 0.7726\n",
      "Epoch 10/100\n",
      "\n",
      "Epoch 00010: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.3021 - acc: 0.8793 - val_loss: 0.3684 - val_acc: 0.8452\n",
      "Epoch 11/100\n",
      "\n",
      "Epoch 00011: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2930 - acc: 0.8898 - val_loss: 0.2795 - val_acc: 0.8887\n",
      "Epoch 12/100\n",
      "\n",
      "Epoch 00012: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2909 - acc: 0.8821 - val_loss: 0.3539 - val_acc: 0.8677\n",
      "Epoch 13/100\n",
      "\n",
      "Epoch 00013: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2909 - acc: 0.8837 - val_loss: 0.3046 - val_acc: 0.8871\n",
      "Epoch 14/100\n",
      "\n",
      "Epoch 00014: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2903 - acc: 0.8878 - val_loss: 0.2724 - val_acc: 0.8871\n",
      "Epoch 15/100\n",
      "\n",
      "Epoch 00015: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2925 - acc: 0.8809 - val_loss: 0.3172 - val_acc: 0.8790\n",
      "Epoch 16/100\n",
      "\n",
      "Epoch 00016: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2846 - acc: 0.8918 - val_loss: 0.2850 - val_acc: 0.8839\n",
      "Epoch 17/100\n",
      "\n",
      "Epoch 00017: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2823 - acc: 0.8890 - val_loss: 0.3085 - val_acc: 0.8952\n",
      "Epoch 18/100\n",
      "\n",
      "Epoch 00018: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2821 - acc: 0.8825 - val_loss: 0.2952 - val_acc: 0.8919\n",
      "Epoch 19/100\n",
      "\n",
      "Epoch 00019: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2823 - acc: 0.8870 - val_loss: 0.2728 - val_acc: 0.8806\n",
      "Epoch 20/100\n",
      "\n",
      "Epoch 00020: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2811 - acc: 0.8849 - val_loss: 0.2560 - val_acc: 0.8984\n",
      "Epoch 21/100\n",
      "\n",
      "Epoch 00021: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2704 - acc: 0.8890 - val_loss: 0.2576 - val_acc: 0.8887\n",
      "Epoch 22/100\n",
      "\n",
      "Epoch 00022: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2780 - acc: 0.8825 - val_loss: 0.2510 - val_acc: 0.8887\n",
      "Epoch 23/100\n",
      "\n",
      "Epoch 00023: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2827 - acc: 0.8870 - val_loss: 0.2771 - val_acc: 0.8855\n",
      "Epoch 24/100\n",
      "\n",
      "Epoch 00024: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2808 - acc: 0.8886 - val_loss: 0.2636 - val_acc: 0.8855\n",
      "Epoch 25/100\n",
      "\n",
      "Epoch 00025: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2737 - acc: 0.8829 - val_loss: 0.2740 - val_acc: 0.8984\n",
      "Epoch 26/100\n",
      "\n",
      "Epoch 00026: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2808 - acc: 0.8902 - val_loss: 0.2813 - val_acc: 0.8855\n",
      "Epoch 27/100\n",
      "\n",
      "Epoch 00027: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2760 - acc: 0.8845 - val_loss: 0.2615 - val_acc: 0.8855\n",
      "Epoch 28/100\n",
      "\n",
      "Epoch 00028: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2656 - acc: 0.8894 - val_loss: 0.2559 - val_acc: 0.8871\n",
      "Epoch 29/100\n",
      "\n",
      "Epoch 00029: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2689 - acc: 0.8874 - val_loss: 0.2449 - val_acc: 0.8839\n",
      "Epoch 30/100\n",
      "\n",
      "Epoch 00030: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2672 - acc: 0.8910 - val_loss: 0.2510 - val_acc: 0.8855\n",
      "Epoch 31/100\n",
      "\n",
      "Epoch 00031: LearningRateScheduler setting learning rate to 0.1.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2750 - acc: 0.8902 - val_loss: 0.3520 - val_acc: 0.8452\n",
      "Epoch 32/100\n",
      "\n",
      "Epoch 00032: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2678 - acc: 0.8902 - val_loss: 0.2495 - val_acc: 0.9000\n",
      "Epoch 33/100\n",
      "\n",
      "Epoch 00033: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2589 - acc: 0.8979 - val_loss: 0.2475 - val_acc: 0.8968\n",
      "Epoch 34/100\n",
      "\n",
      "Epoch 00034: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2561 - acc: 0.8942 - val_loss: 0.2551 - val_acc: 0.9000\n",
      "Epoch 35/100\n",
      "\n",
      "Epoch 00035: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2606 - acc: 0.8934 - val_loss: 0.2482 - val_acc: 0.8984\n",
      "Epoch 36/100\n",
      "\n",
      "Epoch 00036: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2597 - acc: 0.8938 - val_loss: 0.2495 - val_acc: 0.8984\n",
      "Epoch 37/100\n",
      "\n",
      "Epoch 00037: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2583 - acc: 0.8950 - val_loss: 0.2579 - val_acc: 0.8984\n",
      "Epoch 38/100\n",
      "\n",
      "Epoch 00038: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2587 - acc: 0.8906 - val_loss: 0.2529 - val_acc: 0.9016\n",
      "Epoch 39/100\n",
      "\n",
      "Epoch 00039: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2556 - acc: 0.8918 - val_loss: 0.2555 - val_acc: 0.9000\n",
      "Epoch 40/100\n",
      "\n",
      "Epoch 00040: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2595 - acc: 0.8934 - val_loss: 0.2502 - val_acc: 0.8968\n",
      "Epoch 41/100\n",
      "\n",
      "Epoch 00041: LearningRateScheduler setting learning rate to 0.01.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2559 - acc: 0.8926 - val_loss: 0.2529 - val_acc: 0.8984\n",
      "Epoch 42/100\n",
      "\n",
      "Epoch 00042: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2589 - acc: 0.8878 - val_loss: 0.2533 - val_acc: 0.9000\n",
      "Epoch 43/100\n",
      "\n",
      "Epoch 00043: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2563 - acc: 0.8930 - val_loss: 0.2491 - val_acc: 0.8935\n",
      "Epoch 44/100\n",
      "\n",
      "Epoch 00044: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2571 - acc: 0.8938 - val_loss: 0.2509 - val_acc: 0.9000\n",
      "Epoch 45/100\n",
      "\n",
      "Epoch 00045: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2565 - acc: 0.8902 - val_loss: 0.2559 - val_acc: 0.8984\n",
      "Epoch 46/100\n",
      "\n",
      "Epoch 00046: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2567 - acc: 0.8946 - val_loss: 0.2547 - val_acc: 0.9000\n",
      "Epoch 47/100\n",
      "\n",
      "Epoch 00047: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2560 - acc: 0.8934 - val_loss: 0.2566 - val_acc: 0.8968\n",
      "Epoch 48/100\n",
      "\n",
      "Epoch 00048: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2542 - acc: 0.8918 - val_loss: 0.2513 - val_acc: 0.8984\n",
      "Epoch 49/100\n",
      "\n",
      "Epoch 00049: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2569 - acc: 0.8882 - val_loss: 0.2616 - val_acc: 0.8968\n",
      "Epoch 50/100\n",
      "\n",
      "Epoch 00050: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2552 - acc: 0.8906 - val_loss: 0.2570 - val_acc: 0.8968\n",
      "Epoch 51/100\n",
      "\n",
      "Epoch 00051: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2547 - acc: 0.8922 - val_loss: 0.2515 - val_acc: 0.8919\n",
      "Epoch 52/100\n",
      "\n",
      "Epoch 00052: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2551 - acc: 0.8922 - val_loss: 0.2632 - val_acc: 0.8919\n",
      "Epoch 53/100\n",
      "\n",
      "Epoch 00053: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2615 - acc: 0.8870 - val_loss: 0.2480 - val_acc: 0.8935\n",
      "Epoch 54/100\n",
      "\n",
      "Epoch 00054: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2546 - acc: 0.8946 - val_loss: 0.2613 - val_acc: 0.8919\n",
      "Epoch 55/100\n",
      "\n",
      "Epoch 00055: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2546 - acc: 0.8966 - val_loss: 0.2551 - val_acc: 0.8984\n",
      "Epoch 56/100\n",
      "\n",
      "Epoch 00056: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2538 - acc: 0.8987 - val_loss: 0.2583 - val_acc: 0.8952\n",
      "Epoch 57/100\n",
      "\n",
      "Epoch 00057: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2579 - acc: 0.8922 - val_loss: 0.2489 - val_acc: 0.8952\n",
      "Epoch 58/100\n",
      "\n",
      "Epoch 00058: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2575 - acc: 0.8926 - val_loss: 0.2529 - val_acc: 0.8984\n",
      "Epoch 59/100\n",
      "\n",
      "Epoch 00059: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2519 - acc: 0.8918 - val_loss: 0.2584 - val_acc: 0.8984\n",
      "Epoch 60/100\n",
      "\n",
      "Epoch 00060: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2579 - acc: 0.8918 - val_loss: 0.2542 - val_acc: 0.9000\n",
      "Epoch 61/100\n",
      "\n",
      "Epoch 00061: LearningRateScheduler setting learning rate to 0.01.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2570 - acc: 0.8938 - val_loss: 0.2569 - val_acc: 0.8968\n",
      "Epoch 62/100\n",
      "\n",
      "Epoch 00062: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2537 - acc: 0.8922 - val_loss: 0.2561 - val_acc: 0.8984\n",
      "Epoch 63/100\n",
      "\n",
      "Epoch 00063: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2570 - acc: 0.8886 - val_loss: 0.2549 - val_acc: 0.9016\n",
      "Epoch 64/100\n",
      "\n",
      "Epoch 00064: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2541 - acc: 0.8918 - val_loss: 0.2539 - val_acc: 0.9016\n",
      "Epoch 65/100\n",
      "\n",
      "Epoch 00065: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2531 - acc: 0.8950 - val_loss: 0.2528 - val_acc: 0.9016\n",
      "Epoch 66/100\n",
      "\n",
      "Epoch 00066: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2507 - acc: 0.8950 - val_loss: 0.2528 - val_acc: 0.9016\n",
      "Epoch 67/100\n",
      "\n",
      "Epoch 00067: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2486 - acc: 0.8930 - val_loss: 0.2530 - val_acc: 0.9016\n",
      "Epoch 68/100\n",
      "\n",
      "Epoch 00068: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2551 - acc: 0.8902 - val_loss: 0.2529 - val_acc: 0.9016\n",
      "Epoch 69/100\n",
      "\n",
      "Epoch 00069: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2501 - acc: 0.8922 - val_loss: 0.2536 - val_acc: 0.9016\n",
      "Epoch 70/100\n",
      "\n",
      "Epoch 00070: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2510 - acc: 0.8950 - val_loss: 0.2530 - val_acc: 0.9016\n",
      "Epoch 71/100\n",
      "\n",
      "Epoch 00071: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2461 - acc: 0.8930 - val_loss: 0.2529 - val_acc: 0.9016\n",
      "Epoch 72/100\n",
      "\n",
      "Epoch 00072: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2568 - acc: 0.8918 - val_loss: 0.2527 - val_acc: 0.9016\n",
      "Epoch 73/100\n",
      "\n",
      "Epoch 00073: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2546 - acc: 0.8930 - val_loss: 0.2518 - val_acc: 0.9016\n",
      "Epoch 74/100\n",
      "\n",
      "Epoch 00074: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2529 - acc: 0.8914 - val_loss: 0.2525 - val_acc: 0.9016\n",
      "Epoch 75/100\n",
      "\n",
      "Epoch 00075: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2510 - acc: 0.8946 - val_loss: 0.2527 - val_acc: 0.9016\n",
      "Epoch 76/100\n",
      "\n",
      "Epoch 00076: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2496 - acc: 0.8934 - val_loss: 0.2534 - val_acc: 0.9016\n",
      "Epoch 77/100\n",
      "\n",
      "Epoch 00077: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2505 - acc: 0.8894 - val_loss: 0.2529 - val_acc: 0.9016\n",
      "Epoch 78/100\n",
      "\n",
      "Epoch 00078: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2503 - acc: 0.8942 - val_loss: 0.2536 - val_acc: 0.9000\n",
      "Epoch 79/100\n",
      "\n",
      "Epoch 00079: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2541 - acc: 0.8930 - val_loss: 0.2525 - val_acc: 0.9016\n",
      "Epoch 80/100\n",
      "\n",
      "Epoch 00080: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2551 - acc: 0.8938 - val_loss: 0.2527 - val_acc: 0.9016\n",
      "Epoch 81/100\n",
      "\n",
      "Epoch 00081: LearningRateScheduler setting learning rate to 0.001.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2515 - acc: 0.8946 - val_loss: 0.2529 - val_acc: 0.9000\n",
      "Epoch 82/100\n",
      "\n",
      "Epoch 00082: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2555 - acc: 0.8922 - val_loss: 0.2528 - val_acc: 0.9000\n",
      "Epoch 83/100\n",
      "\n",
      "Epoch 00083: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2548 - acc: 0.8930 - val_loss: 0.2527 - val_acc: 0.9000\n",
      "Epoch 84/100\n",
      "\n",
      "Epoch 00084: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2534 - acc: 0.8930 - val_loss: 0.2523 - val_acc: 0.9000\n",
      "Epoch 85/100\n",
      "\n",
      "Epoch 00085: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2510 - acc: 0.8926 - val_loss: 0.2524 - val_acc: 0.9000\n",
      "Epoch 86/100\n",
      "\n",
      "Epoch 00086: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2536 - acc: 0.8942 - val_loss: 0.2520 - val_acc: 0.9000\n",
      "Epoch 87/100\n",
      "\n",
      "Epoch 00087: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2526 - acc: 0.8906 - val_loss: 0.2525 - val_acc: 0.9000\n",
      "Epoch 88/100\n",
      "\n",
      "Epoch 00088: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2535 - acc: 0.8914 - val_loss: 0.2521 - val_acc: 0.9000\n",
      "Epoch 89/100\n",
      "\n",
      "Epoch 00089: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2554 - acc: 0.8906 - val_loss: 0.2519 - val_acc: 0.9000\n",
      "Epoch 90/100\n",
      "\n",
      "Epoch 00090: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2527 - acc: 0.8914 - val_loss: 0.2523 - val_acc: 0.9000\n",
      "Epoch 91/100\n",
      "\n",
      "Epoch 00091: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2511 - acc: 0.9015 - val_loss: 0.2528 - val_acc: 0.9000\n",
      "Epoch 92/100\n",
      "\n",
      "Epoch 00092: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2534 - acc: 0.8942 - val_loss: 0.2523 - val_acc: 0.9000\n",
      "Epoch 93/100\n",
      "\n",
      "Epoch 00093: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2507 - acc: 0.8926 - val_loss: 0.2521 - val_acc: 0.9000\n",
      "Epoch 94/100\n",
      "\n",
      "Epoch 00094: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2516 - acc: 0.8958 - val_loss: 0.2527 - val_acc: 0.9000\n",
      "Epoch 95/100\n",
      "\n",
      "Epoch 00095: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2522 - acc: 0.8946 - val_loss: 0.2535 - val_acc: 0.9000\n",
      "Epoch 96/100\n",
      "\n",
      "Epoch 00096: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2499 - acc: 0.8946 - val_loss: 0.2533 - val_acc: 0.9000\n",
      "Epoch 97/100\n",
      "\n",
      "Epoch 00097: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2537 - acc: 0.8950 - val_loss: 0.2529 - val_acc: 0.9000\n",
      "Epoch 98/100\n",
      "\n",
      "Epoch 00098: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2564 - acc: 0.8926 - val_loss: 0.2525 - val_acc: 0.9016\n",
      "Epoch 99/100\n",
      "\n",
      "Epoch 00099: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2568 - acc: 0.8934 - val_loss: 0.2527 - val_acc: 0.9000\n",
      "Epoch 100/100\n",
      "\n",
      "Epoch 00100: LearningRateScheduler setting learning rate to 0.001.\n",
      "2477/2477 [==============================] - 3s 1ms/step - loss: 0.2533 - acc: 0.8942 - val_loss: 0.2524 - val_acc: 0.9016\n"
     ]
    }
   ],
   "source": [
    "# Training model transformer\n",
    "\n",
    "model_transformer_train = model_transformer.fit(train_X, train_Y, epochs=epochs, batch_size=32, validation_data=(valid_X, valid_Y), \n",
    "   callbacks=callbacks_list)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEICAYAAABfz4NwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJztnXl8FtXZ978XYQlhJ0RBkAQVZQ9LRNzYFEUe61YUMLgr1adq9Wnti4UqxdLXpSr6lFpRwQ1FrBtWLYqitq9aCbugCCJKADEssgiKgev948wkkzv3ljv3nYVc389nPjNz5syZa87MnN/ZR1QVwzAMw6hX3QYYhmEYNQMTBMMwDAMwQTAMwzA8TBAMwzAMwATBMAzD8DBBMAzDMAAThFqFiJwqImtFZI+InF3d9lQ1IvK4iPw22X6rExH5SETGpiDcb0TkFG/7DyLyl3j8JnCd00VkWaJ2GjWL+tVtQE1HRPYEdjOAH4ED3v4vVHVWFZrzR+B+VZ1WhddMCBF5FBjt7TYEBBd3AAtU9WcVDVNVL0+F30MdVb09GeGISDqwDzhSVQu9sOcDuckI36h+TBBioKpN/W0RWQ9c7X0EYRGR+qpanCJzsoGViZyYYrvKha+qVwNXe8f+CHSIlkin2j7DqAh19X20KqNKIiJ/FJHnRORZEdkNjBWRE72qgO9EZLOIPCgiDTz/9UVEReQXXvXPDhF5MBDesSLyvojsFJGtIvKM574e6Ai84VUZpYlIBxH5h4hsF5E1InJlDLv+KCKzPbc9IrJMRI4WkYkiUiQiX4vI6YEwWorITO8eCkVksojU845d7dn5oIhsByZWMN66iEixiFwjIhuA1724eUFEtnhxt0BEjgucM1tEJnrbw734+51n+0YRyU/Q72Ei8oaI7PKe250iElb047RxqojME5HdIvL/RCQ7cPy/vGf1nYjcFyV+ckTkexFpFnA70XsWaV78ves9+yIReSLoNySsO70Sm79/lfesi0TklhC/J4vIfzz7NonI/SLiZxzf99arvffnPD9uA+f3FJF/eecvF5Gz4o2bCsZzE+/d2yDuW3nPt1NEBnvPcad3nxd77mWq50TkWv85i0i6uO/yOhH5AvjEc3/Ie/d3icjHIjIgxMbbRWSdd3yhiLQVkcdEZErI/bwpIteFu9eahAlCcjgfeAZoATwHFAO/AtoAJwPDgV+EnDMC6Af0wSXWfkI8BXgNaAV0AKYBqGoOsAk4S1WbquoB71pfAkcAo4C7RWRQFLsAzgUeA1riShvzPXvbAf8XeChw/lO4KoKjPVv/C7gicPwk4FMgC7grjngKJQ04ATjOswvgFe96bYHPgCeinJ+Nq4o6Arge+JuINE3A73SgCDgcGAdcFsPuWDZeDNwKtAY2A38AEJF2wBzg17g4KwLywl1AVdcDy4DzQsJ9znv2AJM9G3ri4nBCDLsRkT7AVNz70gHIwb2nPj/h4icTOBX4GV5JDxjorY/z3sGXQ8JOx727L3v3dwvwvIh0CrmHcnETgWjx/CDQBTjeC2sioCJyDPAP4B7vHvpRsVL12ZR+lwAf4uI307PnefEyd959nAecgfuexgE/eHZeLCICICJHAKdQ+g3WXFTVljgXYD1weojbH4F3Ypz3G+B5b7s+oMCAwPEXgd9428/gEuX2YcIpBAZ7251wH2+TwPF7gEcj2eW5vRHYPx/YCdTz9lt5tjUF2uPEoFHA/yXAW9721cC6OOPtj8DjIW5dvGsdEeW8tsBBIN3bnw1M9LaHB2333HYBvSviF0j3rpEdOPZnYH6c9xbOxr8Ejl8ALPW2xwHvBo6lAd8CYyOEfT3wesDvFqB/BL+jgQ8D+98Ap3jbdwbeiz8FnwUus3DQ9xsm3PHAs952uvfMOgSODwfWetvDgK8ACRx/CRgfK24qEs9AA9y7f1wYf3/w7Q1z7KNgXAPX+s85cG8nRbFBgL3+db17PTOCv3XAqVr6/b8Yz31W92IlhOSwIbjjFedfE9d7YxcuJ9cm5JxvAtt7cYkwuNxjA6BARFaISKTc6hHAVlX9PuD2FS4hD2uXx5bA9j6gSFUPBvbxbMkGGgF+kf07XGnl8BjhV4SDqrrJ3/GK4Pf6RXBcrlBwubNwBG2HsvEYr9+23jUKA8ci3lecNkZ6tkcEw1aX098Y6Vq40sQQEWkDnA7sUtWPPTuOEJHnxVV/7QIepfw7Fo5QG3bixNK/v27iqs+2eOHeFme4fthfq5cKeoS+k5Hipgwx4rkdLmP1RZhTj4zgHi+h3/KtIrJaRHYCO3DC0cbL/bcPdy3v/p8E/OqpsbjSdo3HBCE5hE4Z+zCuDvIYVW2O+6gkroBUN6vq1araDvglMD2kyO2zCfdiNgm4daRsAlOZqWw34D7Y1qra0luaq2qvJIUf7vwrcLnMIbicaxfPPa64S5BvPDuCidaRUfxXxsbNwbDFtce0j+RZVb/F1duPxFW1PBM4fA/wPdDDe8euTtCGFt59+DwCLAaO9sKdHAg31vPehHsHg4S+k/ESLZ4346o5jw5z3oYI7uDiKyOw3zaMn5J7FJFhwA24knRLXNXUPlwJSHH3FelaTwIjRaQfLr5fi+CvRmGCkBqa4XJd34tIV8q3H0RERC4SET+R+A73gh4I9aeqXwIFwJ9EpJGI9MZ9RE9X1ngv/A3Ae8CfRaS5iNQTkWNEZGCscytBM1wd7DagCa6qKaWo6g/Aq8AfvIbFHrjENxU2zgWOF5GzvXroW3CJTDSewT3X8ygrCM2APcAuEekI/E+cNswBLhCRE0SkkWd/sOTUDNipqntEpDtwjX9AVX/EvddHRQj7X0A9EbnJy+EPw9Wvz4nTtiAR41lVf8IluA+IyOHiGtlPEZE0XE78bBE537MhS0T8TMxSXCKdLiJdgMvjsOEnXFtPQ5w4pgeOP4r7/o4SRx8RaenZuA5YBczEtfvsTyAOqhwThNTwa1zD5G5caaEijUknAAtF5Htc28IvVfXrCH5HAZ1xudy/A79T1XcTNToMY3Ef4ypccfl5wueqksVjuI/vG2AF8O8UXivIL3DVHUW4j/xZSsdMhJKwjaq6GVfXP5XSRuyCGKe9CPTC1dOvDrjfhmuo3Imrp38hThuW4N7Pv+Oqyb4Gtga83AxcLW78zTTKv7u34RpWvxORc0LC/gHXKDsSl5DfB4zyEseKEiueb8RV1yzxrnUHLue+FtdB4XfAdlz8dvfOuRtX1VSE60gQK/P0Kq6E9gWuTWCrd67Pnbic/zu4Nqm/4apZfZ7ANUjXiuoi8Bp/DMMoRUQewDUSx12yM4xQROQM4K+qekx12xIvNjDNqPN41USKKwmdCFwKjKlWo4xajYg0xJViple3LRXBqowMwzVavoprdHwa+KOq/rN6TTJqK1573g5cG0SNn2YmiFUZGYZhGICVEAzDMAyPWtWG0KZNG83JyaluMwzDMGoVixYt2qqqWbH81SpByMnJoaAgVi89wzAMI4iIfBWPP6syMgzDMAATBMMwDMPDBMEwDMMATBAMwzAMDxMEwzAMAzBBMFLMrFmQkwP16rn1rFnVbZFRW7F3KfXUqm6nRu1i1iwYNw727nX7X33l9gHy8yOfZxih2LtUNVgJwUgZEyaUfsA+e/fCr35VmtNr08YttTnXl+qca7LDr4057Ujv0oSYf5E2KkSc/zMdDqwG1uL9HzXkeDbwNrAceBfvn6u4f9Z+iPvJ9XLc3Oj+OY/jfhC/1Ft6x7KjX79+atQ8nn5aNTtbVcStn37auYuoQsWWjIzS82sDTz/tbK7sPUSKw8qEHy7MeMKLZEtl7qOyRHqXRJITfk0kmXEJFGg8aX1MD+7n3l/g/pLUEFgGdAvx8zxwmbc9FHjK2z4W6OxtH4H79V1LLRWEkfEY6S8mCDWPcAlMgwaqmZkVFwN/yc6u7rsqS7QPMzu78vcQLZFONPxIYUZ6Ln54lRWgZIhjOJIVz/5zzMx0S2UT21SJZ7LjMpmCcCIwL7B/K3BriJ+VwJHetuB+Bh4urGUBgTBBOASI9KFWZklWri8ZOazrriufOw1+mNFKQaE5/UREJdGccUWfix9eZRLeZCfasUpK4BL1RMWqsoltuDD95xUskSVS8ktGXAZJpiCMBB4N7F8C/CXEzzPAr7ztCwAFMkP89Ac+BeppqSCs9qqS7gcaxbLFBKH6CX3BK5LoZGfHV3LIzCx7jeuuC1/1ES63528HP85EP/qnn46cIPsfZqw48M9PVFSixWU0KhpmWlr0+w0KVKRnEkts4onvaInk00+Hf39CE+FwxPOuVjSxjRVmgwaqDRuWd4v2DcR6DolmlqpaEI7A/ft1CfAA7l+tLQPH23mJ/4AQN8H9g/QJ4LYI1x+H+y9qQceOHROLDSMpxMplxUoY4wkj3EeUzCVUbKIJRKwPPlKdfEUT9opWr6Wlxa7uiGR7ZmZi9sYSmAYNot9jPETLFUerPgsukUQ/HoGsaGKbiJDHG9dNm1YuLkOp0iqjEP9NgcLAfnNgcbTqIWAw8I9YtlgJoXqpTPVQ6EedlqYlCVQwcatM20MiS7gEJN7ER8TllOP1n6rFz3WGlpAi3evTT5fGf7wJVGUSt3CliXBEu05FbAiXaCZSQohV5ZjqZx6aMaopbQj1gXVAp0CjcvcQP20CVUFTgMnedkNc76ObwoTbzlsLMBW4M5YtJgjVS6IJQ0aGSxTi6d1SHQlqMCGoaI5fpPQeqlrMKro0bVrxqr5U5IIjiXBFRCqWzX6YwWrFeG2KVDUVFNSqeNa+3TWql5ELixHA57jeRhM8t8nAOd72SGCN5+dRvz0AGAv8RGnX0pLupcA7wArgE9x/bJvGssMEIXGS0cCayEfgXyvSuWlpzqYmTVL/gUVbKvOR+/cYrdrElvDP3W+PqEy1W+jiNzRXJEz/nFjnZWbG/5yT8T4kq5dWUgWhpiwmCIkRT2+IaOcmWjT26+urO/Hx77devcjHK5M7TaSB3Zaat2RkRK67r84lGd2w4xUEm7qiDhBulKeqW0eaAmDWLDeieNu2xK+7bVvlzk8m/ucViQMHKhf2V3H9jyox6tWDgwdTF35tpEEDaN48ue9X6DdSU/j666q7lk1dEYXt22H//vLuO3fW3JfH56efYMcOtx3rhfKnk/Dx542pyMfWpIlbDmWaNAGRqrteRgY8/bQTq6q8bmVJta0ibqkpmY1UU69e1U0zYoIQhd694fe/L+t24AAMGABXXlk9NsXLffdBt24u99qxY2z/27a5jywnx4lDvIKXne2usWePm5OousjMdAukJkHKyICHH4annnL3nAqaNHH3IOKuMX16aaktnmcYSr0Ev+7s7MTvUQSGDnU5+FSQkQGtW4fPqFWEtLTk2JPM62ZmuvsL5cCB0lLouHEpFoV46pVqylKVbQg7d7pKhqOOUj14sNT9lVece5Mmqj/8kLzr7dql+vrryQtv7Fhn5/btle8rH63ONdj+UJEeKbHGG4ionnZabLujdRsVSU7PlXDtLNH8RxosV5kpExJ5htGmqogVn5V5Z2J1JKjsc6hsz6dIvd5SvcQzPUg88ZZImwLWqFw5Vq0qfQCffVbqPmSIav36zv3NN5N3vWnTXJgbNiQnvMGDXXgrV7r9cFMwVGbxR1QGiXfUbqzRxtGmfIi3T7tPovccbUqEeEYwp4JgA3+89xVuIFrwOUSLz9DnE++AwdABXsnIkATjNRnjYXy7Kvsd+N2Og/EULhPSsGF8EwjGE1eJjFY2Qagkb75Z+gDuu8+5rVjh9m+/XTU9XfWmm5J3vYkTXdjvv1/WfetW1RtvVN2zp2LhHXOMC2/8+OT3gInUFa4yvZlSRaR7j9bNNdF5goJjElJNvFOIhCZYyZzILdZEeeHOhbKDEuPpbhxuvEq4CRVjCVaiA9YqGmZoLj/e+ZbitcdKCFr1gjBjhoudVq1c1YWq6t/+5tzWr1c96yzVzp2Td71rr3VhP/lkWffHHnPuL74Y/4d98KATLD9nUlkBqEg1R7ISn2QRrXhekQQtSLTceXWR7MnQ4iFZM3ImUgoM955FK0FFy8SEGy/QsGHsaqXQXH8yiFXyS3RcgglCBB58UHXcuNj+Jk92sXPjje6F2bPHlQxEVPfvV/3LX9zxL75Q/fpr1e7dVd95J3xYf/ub6qhR0RPLkSNdeJMnlz33ppuc+9lnR693DIb70EOVFwF/ORTmm090xslIVEfiG4tkJc6JXLcmZQB8KmJXtBx9JJGpSK6/IkQrIVQmfk0QInDBBapt28b2N26calaW6jPPuFhaubLUTVV1wQLn/s47qq+95rZbtFD95JPyYfXsqdqoUfQPdtAg53bllWXPPf30iififukg1lK/fuwSRLi2gkOJRBK06kp847GrJibORvyk6t0yQYjA8OEuxx/sORSOESNU+/RxdfqgOm+e6s9+ptqrlzu+fLlzf/55V80Drj70pJPKhvP119ETXD9X2b272+/WrWwDVapmVATVYcPia6isCYldTcMSXyNVpOLdilcQ6tw4hH373KCtPXui+9u4ETp0cAtAYSF88w20bev2/T7vwdG4+fnw8cfuGj5vvBH9Ol995fr+r1zp9letcm6qLlzVCt1e3HTt6uzMz4f16911nnoqfD9p+3dtefx4O3jQre1H70ayqM53q04KArhRyNEoLHRicMQRpfubN0O7dm6/devScLZvdwNyzjwTioth8eLScF57Lfp1RFI77UEo6enQqBGceCKsWVP2WH5+5CkSqnL4vGEY1UOdEwR/BG60Ye/79rnjHTq4xPOww1yC+M03pYKQnu5GFfolhFat4KST3LH//MetH38cXn21NNz6YWaOSlUJIByNGzshaNkSOneGLVtg166yfvySTyiJjJQ1DKN2UecEIZ4SwsaNbu1XF3XoAMuXu9y/LwjgSgl+CaF1a1edlJMDH33khpdfe23ZBF+1euf7mToVsrJKBQHKlxIyMspP/ZCRAVOmVI2NhmFUH3VWEKKVEAoL3TooCMuWue2gIGRmlpYQ/Jz1gAFOECZMgB9/LBvugQPwww+Vv4eK4pdMTj8dvvuuvCD893/Dvfe6/e+/d8LVrl34OXWi8eabMHgwbNoU2c8770BeHhQUVOqWDMNIAXVu+utgCaG42E2SlZFRmlg3aRJeEPzJtPxGZXClgm3bXJi++wknwOzZka9f0WmW09OdsHTsCCNGwBNPVHym1eJit962zc3U2rIlHHOMc1u40CX4J57ohMEXyhtvhPHjy4bx44/RSzjTpsF778HZZ7sG6tAqsk2b4PzzXTXV2WfDK684W8DdX+PG7jpffBH5GmlpcPTRtWv2T8OoLdS5EkKwDeEPf3C5VYD774cuXVzu2K8yat/erX1hgPIlhGCVEZS2I0RKOOOdZTEjw4Xvi8GUKfDXv7rEOzu7NPfep0/kMIO2+vfslxAyMtx9zZzpRGr9+tL7Bnj99dLtfftg0CA49tjIDeA//ADz58Pxx7vqtR49XHwGl6FDoWlTmDfP+R8woPRY9+7OhkGDyp8XXDp3hjFj7P8AhpEK6lQJ4eDB0mqcbdvg88/h009dgrd4sSsZ7NnjGlubNi1N1H1hgPBVRvv2lVYZff45NGzoql7CEW8JYd++UvEK/YlNsPrmm2/cnPkTJ5avotq82a2bN3e58u3bSwUBXOK6YIHb3rgR1q1z2yeeCB984P6n0KIFXHIJfPihi48RI8r+OwGcUOzf7+ydNMmVllavDn9fAwe6+Fy8uLTxffduuPlm1xX2xx/hrrvgyCPDn79okaveyshwgmIYdYWLLir9dlNFnRKEYP399u0uRwquB5G/vWWLWzIyXAPx11+7hliAZs3K5vz9KqMDB5yYtGmTvJ92hPY+8scChNblt20Lv/mN8//b34YPK1gqCicIGRnOz4cfOvfrrnPbc+bA4YfDCy/APfdA376uqucXvyh/jdxcV701eLALr2/f6Pd31FFuCd7HqFGupBYqOEFGj3bjSB580JVuDKOuMHBgDREEERkOPACkAY+q6p0hx7OBGUAWsB0Yq6qF3rHLgIme1z+q6hOeez/gcaAx8DrwK29EXcoI1r1v21YqAuvXl25/+62r8ti6tbRa4ttv3fr7791PR/zqoWDiP3euS6hSSbSxANddF1kQ/DaEjRtdDtx/qY491q3z8+GRR+Df/3b755/vqqIeeMB1ue3YEW66ybUJbNnicvQ+Bw+6EsS777rSQ7gffMTDOee49o2GDaP7E3F2TZhQel+GURfwM6YpJdZQZpwIfAEcBTQElgHdQvw8D1zmbQ8FnvK2WwPrvHUrb7uVd+xjYAAgwBvAWbFsqezUFcFpJPzpoUF16tTS7ZdeCj/7YaqWVq3KuyU6137LltGvdeGFbv3Xvzr/n3zi5kpaskRLpt5o2dIde+KJ0vPuvjv6dbdvVz3vvOT+H8IwjORBEqeu6A+sVdV1qrofmA2cG+KnG/COt70gcPxM4C1V3a6qO4C3gOEi0g5orqofecY+CZwXj4BVBr+HkQisXVvq/v77pdvffpv6nH6QwYPdOj29tKH42mvL57TjGQsQbPwOpX790t47fgmhe3d46y3XAJyW5kpAfhijRrnqoowMuPrq6Ndt1QpeegmGDYvuzzCMmk08gtAe2BDYL/TcgiwDLvC2zweaiUhmlHPbe9vRwgRARMaJSIGIFBQVFcVhbmR8QTjssFK3evVcV0kfvyE21TRq5NZ+w+j555fOXRKuN1E8YwH8xDw9vax7Roar9vEbjUPrIevXL9vF1rdv5ky3tGqV0C0ahlHLSFa3098Ag0RkCTAI2AhUsMd9eFR1uqrmqWpeViUr0fw2hGBOuk+f0rYAEbj77kpdIm7GjHHrXr1cL6D2IXKYyARXRx3lEvtwYtK5s2tQhvANUzk5bh2Mm7POcj0bDMOoG8QjCBuBYCfADp5bCaq6SVUvUNU+wATP7bso5270tiOGmQr8EoKf6LVq5XrHQGlf/mDDsz/4KTMzdmNnEBHXyKvquoRmZ5dewy+d+Ndr0wb+9a+yg8ASZdIkF9Yll5QXk+AcRfEKgmEYdYt4BGEh0FlEOolIQ2A0MDfoQUTaiIgf1q24HkcA84AzRKSViLQCzgDmqepmYJeIDBARAS4FXknC/UQlVBByckoTQpHyXT1VXWL+wAOuy2m8qJYO7PJz+jfe6Kpy/DmU/BlRMzNdKSHSpHIVISvLtQeEw+8ZBSYIhmGEJ6YgqGoxcD0ucf8UmKOqK0Vksoic43kbDKwWkc+Bw4Ep3rnbgTtworIQmOy5Afw38CiwFteLKcafAypPaJVRUBAidWH86isYO7bi4wtCu4i2aOEabf3rfPKJWwcT6lRiJQTDMGIR1zgEVX0dN1Yg6HZbYPvvwN8jnDuD0hJD0L0AiJCfTQ2hJYTs7NLqnCZNIo8uToTQ6aJbtCi7/9NPrjG3efPkXTMaviA0bFi+0Rlcb6eBA6Ffv6qxxzCMmkedmssonCAsWeK2kykG4bqIBgXBn1iudeuqm6TNL4m0bBn+mjk5rrdVmzZVY49hGDWPOikIvXrBHXe43PKttyb/OuG6iAYF4dRT3bqqqougtISQ6qHvhmHUXurUXEZ+G0KTJm4yuJycsv8/jpeMDDdVc7h2hezs8F1E/aqhtm3drJ2QnIbkeAmWEAzDMMJR50oIIqVdSBP5T3BmpisBPPBAxUYT+yWEYEO2lRAMw6hJ1KkSwr59Lmfv16F37Bj9B/d+Irp9e+k/CUJz/xMmOGGJdNwnnCBUZQnBBMEwjFjUKUHYu7dsrn7KFDeIK9wcq9nZpTOgRiL03wTRqO4SQvPmbpqO0N5OhmEYPnWuyqhx49L9/PzwYgCJVSdF47DD4Mwz3XQQWVluuufTTkvuNaJRrx5cfLFNQGcYRmTqVAkhVBDAlQTCVRuFjiOoLPXrwz//Wbr/SsrHZZfnqaeq/pqGYdQe6lwJIbQheMqUxKaaNgzDONSoU4Kwd2/5EkJ+fmJTTRuGYRxq1PkqI6hY47BhGMahSp0qIWzc6H4eX6+em6KhTRu3nZMDs2ZVt3WGYRjVS50pIcya5RqP/V5FwVHGX30F48a5bSspGIZRV6kzJYQJEyJ3MQXXvjBhQtXZYxiGUdOoM4IQz7iCZI89MAzDqE3UGUGIZ1xBssceGIZh1CbqjCDEGlfQsKGNPTAMo25TZwThoovcOtIPaZo1swZlwzDqNnEJgogMF5HVIrJWRMaHOd5RRBaIyBIRWS4iIzz3fBFZGlgOikhv79i7Xpj+scOSe2tl8f97EKlhefv28O6GYRh1hZjdTkUkDZgGDAMKgYUiMldVVwW8TQTmqOpDItIN9//lHFWdBczywukJvKyqSwPn5Xv/Vk45/s9xWrcOn/hb+4FhGHWdeEoI/YG1qrpOVfcDs4FzQ/wo4P8uvgWwKUw4Y7xzqwW/hDBypM1dZBiGEY54BKE9sCGwX+i5BZkEjBWRQlzp4IYw4YwCng1xm+lVF/1eJHztvoiME5ECESkoKiqKw9zw+IIwdKjNXWQYhhGOZDUqjwEeV9UOwAjgKREpCVtETgD2quongXPyVbUncKq3XBIuYFWdrqp5qpqXlZWVsIF79rh106Yu8V+/Hg4edGsTA8MwjPgEYSNwZGC/g+cW5CpgDoCqfgikA20Cx0cTUjpQ1Y3eejfwDK5qKmXs3u3WzZql8iqGYRi1l3gEYSHQWUQ6iUhDXOI+N8TP18BpACLSFScIRd5+PeAiAu0HIlJfRNp42w2As4FPSCH/+IdbDxpkk9kZhmGEI2YvI1UtFpHrgXlAGjBDVVeKyGSgQFXnAr8GHhGRm3ENzJerlnTwHAhsUNV1gWAbAfM8MUgD5gOPJO2uQpg1C/7619J9m8zOMAyjPKLRZnyrYeTl5WlBQcV7qebkhP9NZna2a0MwDMM4lBGRRaqaF8tfnRipHGnSOpvMzjAMo5Q6IQiRBp3ZYDTDMIxS6oQgTJkC9UNaS2wwmmEYRlnqhCDk58OJJ0Jamg1GMwzDiESd+YXmYYfBccfBypXVbYlhGEbNpE6UEMANTLNBaYZhGJExQTAMwzAAEwTDMAzDo84Iwq5dJgiGYRjRqDOCXHHLAAAb1UlEQVSCYCUEwzCM6NQJQVB1gtC8eWy/hmEYdZU6IQg//gjFxVZCMAzDiEadEAT7F4JhGEZsTBAMwzAMoI4Iwq5dbm2CYBiGEZk6IQhWQjAMw4iNCYJhGIYBmCAYhmEYHnEJgogMF5HVIrJWRMaHOd5RRBaIyBIRWS4iIzz3HBHZJyJLveVvgXP6icgKL8wHRUSSd1tlMUEwDMOITUxBEJE0YBpwFtANGCMi3UK8TQTmqGofYDQQ+KU9X6hqb2+5NuD+EHAN0Nlbhid+G9HxBcEGphmGYUQmnhJCf2Ctqq5T1f3AbODcED8K+MltC2BTtABFpB3QXFU/UlUFngTOq5DlFcAXhKZNU3UFwzCM2k88gtAe2BDYL/TcgkwCxopIIfA6cEPgWCevKuk9ETk1EGZhjDABEJFxIlIgIgVFRUVxmFueXbugcePyv9E0DMMwSklWo/IY4HFV7QCMAJ4SkXrAZqCjV5X0P8AzIlKhihtVna6qeaqal5WVlZBxNrGdYRhGbOLJM28Ejgzsd/DcglyF1wagqh+KSDrQRlW/BX703BeJyBfAsd75HWKEmTRMEAzDMGITTwlhIdBZRDqJSENco/HcED9fA6cBiEhXIB0oEpEsr1EaETkK13i8TlU3A7tEZIDXu+hS4JWk3FEYTBAMwzBiE7OEoKrFInI9MA9IA2ao6koRmQwUqOpc4NfAIyJyM66B+XJVVREZCEwWkZ+Ag8C1qrrdC/q/gceBxsAb3pISRo6EPXtSFbphGMahgbhOPrWDvLw8LSgoqG4zDMMwahUiskhV82L5qxMjlQ3DMIzYmCAYhmEYgAmCYRiG4WGCYBiGYQAmCIZhGIaHCYJhGIYBmCAYhmEYHiYIhmEYBmCCYBiGYXiYIBiGYRiACYJhGIbhYYJgGIZhACYIhmEYhocJgmEYhgGYIBiGYRgeJgiGYRgGYIJgGIZheJggGIZhGECcgiAiw0VktYisFZHxYY53FJEFIrJERJaLyAjPfZiILBKRFd56aOCcd70wl3rLYcm7LcMwDKOi1I/lQUTSgGnAMKAQWCgic1V1VcDbRGCOqj4kIt2A14EcYCvwM1XdJCI9gHlA+8B5+apqP0k2DMOoAcRTQugPrFXVdaq6H5gNnBviR4Hm3nYLYBOAqi5R1U2e+0qgsYg0qrzZhmEYRrKJRxDaAxsC+4WUzeUDTALGikghrnRwQ5hwfg4sVtUfA24zveqi34uIhLu4iIwTkQIRKSgqKorDXMMwDCMRktWoPAZ4XFU7ACOAp0SkJGwR6Q7cBfwicE6+qvYETvWWS8IFrKrTVTVPVfOysrKSZK5hGIYRSjyCsBE4MrDfwXMLchUwB0BVPwTSgTYAItIBeAm4VFW/8E9Q1Y3eejfwDK5qyjAMw6gm4hGEhUBnEekkIg2B0cDcED9fA6cBiEhXnCAUiUhL4DVgvKr+P9+ziNQXEV8wGgBnA59U9mYMwzCMxIkpCKpaDFyP6yH0Ka430UoRmSwi53jefg1cIyLLgGeBy1VVvfOOAW4L6V7aCJgnIsuBpbgSxyPJvjnDMAwjfsSl27WDvLw8LSiwXqqGYRgVQUQWqWpeLH82UtkwDMMATBAMwzAMDxMEwzAMAzBBMAzDMDxMEAzDMAzABMEwDMPwMEEwDMMwABMEwzAMw8MEwTAMwwBMEAzDMAwPEwTDMAwDMEEwDMMwPEwQDMMwDMAEwTAMw/AwQTAMwzAAEwTDMAzDwwTBMAzDAEwQDMMwDA8TBMMwDAOIUxBEZLiIrBaRtSIyPszxjiKyQESWiMhyERkROHard95qETkz3jANwzCMqiWmIIhIGjANOAvoBowRkW4h3iYCc1S1DzAa+Kt3bjdvvzswHPiriKTFGaZhGIZRhcRTQugPrFXVdaq6H5gNnBviR4Hm3nYLYJO3fS4wW1V/VNUvgbVeePGEaRiGYVQh8QhCe2BDYL/QcwsyCRgrIoXA68ANMc6NJ0wARGSciBSISEFRUVEc5hqGYRiJkKxG5THA46raARgBPCUiSQlbVaerap6q5mVlZSUjSMMwDCMM9ePwsxE4MrDfwXMLchWujQBV/VBE0oE2Mc6NFaZhGIZRhcSTi18IdBaRTiLSENdIPDfEz9fAaQAi0hVIB4o8f6NFpJGIdAI6Ax/HGaZhGIZRhcQsIahqsYhcD8wD0oAZqrpSRCYDBao6F/g18IiI3IxrYL5cVRVYKSJzgFVAMfBLVT0AEC7MFNyfYRiGESfi0u3aQV5enhYUFFS3GYZhGLUKEVmkqnmx/NlIZcMwDAMwQTAMwzA8TBAMwzAMwATBMAzD8DBBMAzDMAATBMMwDMPDBMEwDMMATBAMwzAMDxMEwzAMAzBBMAzDMDxMEAzDMAzABMEwDMPwMEEwDMMwABMEwzAMw8MEwTAMwwBMEAzDMAwPEwTDMAwDMEEwDMMwPOISBBEZLiKrRWStiIwPc/x+EVnqLZ+LyHee+5CA+1IR+UFEzvOOPS4iXwaO9U7urRmGYRgVoX4sDyKSBkwDhgGFwEIRmauqq3w/qnpzwP8NQB/PfQHQ23NvDawF3gwEf4uq/j0J92EYhmFUknhKCP2Btaq6TlX3A7OBc6P4HwM8G8Z9JPCGqu6tuJmGYRhGqolHENoDGwL7hZ5bOUQkG+gEvBPm8GjKC8UUEVnuVTk1ihDmOBEpEJGCoqKiOMw1DMMwEiHZjcqjgb+r6oGgo4i0A3oC8wLOtwJdgOOB1sD/CRegqk5X1TxVzcvKykqyuYZhGIZPPIKwETgysN/BcwtHuFIAwEXAS6r6k++gqpvV8SMwE1c1ZRiGYVQTMRuVgYVAZxHphBOC0cDFoZ5EpAvQCvgwTBhjcCWCoP92qrpZRAQ4D/ikgrYbRp3lp59+orCwkB9++KG6TTFqEOnp6XTo0IEGDRokdH5MQVDVYhG5HlfdkwbMUNWVIjIZKFDVuZ7X0cBsVdXg+SKSgythvBcS9CwRyQIEWApcm9AdGEYdpLCwkGbNmpGTk4PLUxl1HVVl27ZtFBYW0qlTp4TCiKeEgKq+Drwe4nZbyP6kCOeuJ0wjtKoOjddIwzDK8sMPP5gYGGUQETIzM6lM5xsbqWwYtRQTAyOUyr4TJgiGYRgGYIJgGHWCWbMgJwfq1XPrWbMqF962bdvo3bs3vXv3pm3btrRv375kf//+/XGFccUVV7B69eqofqZNm8asyhprxE1cbQiGYdReZs2CceNgrzdHwFdfuX2A/PzEwszMzGTp0qUATJo0iaZNm/Kb3/ymjB9VRVWpVy98vnPmzJkxr/PLX/4yMQOrkeLiYurXr51Jq5UQDOMQZ8KEUjHw2bvXuSebtWvX0q1bN/Lz8+nevTubN29m3Lhx5OXl0b17dyZPnlzi95RTTmHp0qUUFxfTsmVLxo8fT25uLieeeCLffvstABMnTmTq1Kkl/sePH0///v057rjj+OCDDwD4/vvv+fnPf063bt0YOXIkeXl5JWIV5Pbbb+f444+nR48eXHvttfgdIj///HOGDh1Kbm4uffv2Zf369QD86U9/omfPnuTm5jLBiyzfZoBvvvmGY445BoBHH32U8847jyFDhnDmmWeya9cuhg4dSt++fenVqxf/+Mc/SuyYOXMmvXr1Ijc3lyuuuIKdO3dy1FFHUVxcDMCOHTvK7FclJgiGcYjz9dcVc68sn332GTfffDOrVq2iffv23HnnnRQUFLBs2TLeeustVq1aVe6cnTt3MmjQIJYtW8aJJ57IjBkzwoatqnz88cfcc889JeLyv//7v7Rt25ZVq1bx+9//niVLloQ991e/+hULFy5kxYoV7Ny5k3/+858AjBkzhptvvplly5bxwQcfcNhhh/Hqq6/yxhtv8PHHH7Ns2TJ+/etfx7zvJUuW8OKLL/L222/TuHFjXn75ZRYvXsz8+fO5+WY3/+eyZcu46667ePfdd1m2bBn33nsvLVq04OSTTy6x59lnn+XCCy+sllKGCYJhHOJ07Fgx98py9NFHk5eXV7L/7LPP0rdvX/r27cunn34aVhAaN27MWWedBUC/fv1KcumhXHDBBeX8/Pvf/2b06NEA5Obm0r1797Dnvv322/Tv35/c3Fzee+89Vq5cyY4dO9i6dSs/+9nPADewKyMjg/nz53PllVfSuHFjAFq3bh3zvs844wxatWoFOOEaP348vXr14owzzmDDhg1s3bqVd955h1GjRpWE56+vvvrqkiq0mTNncsUVV8S8XiowQTCMQ5wpUyAjo6xbRoZzTwVNmjQp2V6zZg0PPPAA77zzDsuXL2f48OFhR1c3bNiwZDstLS1idUmjRo1i+gnH3r17uf7663nppZdYvnw5V155ZUKjvOvXr8/BgwcByp0fvO8nn3ySnTt3snjxYpYuXUqbNm2iXm/QoEF8/vnnLFiwgAYNGtClS5cK25YMTBAM4xAnPx+mT4fsbBBx6+nTE29Qrgi7du2iWbNmNG/enM2bNzNv3rzYJ1WQk08+mTlz5gCwYsWKsCWQffv2Ua9ePdq0acPu3bt54YUXAGjVqhVZWVm8+uqrgEvk9+7dy7Bhw5gxYwb79u0DYPv27QDk5OSwaNEiAP7+98i/ctm5cyeHHXYY9evX56233mLjRjf929ChQ3nuuedKwvPXAGPHjiU/P7/aSgdggmAYdYL8fFi/Hg4edOuqEAOAvn370q1bN7p06cKll17KySefnPRr3HDDDWzcuJFu3brxhz/8gW7dutGiRYsyfjIzM7nsssvo1q0bZ511FieccELJsVmzZnHvvffSq1cvTjnlFIqKijj77LMZPnw4eXl59O7dm/vvvx+AW265hQceeIC+ffuyY8eOiDZdcsklfPDBB/Ts2ZPZs2fTuXNnwFVp/fa3v2XgwIH07t2bW265peSc/Px8du7cyahRo5IZPRVCQqYeqtHk5eVpQUFBdZthGNXOp59+SteuXavbjBpBcXExxcXFpKens2bNGs444wzWrFlT67p+zp49m3nz5sXVHTca4d4NEVmkqnkRTimhdsWYYRhGCHv27OG0006juLgYVeXhhx+udWJw3XXXMX/+/JKeRtVF7Yo1wzCMEFq2bFlSr19beeihh6rbBMDaEAzDMAwPEwTDMAwDMEEwDMMwPEwQDMMwDMAEwTCMBBgyZEi5QWZTp07luuuui3pe06ZNAdi0aRMjR44M62fw4MHE6l4+depU9gZm7BsxYgTfffddPKYbUYhLEERkuIisFpG1IjI+zPH7RWSpt3wuIt8Fjh0IHJsbcO8kIv/xwnxORBqGhmsYRs1kzJgxzJ49u4zb7NmzGTNmTFznH3HEEVFH+sYiVBBef/11WrZsmXB4VY2qlkyBUZOIKQgikgZMA84CugFjRKRb0I+q3qyqvVW1N/C/wIuBw/v8Y6p6TsD9LuB+VT0G2AFcVcl7MYw6yU03weDByV1uuin6NUeOHMlrr71W8jOc9evXs2nTJk499dSScQF9+/alZ8+evPLKK+XOX79+PT169ADctBKjR4+ma9eunH/++SXTRYDrn+9PnX377bcD8OCDD7Jp0yaGDBnCkCFDADelxNatWwG477776NGjBz169CiZOnv9+vV07dqVa665hu7du3PGGWeUuY7Pq6++ygknnECfPn04/fTT2bJlC+DGOlxxxRX07NmTXr16lUx98c9//pO+ffuSm5vLaaedBrj/Q/z5z38uCbNHjx6sX7+e9evXc9xxx3HppZfSo0cPNmzYEPb+ABYuXMhJJ51Ebm4u/fv3Z/fu3QwcOLDMtN6nnHIKy5Yti/6gKkg84xD6A2tVdR2AiMwGzgXKTxjiGAPcHuEYXhgCDAUu9pyeACYBNaMzrmEYUWndujX9+/fnjTfe4Nxzz2X27NlcdNFFiAjp6em89NJLNG/enK1btzJgwADOOeeciP/7feihh8jIyODTTz9l+fLl9O3bt+TYlClTaN26NQcOHOC0005j+fLl3Hjjjdx3330sWLCANm3alAlr0aJFzJw5k//85z+oKieccAKDBg2iVatWrFmzhmeffZZHHnmEiy66iBdeeIGxY8eWOf+UU07ho48+QkR49NFHufvuu7n33nu54447aNGiBStWrADcPwuKioq45ppreP/99+nUqVOZeYkisWbNGp544gkGDBgQ8f66dOnCqFGjeO655zj++OPZtWsXjRs35qqrruLxxx9n6tSpfP755/zwww/k5uZW6LnFIh5BaA9sCOwXAieE8ygi2UAn4J2Ac7qIFADFwJ2q+jKQCXynqv50hYXedcKFOQ4YB9AxVfP1GkYtxssEVzl+tZEvCI899hjgqkN+97vf8f7771OvXj02btzIli1baNu2bdhw3n//fW688UYAevXqRa9evUqOzZkzh+nTp1NcXMzmzZtZtWpVmeOh/Pvf/+b8888vmXn0ggsu4F//+hfnnHMOnTp1onfv3kDkKbYLCwsZNWoUmzdvZv/+/XTq1AmA+fPnl6kia9WqFa+++ioDBw4s8RPPFNnZ2dklYhDp/kSEdu3acfzxxwPQvHlzAC688ELuuOMO7rnnHmbMmMHll18e83oVJdmNyqOBv6vqgYBbtjeHxsXAVBE5uiIBqup0Vc1T1bysrKwKG5Tsf8kahuE499xzefvtt1m8eDF79+6lX79+gJssrqioiEWLFrF06VIOP/zwhKaa/vLLL/nzn//M22+/zfLly/mv//qvhMLx8afOhsjTZ99www1cf/31rFixgocffrjSU2RD2Wmyg1NkV/T+MjIyGDZsGK+88gpz5swhPwUzFMYjCBuBIwP7HTy3cIwGng06qOpGb70OeBfoA2wDWoqIX0KJFmbC+P+S/eorUC39l6yJgmFUnqZNmzJkyBCuvPLKMo3J/tTPDRo0YMGCBXz11VdRwxk4cCDPPPMMAJ988gnLly8H3NTZTZo0oUWLFmzZsoU33nij5JxmzZqxe/fucmGdeuqpvPzyy+zdu5fvv/+el156iVNPPTXue9q5cyft27vKiieeeKLEfdiwYUybNq1kf8eOHQwYMID333+fL7/8Eig7RfbixYsBWLx4ccnxUCLd33HHHcfmzZtZuHAhALt37y4Rr6uvvpobb7yR448/vuRnPMkkHkFYCHT2egU1xCX6c0M9iUgXoBXwYcCtlYg08rbbACcDq9RNsboA8PudXQaUb3mqJFX5L1nDqIuMGTOGZcuWlRGE/Px8CgoK6NmzJ08++WTMn71cd9117Nmzh65du3LbbbeVlDRyc3Pp06cPXbp04eKLLy4zdfa4ceMYPnx4SaOyT9++fbn88svp378/J5xwAldffTV9+vSJ+34mTZrEhRdeSL9+/cq0T0ycOJEdO3bQo0cPcnNzWbBgAVlZWUyfPp0LLriA3Nzckmmrf/7zn7N9+3a6d+/OX/7yF4499tiw14p0fw0bNuS5557jhhtuIDc3l2HDhpWUHPr160fz5s1T9s+EuKa/FpERwFQgDZihqlNEZDJQoKpzPT+TgHRVHR847yTgYeAgTnymqupj3rGjgNlAa2AJMFZVf4xmR0Wnv65Xz5UMyt+PmxfeMGorNv113WTTpk0MHjyYzz77jHr1wufnUz79taq+Drwe4nZbyP6kMOd9APSMEOY6XA+mlNGxo6smCuduGIZRm3jyySeZMGEC9913X0QxqCyH9Ejlqv6XrGEYRqq49NJL2bBhAxdeeGHKrnFIC0J1/kvWMFJNbfrboVE1VPadOOR/kJOfbwJgHHqkp6ezbds2MjMzIw74MuoWqsq2bdtIT09POIxDXhAM41CkQ4cOFBYWUlRUVN2mGDWI9PR0OnTokPD5JgiGUQtp0KBByQhZw0gWh3QbgmEYhhE/JgiGYRgGYIJgGIZheMQ1UrmmICJFQPSJUSLTBtiaRHOSRU21C2qubWZXxTC7Kk5NtS1Ru7JVNebsoLVKECqDiBTEM3S7qqmpdkHNtc3sqhhmV8Wpqbal2i6rMjIMwzAAEwTDMAzDoy4JwvTqNiACNdUuqLm2mV0Vw+yqODXVtpTaVWfaEAzDMIzo1KUSgmEYhhEFEwTDMAwDqCOCICLDRWS1iKwVkfGxz0iZHUeKyAIRWSUiK0XkV577JBHZKCJLvWVENdi2XkRWeNcv8Nxai8hbIrLGWyf/J67RbTouECdLRWSXiNxUXfElIjNE5FsR+STgFjaOxPGg984tF5G+VWzXPSLymXftl0SkpeeeIyL7AnH3tyq2K+KzE5FbvfhaLSJnVrFdzwVsWi8iSz33qoyvSOlD1b1jqnpIL7jffn4BHAU0BJYB3arJlnZAX2+7GfA50A2YBPymmuNpPdAmxO1uYLy3PR64q5qf4zdAdnXFFzAQ6At8EiuOgBHAG4AAA4D/VLFdZwD1ve27AnblBP1VQ3yFfXbed7AMaAR08r7ZtKqyK+T4vcBt1RBfkdKHKnvH6kIJoT+wVlXXqep+3H+cz60OQ1R1s6ou9rZ3A58C7avDljg5F3jC234COK8abTkN+EJVEx2pXmlU9X1ge4hzpDg6F3hSHR8BLUWkXVXZpapvqmqxt/sRkPicyEm0KwrnArNV9UdV/RJYS4p+sRvNLnE/l7gIeDYV145GlPShyt6xuiAI7YENgf1CakAiLCI5QB/gP57T9V6xb0ZVV814KPCmiCwSkXGe2+Gqutnb/gY4vBrs8hlN2Y+0uuPLJ1Ic1aT37kpcTtKnk4gsEZH3ROTUarAn3LOrKfF1KrBFVdcE3Ko8vkLShyp7x+qCINQ4RKQp8AJwk6ruAh4CjgZ6A5txRdaq5hRV7QucBfxSRAYGD6oro1ZLH2URaQicAzzvOdWE+CpHdcZRJERkAlAMzPKcNgMdVbUP8D/AMyLSvApNqpHPLsAYymY8qjy+wqQPJaT6HasLgrARODKw38FzqxZEpAHuYc9S1RcBVHWLqh5Q1YPAI6SoqBwNVd3orb8FXvJs2OIXQb31t1Vtl8dZwGJV3eLZWO3xFSBSHFX7eycilwNnA/leQoJXJbPN216Eq6s/tqpsivLsakJ81QcuAJ7z3ao6vsKlD1ThO1YXBGEh0FlEOnk5zdHA3OowxKuffAz4VFXvC7gH6/3OBz4JPTfFdjURkWb+Nq5B8hNcPF3mebsMeKUq7QpQJtdW3fEVQqQ4mgtc6vUEGQDsDBT7U46IDAd+C5yjqnsD7lkikuZtHwV0BtZVoV2Rnt1cYLSINBKRTp5dH1eVXR6nA5+paqHvUJXxFSl9oCrfsapoPa/uBdca/zlO3SdUox2n4Ip7y4Gl3jICeApY4bnPBdpVsV1H4Xp4LANW+nEEZAJvA2uA+UDraoizJsA2oEXArVriCydKm4GfcPW1V0WKI1zPj2neO7cCyKtiu9bi6pf99+xvnt+fe894KbAY+FkV2xXx2QETvPhaDZxVlXZ57o8D14b4rcr4ipQ+VNk7ZlNXGIZhGEDdqDIyDMMw4sAEwTAMwwBMEAzDMAwPEwTDMAwDMEEwDMMwPEwQDMMwDMAEwTAMw/D4/0R2a86JnmjaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEICAYAAACzliQjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJztnXuYFNW1t98FzDBchttAolwHFYXhIuCIJgSRaBQ1ircoOCgaDZHoMYkxJxiMGiI5aDyKJsRLjMYjKBo5JiSifCYhwSRHZVBEERFEkAGEAeWOwMD6/thVdE3Tl+qenumhe73PU09V7dq1a9Xu6l+t2ldRVQzDMIz8oEm2DTAMwzAaDhN9wzCMPMJE3zAMI48w0TcMw8gjTPQNwzDyCBN9wzCMPMJEP8cQkWEislJEdorI17NtT0MjIr8Tkf/MdNxsIiKvicjYekj3ExH5irf9UxH5VZi4aVznTBF5O107E6TbW0RqMp1urtMs2wbkAiKyM7DbEtgLHPD2v62qMxvQnLuA+1V1egNeMy1E5DFgtLdbCAgu7wDmq+r5qaapqlfXR9xcR1XvyEQ6IlIE7AG6qWqVl/ZfgBMzkb5Rd0z0M4Cqtva3RWQ1cJ33oMdERJqpan15KD2ApemcWM92HZa+ql4HXOcduwvomkiI69s+w8gHrHinARCRu0TkWRF5RkR2AGNF5EveZ/tWEdkgIg+KSIEXv5mIqIh82yuq+UxEHgykd7yILBCRbSKyWUSe9sJXA92Bl7zinaYi0lVE/iwin4rIChH5ZhK77hKRWV7YThF5W0SOFZHbRKRaRD4WkTMDabQTkSe8e6gSkcki0sQ7dp1n54Mi8ilwW4r51ltEakTkWyKyFpjr5c1sEdno5d18ETkhcM4sEbnN2x7p5d+PPdvXiUhFmnG/ICIvich273ebKiIxX+whbZwmIvNEZIeI/EtEegSOn+f9VltF5L4E+VMqIrtEpDgQ9iXvt2jq5d/fvd++WkSeDMaNSmuq9+Xl71/r/dbVIvLDqLhDReR1z771InK/iPgO5AJvvdx7fi708zZwfn8RedU7f4mInBM2bxIhIt1FZK53vx+IyLgom9/yfr9PROS/vPBW3jU/9ex5XUTah7nekYqJfsNxEfA00BZ4FqgBvgt0BIYCI4FvR51zLnASMAgnyL7YTgFeBNoDXYHpAKpaCqwHzlHV1qp6wLvWR0Bn4HLgHhEZnsAugFHAb4F2uK+Gv3j2Hg38F/BQ4PyncJ/zx3q2ngdcEzj+ZWAZ0Am4O0Q+RdMUOAU4wbML4I/e9Y4C3geeTHB+D1yxUWfgRuBhEWmdRtxHgWrgi8B4YFzMFCIks/EK4FagA7AB+CmAiBwNPAf8AJdn1UB5rAuo6mrgbeDCqHSf9X57gMmeDf1xeTgpid2IyCBgGu556QqU4p5Tn/24/CkBhgHn432xAad56xO8Z/APUWkX4Z7dP3j390Pg9yLSM+oeDsubEPweWI57Tq8A7heRod6xXwE/V9U2QC/v+nh2NwO6ePd4I7Av5PWOTFTVlgwuwGrgzKiwu4C/JTnvFuD33nYzQIFTA8f/F7jF234aJ7xdYqRTBZzubffE/UFbBY7/Angsnl1e2EuB/YuAbUATb7+9Z1tr3B9lD9A8EP9K4BVv+zpgVch8uwv4XVRYb+9anROcdxRwECjy9mcBt3nbI4O2e2HbgYGpxAWKvGv0CBy7F/hLyHuLZeOvAscvBhZ72+OBvweONQU2AWPjpH0jMDcQdyMwJE7c0cD/BfY/Ab7ibU8NPBc/D/4WOIfgoB83RroTgWe87SLvN+saOD4SWOltfw1YA0jg+AvAxGR5E+O6vYEab7sX8DnQInD8fuBhb/sN3AuvJCqN7wD/APqF/Y8f6Yt5+g3H2uCO9+n9ovepuR3nkXWMOueTwPZunNCC8wILgEoReSf4GRtFZ2Czqu4KhK3BiXVMuzw2Brb3ANWqejCwj2dLD6A54BdjbMV9dXwxSfqpcFBV1/s7XtHJf4vIKi/f3sd55yVxzg/aDrXzMWzco7xrVAWOxb2vkDbG+207B9NW57Gvi3ct3FfBCBHpCJwJbFfVNzw7OovI772iqu3AYxz+jMUi2oZtuBeif39lXlHXRi/d20Om66f9sXqK6xH9TMbLm2TpVqvqnkBYMN1xwADgA68I52wv/Lc40X9eXPHkz0Wkach7OSIx0W84ooczfQR4FzhO3Sfn7ThhSJ6Q6gZVvU5VjwZuAB6N+jz2WQ90FJFWgbDu1BaRugyzuhb3p+ygqu28pY2qDshQ+rHOvwbnLY7AeaC9vfBQeZcmn3h2BIWpW4L4dbFxQzBtcfUjXeJFVtVNuHL0S3FFGk8HDv8C2IXzYtvgvrzSsaGtdx8+vwHeBI710p0cSDfZ770e9wwGiX4m02E90ElEWsRKV1WXqerlwBeAB4H/FZFCVd2rqreram9c0dQ3iLQoy0lM9LNHMc572iUifTi8PD8uInKZiPhCsBX3RzsQHU9VPwIqgZ+LSHMRGYgTpBl1Nd5Lfy3OS7pXRNqISBMROU5ETkt2bh0oxn3GbwFa4YqF6hVV/Rz4E/BTESkSkX44ga0PG+cAJ4vI18VV7P8QV7adiKdxv+uF1Bb9YmAnsF1EugM3h7ThOeBiETlFRJp79ge/gIqBbaq6U0T6At/yD6jqXtxzfUyctF8FmojI97wvoq8BZ3nXrAsrgXeAu7xnfTDOu58BICJXiUiJ9+W0DfefUXF9CMq8l+t2XN3VwdiXyA1M9LPHD3AP5Q6c1/9s4ui1OAVYKCK7cGX9N6jqx3HiXo4r7/wEeB74sar+PV2jYzAWJ2zvAZ/hKtOOymD60fwWV7n5Ce5P/s96vFaQb+MVIeCKSZ4h0qcgmrRtVNUNOE9zGpGK48okp/0vruhipaouD4TfDnwFJ3IvALND2vAW7vl8Hlek9TGwORDl+8B14vqnTOfwZ/d2XOXsVhG5ICrtz4Gv475MtgD3AZer6qowtiWwWXFeehku358Ffqiqft5/HdeiaAeuMcJlqrof9xX1R9z/8F1gboz7ySmkdtGaYRhhEJEHcBWzob/QDKMxYJ2zDCMEXpGO4r5ovgRcBYzJqlGGkQYm+oYRjra4PglH4YoP7lLVl7NrkmGkjhXvGIZh5BFWkWsYhpFHNLrinY4dO2ppaWm2zTAMwziiWLRo0WZV7ZQsXqMT/dLSUiork7VQMwzDMIKIyJow8ax4xzAMI48w0TcMw8gjTPQNwzDyiEZXpm8YRsOyf/9+qqqq+Pzzz7NtihGCoqIiunbtSkFBQVrnm+gbRp5TVVVFcXExpaWliNTnYKVGXVFVtmzZQlVVFT17xhpYNzlWvGMYec7nn39OSUmJCf4RgIhQUlJSp68yE33DMEzwjyDq+lvljOjv3Al33AFvvJFtSwzDMBovOSP6M2bA5MlwyilQWgozZ2bbIsMwwrBlyxYGDhzIwIEDOeqoo+jSpcuh/X37ws1Rfs0117B8+fKEcaZPn87MDAnDV77yFRYvXpyRtBqanKjInTkTbg7MCbRmDYwf77YrKrJjk2HkKjNnwqRJ8PHH0L07TJlSt/9ZSUnJIQG98847ad26NbfcckutOIcm9W4S20994oknkl7nhhtuSN/IHCInPP1Jk2DPntphu3e7cMMwMsfMmc6hWrMGVCMOVn18Wa9cuZKysjIqKiro27cvGzZsYPz48ZSXl9O3b18mT558KK7vedfU1NCuXTsmTpzIiSeeyJe+9CU2bdoEwG233ca0adMOxZ84cSJDhgzhhBNO4N///jcAu3bt4pJLLqGsrIxLL72U8vLypB79jBkz6N+/P/369ePHP/4xADU1NVx55ZWHwh988EEA7r//fsrKyhgwYABjx47NeJ6FISc8/Y/jTBQYL9wwjPSYNMk5VEF8B6s+vqrff/99/ud//ofy8nIApk6dSocOHaipqWHEiBFceumllJWV1Tpn27ZtDB8+nKlTp3LzzTfz+OOPM3HixMPSVlXeeOMN5syZw+TJk3n55Zf55S9/yVFHHcXs2bN5++23GTx4cEL7qqqquO2226isrKRt27aceeaZ/PnPf6ZTp05s3ryZd955B4CtW7cCcM8997BmzRoKCwsPhTU0oTx9ERkpIstFZKWIHJZ7InK1iFSLyGJvuS5wbJyIrPCWcZk03qd799TCDcNIj4Z2sI499thDgg/wzDPPMHjwYAYPHsyyZct47733DjunRYsWnHPOOQCcdNJJrF69OmbaF1988WFx/vnPfzJ69GgATjzxRPr27ZvQvtdff52vfvWrdOzYkYKCAq644goWLFjAcccdx/Lly7npppuYN28ebdu2BaBv376MHTuWmTNnpt25qq4kFX0RaYqb/Pgc3KTDY0SkLEbUZ1V1oLc85p3bAbgDN5H3EOAOEWmfMes9pkyBli1rh7Vs6cINw8gcDe1gtWrV6tD2ihUreOCBB/jb3/7GkiVLGDlyZMz26oWFhYe2mzZtSk1NTcy0mzdvnjROupSUlLBkyRKGDRvG9OnT+fa33VTK8+bN4/rrr2fhwoUMGTKEAwcOZPS6YQjj6Q8BVqrqKlXdB8wCRoVM/2zgFVX9VFU/A14BRqZnanwqKuDRR8Gv4+nRw+1bJa5hZJZsOljbt2+nuLiYNm3asGHDBubNm5fxawwdOpTnnnsOgHfeeSfml0SQU045hfnz57NlyxZqamqYNWsWw4cPp7q6GlXlG9/4BpMnT+bNN9/kwIEDVFVV8dWvfpV77rmHzZs3szu6rKwBCFOm3wVYG9ivwnnu0VwiIqcBHwDfV9W1cc7tEn2iiIwHxgN0T9NlqKhwLXguvhgeeiitJAzDSILvSGWy9U5YBg8eTFlZGb1796ZHjx4MHTo049f4j//4D6666irKysoOLX7RTCy6du3Kz372M04//XRUlfPPP5/zzjuPN998k2uvvRZVRUS4++67qamp4YorrmDHjh0cPHiQW265heLi4ozfQzKSzpErIpcCI1X1Om//SuAUVb0xEKcE2Kmqe0Xk28DlqvpVEbkFKFLVu7x4PwH2qOq98a5XXl6u6U6i0q0bnHUW/Pa3aZ1uGHnJsmXL6NOnT7bNaBTU1NRQU1NDUVERK1as4KyzzmLFihU0a9a42rzE+s1EZJGqlsc55RBh7mQd0C2w39ULO4SqbgnsPgbcEzj39Khz/x7immlRWAgh+3IYhmEcxs6dOznjjDOoqalBVXnkkUcaneDXlTB3sxDoJSI9cSI+GrgiGEFEjlbVDd7uBcAyb3se8PNA5e1ZwK11tjoOJvqGYdSFdu3asWjRomybUa8kFX1VrRGRG3EC3hR4XFWXishkoFJV5wA3icgFQA3wKXC1d+6nIvIz3IsDYLKqfloP9wFA8+Ym+oZhGIkI9d2iqnOBuVFhtwe2byWOB6+qjwOP18HG0JinbxiGkZicGIbBx0TfMAwjMSb6hmEYeUTOif7evdm2wjCMVBgxYsRhHa2mTZvGhAkTEp7XunVrANavX8+ll14aM87pp59Osibg06ZNq9VJ6txzz83IuDh33nkn994bt3V61sg50TdP3zCOLMaMGcOsWbNqhc2aNYsxY8aEOr9z5848//zzaV8/WvTnzp1Lu3bt0k6vsWOibxhGVrn00kt58cUXD02Ysnr1atavX8+wYcMOtZsfPHgw/fv3549//ONh569evZp+/foBsGfPHkaPHk2fPn246KKL2BMYc33ChAmHhmW+4447AHjwwQdZv349I0aMYMSIEQCUlpayefNmAO677z769etHv379Dg3LvHr1avr06cO3vvUt+vbty1lnnVXrOrFYvHgxp556KgMGDOCiiy7is88+O3R9f6hlf6C3f/zjH4cmkRk0aBA7duxIO29jkVO9Dkz0DaNufO97kOkJoQYOBE8vY9KhQweGDBnCSy+9xKhRo5g1axaXXXYZIkJRUREvvPACbdq0YfPmzZx66qlccMEFceeJfeihh2jZsiXLli1jyZIltYZGnjJlCh06dODAgQOcccYZLFmyhJtuuon77ruP+fPn07Fjx1ppLVq0iCeeeILXX38dVeWUU05h+PDhtG/fnhUrVvDMM8/wm9/8hssuu4zZs2cnHB//qquu4pe//CXDhw/n9ttv56c//SnTpk1j6tSpfPTRRzRv3vxQkdK9997L9OnTGTp0KDt37qSoqCiF3E6OefqGYWSdYBFPsGhHVfnxj3/MgAEDOPPMM1m3bh0bN26Mm86CBQsOie+AAQMYMGDAoWPPPfccgwcPZtCgQSxdujTpYGr//Oc/ueiii2jVqhWtW7fm4osv5tVXXwWgZ8+eDBw4EEg8fDO48f23bt3K8OHDARg3bhwLFiw4ZGNFRQUzZsw41PN36NCh3HzzzTz44INs3bo14z2CzdM3DOMQiTzy+mTUqFF8//vf580332T37t2cdNJJAMycOZPq6moWLVpEQUEBpaWlMYdTTsZHH33Evffey8KFC2nfvj1XX311Wun4+MMygxuaOVnxTjxefPFFFixYwJ/+9CemTJnCO++8w8SJEznvvPOYO3cuQ4cOZd68efTu3TttW6PJKU/feuQaxpFJ69atGTFiBN/85jdrVeBu27aNL3zhCxQUFDB//nzWrFmTMJ3TTjuNp59+GoB3332XJUuWAG5Y5latWtG2bVs2btzISy+9dOic4uLimOXmw4YN4w9/+AO7d+9m165dvPDCCwwbNizle2vbti3t27c/9JXw1FNPMXz4cA4ePMjatWsZMWIEd999N9u2bWPnzp18+OGH9O/fnx/96EecfPLJvP/++ylfMxHm6RuG0SgYM2YMF110Ua2WPBUVFZx//vn079+f8vLypB7vhAkTuOaaa+jTpw99+vQ59MVw4oknMmjQIHr37k23bt1qDcs8fvx4Ro4cSefOnZk/f/6h8MGDB3P11VczZMgQAK677joGDRqUsCgnHk8++STXX389u3fv5phjjuGJJ57gwIEDjB07lm3btqGq3HTTTbRr146f/OQnzJ8/nyZNmtC3b99Ds4BliqRDKzc0dRla+dZb4f77oQ5fbYaRd9jQykcedRlaOaeKd/zOWY3sPWYYhtFoyDnRB8jwdJeGYRg5Q06KvpXrG0ZqNLZiXiM+df2tTPQNI88pKipiy5YtJvxHAKrKli1b6tRhK+da74CJvmGkQteuXamqqqK6ujrbphghKCoqomvXrmmfb6JvGHlOQUEBPXv2zLYZRgNhxTuGYRh5RE6Jvt8z2kTfMAwjNjkl+ubpG4ZhJCYnRd9mzzIMw4hNToq+efqGYRixCSX6IjJSRJaLyEoRmZgg3iUioiJS7u2XisgeEVnsLQ9nyvBYmOgbhmEkJmmTTRFpCkwHvgZUAQtFZI6qvhcVrxj4LvB6VBIfqurADNmbEBN9wzCMxITx9IcAK1V1laruA2YBo2LE+xlwN5C1MS5N9A3DMBITRvS7AGsD+1Ve2CFEZDDQTVVfjHF+TxF5S0T+ISIxZyAQkfEiUikilXXpFWiibxiGkZg6V+SKSBPgPuAHMQ5vALqr6iDgZuBpEWkTHUlVH1XVclUt79SpU9q2mOgbhmEkJozorwO6Bfa7emE+xUA/4O8isho4FZgjIuWquldVtwCo6iLgQ+D4TBgeCxN9wzCMxIQR/YVALxHpKSKFwGhgjn9QVbepakdVLVXVUuA14AJVrRSRTl5FMCJyDNALWJXxu/Dwe+RaO33DMIzYJG29o6o1InIjMA9oCjyuqktFZDJQqapzEpx+GjBZRPYDB4HrVfXTTBgeC/P0DcMwEhNqlE1VnQvMjQq7PU7c0wPbs4HZdbAvJUz0DcMwEmM9cg3DMPKInBL9ggK3NtE3DMOITU6JfpMm0KyZib5hGEY8ckr0wRXxmOgbhmHExkTfMAwjjzDRNwzDyCNyUvStc5ZhGEZsck70mzc3T98wDCMeOSf6VrxjGIYRHxN9wzCMPMJE3zAMI48w0TcMw8gjTPQNwzDyCBN9wzCMPMJE3zAMI4/ISdG3zlmGYRixyUnRN0/fMAwjNjkn+tYj1zAMIz45J/rm6RuGYcTHRN8wDCOPyEnRt4pcwzCM2OSc6BcUwP792bbCMAyjcRJK9EVkpIgsF5GVIjIxQbxLRERFpDwQdqt33nIROTsTRieisNCJvmp9X8kwDOPIo1myCCLSFJgOfA2oAhaKyBxVfS8qXjHwXeD1QFgZMBroC3QG/iIix6vqgczdQm0KCpzgHzjgJkk3DMMwIoTx9IcAK1V1laruA2YBo2LE+xlwN/B5IGwUMEtV96rqR8BKL716o7DQra2IxzAM43DCiH4XYG1gv8oLO4SIDAa6qeqLqZ7rnT9eRCpFpLK6ujqU4fEoKHBra8FjGIZxOHWuyBWRJsB9wA/STUNVH1XVclUt79SpU53sMU/fMAwjPmFKvdcB3QL7Xb0wn2KgH/B3EQE4CpgjIheEODfjmKdvGIYRnzCe/kKgl4j0FJFCXMXsHP+gqm5T1Y6qWqqqpcBrwAWqWunFGy0izUWkJ9ALeCPjdxHAPH3DMIz4JPX0VbVGRG4E5gFNgcdVdamITAYqVXVOgnOXishzwHtADXBDfbbcAfP0DcMwEhGqUaOqzgXmRoXdHifu6VH7U4ApadqXMubpG4ZhxCcne+SCefqGYRixyDnRN0/fMAwjPjkn+ubpG4ZhxCfnRN88fcMwjPjknOibp28YhhGfnBN98/QNwzDik3Oib56+YRhGfHJO9M3TNwzDiE/Oib55+oZhGPHJOdE3T98wDCM+OSf65ukbhmHEJ+dE3zx9wzCM+OSc6JunbxiGEZ+cE33z9A3DMOKTc6Jvnr5hGEZ8ck70mzYFEfP0DcMwYpFzoi/ivH3z9A3DMA4n50QfXLm+efqGYRiHk5Oib56+YRhGbHJS9M3TNwzDiE1Oir55+oZhGLHJSdE3T98wDCM2oURfREaKyHIRWSkiE2Mcv15E3hGRxSLyTxEp88JLRWSPF75YRB7O9A3Ewjx9wzCM2DRLFkFEmgLTga8BVcBCEZmjqu8Foj2tqg978S8A7gNGesc+VNWBmTU7MebpG4ZhxCaMpz8EWKmqq1R1HzALGBWMoKrbA7utAM2cialjnr5hGEZswoh+F2BtYL/KC6uFiNwgIh8C9wA3BQ71FJG3ROQfIjIs1gVEZLyIVIpIZXV1dQrmx8Y8fcMwjNhkrCJXVaer6rHAj4DbvOANQHdVHQTcDDwtIm1inPuoqparanmnTp3qbIt5+oZhGLEJI/rrgG6B/a5eWDxmARcCqOpeVd3ibS8CPgSOT8/U8JinbxiGEZswor8Q6CUiPUWkEBgNzAlGEJFegd3zgBVeeCevIhgROQboBazKhOGJME/fMAwjNklb76hqjYjcCMwDmgKPq+pSEZkMVKrqHOBGETkT2A98BozzTj8NmCwi+4GDwPWq+ml93EgQ8/QNwzBik1T0AVR1LjA3Kuz2wPZ345w3G5hdFwPTwTx9wzCM2FiPXMMwjDwip0R/5kwoLYVnnoFVq9y+YRiGESFU8c6RwMyZMH487N7t9g8ccPsAFRXZs8swDKMxkTOe/qRJEcH32b3bhRuGYRiOnBH9jz9OLdwwDCMfyRnR7949tXDDMIx8JGdEf8oUaNmydljLli7cMAzDcOSM6FdUwKOPQo8ekbBHHrFKXMMwjCA5I/rgBH716oh3f9llWTXHMAyj0ZFTou9TUODW1ivXMAyjNjkt+tYr1zAMozY5KfqFhW5tnr5hGEZtclL0zdM3DMOITU6Kvnn6hmEYsclJ0TdP3zAMIzY5Kfrm6RuGYcQmJ0XfPH3DMIzY5KTom6dvGIYRm5wU/frw9D/9FFQzl55hGEY2yEnRz7Snv2kTHH00vPJKZtIzDMPIFjkp+r6nf8UV0KSJm0KxLlMnbtzoXiBr12bEPMMwjKyRk6L/17+69aZNrkhmzRo3dWK6wr9nj1t//nlm7Ms0Cxa4L5Ft27JtiWEYjZ1Qoi8iI0VkuYisFJGJMY5fLyLviMhiEfmniJQFjt3qnbdcRM7OpPHxuPfew8PqMnWiL/r+urHx7rvwySdQVZVtSwzDaOwkFX0RaQpMB84ByoAxQVH3eFpV+6vqQOAe4D7v3DJgNNAXGAn82kuv3pg5M77Hm+7Uib6H31g9/Z073do8fcMwkhHG0x8CrFTVVaq6D5gFjApGUNXtgd1WgN/OZRQwS1X3qupHwEovvXojkTef7tSJjb14xxf97dsTxzMMw2gWIk4XIFiFWQWcEh1JRG4AbgYKga8Gzn0t6twuaVkakkTefLpTJzZ20d+xw61N9A3DSEbGKnJVdbqqHgv8CLgtlXNFZLyIVIpIZXV1dZ3siOfNl5SkP3ViYxd98/QNwwhLGNFfB3QL7Hf1wuIxC7gwlXNV9VFVLVfV8k6dOoUwKT5TpkCLFrXDiorggQci+wcPuhY9YTlSyvRN9A3DSEYY0V8I9BKRniJSiKuYnROMICK9ArvnASu87TnAaBFpLiI9gV7AG3U3Oz4VFfDgg7XDmjeHK6+MtNd/4QU47jhYvz5cmo3d049VvLNihZsv2DAMI0hS0VfVGuBGYB6wDHhOVZeKyGQRucCLdqOILBWRxbhy/XHeuUuB54D3gJeBG1T1QD3cRy3GjXPe/nHHuf1t22q31589G2pq4IMPwqXX2EU/lqd/4YVw003ZsccwjMZLmIpcVHUuMDcq7PbA9ncTnDsFSLMKNT0KCmDxYhg+/PBju3fDiy+67bBFPKmI/quvwh13wMsvR4aDqG+iRX/jRnjvPSgubpjrG4Zx5JCTPXIBjj/edViKhS+OYUXfF/swnbP+7/9g/vz0+wSkQ7ToL1jg1tZu3zCMaHJW9AG6do0d3rKlW4ct807F09+9260bsnesX6bvi7yJvmEY8chp0f/5zw8Pa9nSVehC/RTvZEP0oz39f/yj9n6Q9evD12UYhpF75LToX3mlq9AtKoqEtWgREclUi3fCiL7/gmgo0VetLfqffgrvvAOtW8OuXa7COsgtt8BFFzWMbYZhND5yWvTBefbByVS2bIkMkfzxx67NfjLS8fTXrXPLxIlwoB7bK+3ZE7mH7dsHzRXgAAAZEklEQVThNa//89e+FgkLsmkTLF+efIIZVVch7RcV3Xmna+pqGMaRTc6L/vbth4uuKog44duwIXka6RbvzJwJd98NK1emZnMq+F5+QYG7148+cvtDvBGOosv1/fzw6zNUYdYs9zIM8uqrMHkyXH01/OlP8NOfwpgx8NZbie0JO7vYwYOwd6/Lrx07ar+8DMOoP0I12TySiefR+uK0Zg10STIaUCqiHyzeefddt+1XtKaLqutf8M1vwpe+VPuYL/qdO7svl48/di8Av49CtKfv769YAb16wb/+5cR8zBh4+ulIvKlT3VfSRx/BZZe5epD9++Eb34AlSyKV4bt2uRfB3/4Gjz/u0j/9dLj4YjjvPGjXzjUfXbvW5fNf/uJeIq+9Frs1VEGB60zXvLkrloveLix0E9p8/rl7adgUlkYuceKJzgmrT3Je9Fu0iC0uxcVOjFevhi9/OXEaqZTpBz19/wujrqK/fTs89hh88YvxRb9LF/cCe/99t92unQuP5elD5OvjV79y62eegUsvdRPQ7N8PL73khrR49VXX52DqVOjYEc48E556Ck47zfV+fvvtiId+xhlu7KNXXokUBbVte7gNAwbAt77l7qdpU7fU1DgR98XcX0dv79sHbdrAF77gXgBNcv5b1cgnjj22/q+R86J/0knOm432CH2hClOZG/T0/aKhePiiv3EjfPaZ267rmDhbt9ZeB/FfKJ07u/V777mmqm3buv1owfX3V6xwRVuzZzsBnjMHLrnEedPNmjmB/853XPHO3LnOwxeBwYPdOEYzZ7q8u+02KC+Hk0+Go45yaR886Dz5V1+FDz90x/v0cfHLy6F377rlh2EY6ZPzoj9okCt+2Lu3dkuWXbvc+pVX4NZb3XZNjSviqKhw3qePL/oHDzovOFFPWz+uqrsmhPf0Dx6EadPg8strFzn5Yu+/RIIEi3fAiezJJ8cW/ZqayEtpxQr4zW9c2A9/COefD88+68rxS0tdeGGh+2K47rpIGt/9rhvmAtz5wWM+TZq4r6foL6hhw0Jlg2EY9UjOfxy3a+eEOF7Z72uB0f7nz3eC9q9/1Y4TLB5KVsSze/fhwx+E9fRXrIAf/MC1vAlWrKYi+qq1Pf3gtYMvnw8+gCeecEUyvXo50Z8xA445xol2vBfb5Ze7+XhPPtnVMRiGcWSR86Lftq3zoOM1m9yzJzL6pj9sQ3SRyOefR8qOw4h+r161w6JFf9Uq6NEj0tLGx7/usmWuYtUnVvHO7NlOfDdudPvBL4N4xTu+HZ07u2uvXg3XXJP4fqJp3ty9KF96ycrTDeNIJOf/tn6FZiL80TfnzXP7vvfss2dPJJ1kor9nT0T0jzvOFRNFF++8/bZrZbNoUe1wP96QIe6rwy+O8oU76Om/8op7SflNKH1PH6BbNyfOBQWxRf+kk9y6uDi9jlrdu7tJaQzDOPLIK9FPVgH7pz+57aDoqzohb9/e7Yfx9Dt3dk0a+/d3LU2iPf1PP3Xr6F67vuifcooTfL8tfazinSVL3HrxYrcOin7Xru5eo1vO+HaUl7v1ZZdFml4ahpEf5JXoxxpqOUj02DXgmggG00kk+qpO9Fu1gv/6L1fpGUv0/fL6eKLve+IrvKlooot3Dh50Qy2AWzdp4po/+nTz5iqLFn1/e/hwGD3aDclgGEZ+kTeiLxKpeCwoSHzOc8+5Mn6IVOKG8fT37XPC36KFm8Bk+PBIf4AgYUXfHxjNF/u9e509q1dHvkb27nXXaNPG7Tdr5tqwgxP94AvH3/7CF1y7fGs6aRj5R86Lvl+h2aWLG2MfoGfPxEU9+/fDpEluOxXR95tDBotM0ineOeYYZ3e0p+9v+0U7rVq5devWru6gVSt3n34Fa7ziHf8FYRhG/pHzou97+j16OLEH5+Em677vT4JSH6Ifz9Pfvt0JdosW7gUV7emDK9dfssS9tM4+24W1bh25VrfANPRt2kSmivTT98MNw8hPcl70fU+/tBQ6dXKtWv761+Tnde/u1r7I+5Oon322E2YRt3TseHhRUIsWkXRiFe/4nv66dbUHGduxw8UXcS2AEon+scfCwIEuzBf9Y45xQxwE733bNtcsc/RoJ/oikS8EwzDyj5zvkVtQ4AYxGjrU9bZVjfTGTcS557q1L+R//nPkWPArYcuWSF1B//5uHdbTr6lxQx37wxf4og/O03/mGffS2brVlcNv2hQp3jnxxMigav458+a5Mn2ftm3dS+KFF9wXz1FHubjWvt4w8pecF32INGssLY20xknGQw+5Ct0wQwfs2+fqAPzR8cKIfqdOUF3tinjiib6qG1Zh2zZn+6ZNrm3+ypWu85Y/OJPv6Ud78G3bRr4ydu501/W/fAzDyE/yyudLdbLyLVvgD38In3a84p2dOyPFOKqueOfEE91+sFw/KPp+B68VK5x370/xWFnp0jjhhIin74t+NEGBP3jQDcZm5fmGkd+EEn0RGSkiy0VkpYhMjHH8ZhF5T0SWiMhfRaRH4NgBEVnsLXMyaXyq+OX09UGTJvDii2472tOHSBPL3btdM0u/7D2e6PstjZYtqy36b7wROd6hgyv2idc7Nlrgly410TeMfCep6ItIU2A6cA5QBowRkbKoaG8B5ao6AHgeuCdwbI+qDvSWCzJkd1pMmRK7B2rz5nVP+8AB+OUv3XYs0feLePzy/N693aBmq1fD88+7F8GOHZH4bdq4nrULFzovvVMnV3zjN9f0vwReeQV+8pPYNvme/ogRbr13r4m+YeQ7YTz9IcBKVV2lqvuAWcCoYARVna+qXoNFXgO6ZtbMzFBRAY8+6ppvirj1UUe54ZeTddgKg19fEBR933P3y9b9ljsdO7o29Q884Maqf/ll92IIjtDZpw/8+99uu10712y0psb1vvUFfcCA2r1xg/jNVceOjfRLMNE3jPwmjOh3AdYG9qu8sHhcC7wU2C8SkUoReU1ELkzDxoxSUeG864MH3bp7dyeEF6TwDTJjRuLOXcceGxm5M56n36GDa1PvD6pWVVW7eAegrCwyiqYv+hAp+knGsGFw++1uOGR/bB4TfcPIbzJakSsiY4Fy4BeB4B6qWg5cAUwTkcMmBBOR8d6LobK6ujqTJiWlQwfnfXfu7LznZKNOFhe7F0ey+gF/5E5/vP5oT7+kBO69180Z27Spm8Uqluj7tGsX8dyjh26OR8uWbkLzVq0idQLWescw8pswor8OCPTzpKsXVgsROROYBFygqnv9cFVd561XAX8HBkWfq6qPqmq5qpZ36tQppRuoKx06uLbs1dWuyMWfg7ZZnMasJ5/s1lOmJPb2wVXa/uxnbttv5x/09E8+2U1i8sUvuq+Ompr4ot+2beqefhBf9M3TN4z8JozoLwR6iUhPESkERgO1WuGIyCDgEZzgbwqEtxeR5t52R2Ao8F6mjM8E7ds773vzZldZ6jd/nDatdquYDh3cvj/MQUVF8qEcgjz0kCvu8UU/mPbRR0d630aX6fukU7wTpIfXnspE3zDym6Sir6o1wI3APGAZ8JyqLhWRySLil4T/AmgN/D6qaWYfoFJE3gbmA1NVtVGJfocOrknkwoWu3bsv+mef7V4EP/whFBU5sS4pqT32TtcUqqv37YMrr4S77nL7J5wQGb4hnuiXlEQqaYPFO+bpG4aRLqF65KrqXGBuVNjtge0z45z3b6B/XQysbzp0cB779u3wn//pertCpF39nj2RzlYtWkREf/HiyABrYVGNnO+X+YMTfX80zLfegjvvdJ29/BmqNm50xTvdurny+WMPqxVJjom+YRiQZz1yY+EXmVxxhRs7x/f0Y4l+UVFEtH/wA1fu/6MfuYrYdNi92w3f4A/DAPDII+6FoOrWH3zghL6wEL7zHddOv6go9WudeqobdC3MsBKGYeQueS/65eVutEq/wtUX/a1bXeuaRYsiIuuL/vvvw9/+Bt/7Hkyd6ppwQqTyN5VRLD/+GNYGGsTu3Vv7eE1NxDtv0cKNpJkOxcVuALfgC8YwjPwjLwZcS0TfvpHJxSEi+i+/DNOnu+0zznDroiL3Mnj4YdeZ69prXXi/fm5dU+PG7Pfb1oehZUv43e8Sx/nkk/DpGYZhJCLvPf1ofNFfuNCtly6Fl7yuZkVFToCffBIuvTQyLeHxx0d69K5fn1pZf5hhnps0iVT6zpzpmpbGGs/fMAwjGXnv6Ufji/7bb7timj59Iu3xi4pcOXvz5jAxMOxcYaFrjfPuu4cXz2SCAwdcpe+//gWPPeamc/QJjudfUZH5axuGkVuYpx+FL/p797pB0YIdsPyy/V//uvYMVRCZQCU4rHIm2b3bVfIGBd/HH8/fMAwjGSb6URQVRWaW6t279rHx491Imr5nHcQX/UGDDh/Js6Ag/vDHqRCcWjGaNWsi4/0YhmHEw0Q/CpGItx8t+l/+Mtx4Y+zz/MrcsrLDR/J84gnX0atHj9jnZgq/7X9Q+GfOdC+DJk3spWAYhol+TOKJfiKC8+NGj+Tpl7XHG88/k/ht/8EJ/Pjxtdv9jx1rlb+Gkc+Y6MfAF/3g2DfJ6N7dtaH3pzCMRXA8f4h06urRwzULzcSY/uDE/TvfgXHjYrck2rLFDQnxne9k5nqGYRw5iKYyalgDUF5erpWVlVm14aSTIsMspDKr1oED6ffOBed9T5rkOmx16OCGWg47kXs6iMBTT1mrH8PIBURkkTeMfULM049B69bOa091GsW6CD7ULhbavBkefzwzFcDxULVWP4aRb5jox+Daa+GWW7JthXsJbN4cX/iTjecfBmv1Yxj5hYl+DK66Cr797WxbEeGBBw6vAG7ZEq6/PjMtgmK1+jEMIzcx0T8CiDWh+6OPuk5iq1e7Ad/qWgkcbPVjGEbuYqJ/hBCvGah/LNk4+SUl7uWQqN5hzRrz9o3cw/qq1MZEP0fwJ1yPRcuWroiooiJxr1448ot57A/euMnE75NKGrH6qhzpz3idUdVGtZx00klqpE6PHqrusa69NG2qOmNG8njBpUeP+rV1xgx3DRG3DtpX13Rbtqx9Ly1bZi79hqK+8ifbZOL3STWNeM97fT/j2QCo1BAam3WRj15M9NMj7J8hVrxYS7btTIdc+IOn8jseaS+GdH6f6PssKUktDZHY8UUyfntZx0Q/DwkrBH68eIIvkvzcdMWmPoU57B88eA8lJW6Jdz8zZtQWmlatEsePd52weRUmfzL54kzFxrr+9qkKcFgHJZhH0TYly89UnoVUyMZL2UTfSMqMGfH/iME/UfAlER0/VS800R+/vl8o0QIea/HvJ0zc6PhB2ydMiC/M6eZPsvts2jQ18Qrz8kiUD6m+aBI5GqkIdpjfI9E9ppNWrN831n+ipCTxbx+d/5l84ZjoG6FI9uAXFKgWFiaO06RJcrGbMCGxiJaUpC6SYYTW/0OWlCS/j0S2pLrEE+9WrRILQhhPP9GLOp54+SIVzMNEolpS4mxNlnaYohlwL6RE+RJLGFO9z6DtsQQ13bTSPTdWXqXqTKT2XzbRN0KQjjeV6aVly/h/hFh/fD8s+pi/H0ZgGuMS78UVLRqN4TcL5nm0kJWUqJ5xRnr573+tJCq/T2cpKHDOSbbzq6AgtechFTIq+sBIYDmwEpgY4/jNwHvAEuCvQI/AsXHACm8Zl+xaJvoNS10+fzOx+K2LMi3QR5rg+0vLlukLZjaWkpLUhCyVJcxXZi4vqVY2hxX9pO30RaQpMB04BygDxohIWVS0t4ByVR0APA/c453bAbgDOAUYAtwhIu3TaFlq1BN+b99scfCgs6F798ymq5rZ9BqK3bvhr39Nzf4mWeptU1AAn30WewrPTLB/PxQX1//kQ42VTP8nfMI8LkOAlaq6SlX3AbOAUcEIqjpfVf2R218DunrbZwOvqOqnqvoZ8Aruq8FoRFRUZO+P1aGD62CzZk12rp8LJOtwV1/s31//196yxQ06mG8UFrpJl+qDMKLfBVgb2K/ywuJxLfBSKueKyHgRqRSRyurq6hAmGZmmIWb1iqZpUzdnQGMX/LoOmW3UjV27sm1Bw1NcXH/zXGT0w1BExgLlwC9SOU9VH1XVclUt79SpUyZNMkISPatXfdOqlSvCqM9JYjJBYSE8+WT+FjGAG+RvwoTGlwf+0OIlJe55SufcTFAfzlKiYVXqShjRXwd0C+x39cJqISJnApOAC1R1byrnGo0Df1C3GTNiP8itW0dG+azL5C4TJjjBz1axRCKCYlBS4iayqahI/iXUsqXLtwkTDheUTApMXSkoSE0ge/Rws6v9+tduRrd0SFTnUJe8eeop9xxt3gw7d4Z/Jnv0cMOSxxquPFkaPXq43zl6xNt0Xoh+OrGor/J8gKQ1vUAzYBXQEygE3gb6RsUZBHwI9IoK7wB8BLT3lo+ADomuZ613GgfJOkola/UTr/VJSUn4Jod+00t/XddWJsF229GtQsK0iw7bmSZW/4F0W7jU5d7jnRvdGzXe+dFNBpP9biKx+wMk6gQ2Y0Z69xirOWOYlmjBFjGxnvFEaSR6RuKdl+h/EO+8dHtXk+Emm+cCH3jCPskLm4zz6gH+AmwEFnvLnMC538Q19VwJXJPsWib6Rw6JRDBRr8QwzRGj/5x1aVbaGMauCdspJ9ruRJ3dkolTKsNS1HXcJl/wE91/vPxO9fdNJr6pvMiSpeG/kMI8I2FfIrF6DWfiWcyo6DfkYqKfO8R7mNMZ6TNsN3hwXnWmxlCpD+IJQSwv2Y+fKDyeOKUyzlE64zalIohh8iToQAR7A6cy3lEwvcYy4mpDORgm+kajJZlnl85wuw09uFVdaQibG5PwZYMj8bmoC2FFX1zcxkN5eblWVlZm2wyjnpk5003P+PHHrq0+uBYL3bu7StP6aq6WbwTz2fI2txGRRapanjSeib5hGMaRT1jRt+kSDcMw8ggTfcMwjDzCRN8wDCOPMNE3DMPII0z0DcMw8ohG13pHRKqBuoy72BFojIOxml2p0VjtgsZrm9mVGo3VLkjPth6qmnTEykYn+nVFRCrDNFtqaMyu1GisdkHjtc3sSo3GahfUr21WvGMYhpFHmOgbhmHkEbko+lmc8TUhZldqNFa7oPHaZnalRmO1C+rRtpwr0zcMwzDik4uevmEYhhEHE33DMIw8ImdEX0RGishyEVkpIhOzaEc3EZkvIu+JyFIR+a4XfqeIrBORxd5ybpbsWy0i73g2VHphHUTkFRFZ4a3bN7BNJwTyZbGIbBeR72Ujz0TkcRHZJCLvBsJi5o84HvSeuSUiMriB7fqFiLzvXfsFEWnnhZeKyJ5Avj1cX3YlsC3ubycit3p5tlxEzm5gu54N2LRaRBZ74Q2WZwk0omGeszCD7jf2BWiKm8rxGCLz+JZlyZajgcHedjFumsky4E7glkaQV6uBjlFh9wATve2JwN1Z/i0/AXpkI8+A04DBwLvJ8gc3jehLgACnAq83sF1nAc287bsDdpUG42Upz2L+dt5/4W2gOW7e7Q+Bpg1lV9Tx/wZub+g8S6ARDfKc5YqnPwRYqaqrVHUfMAsYlQ1DVHWDqr7pbe8AlgFdsmFLCowCnvS2nwQuzKItZwAfqmpdemWnjaouAD6NCo6XP6OA/1HHa0A7ETm6oexS1f+nqjXe7mtA1/q4djLi5Fk8RgGzVHWvqn6Emzt7SEPbJSICXAY8Ux/XTkQCjWiQ5yxXRL8LsDawX0UjEFoRKQUGAa97QTd6n2ePN3QRSgAF/p+ILBKR8V7YF1V1g7f9CfDF7JgGwGhq/xEbQ57Fy5/G9Nx9E+cN+vQUkbdE5B8iMixLNsX67RpLng0DNqrqikBYg+dZlEY0yHOWK6Lf6BCR1sBs4Huquh14CDgWGAhswH1aZoOvqOpg4BzgBhE5LXhQ3fdkVtrxikghcAHwey+oseTZIbKZP/EQkUlADTDTC9oAdFfVQcDNwNMi0qaBzWp0v10UY6jtXDR4nsXQiEPU53OWK6K/DugW2O/qhWUFESnA/ZgzVfV/AVR1o6oeUNWDwG+op0/aZKjqOm+9CXjBs2Oj/7norTdlwzbci+hNVd3o2dgo8oz4+ZP1505Erga+DlR4QoFXdLLF216EKzc/viHtSvDbNYY8awZcDDzrhzV0nsXSCBroOcsV0V8I9BKRnp63OBqYkw1DvLLC3wLLVPW+QHiwDO4i4N3ocxvAtlYiUuxv4yoC38Xl1Tgv2jjgjw1tm0ct76sx5JlHvPyZA1zlta44FdgW+Dyvd0RkJPCfwAWqujsQ3klEmnrbxwC9gFUNZZd33Xi/3RxgtIg0F5Genm1vNKRtwJnA+6pa5Qc0ZJ7F0wga6jlriNrqhlhwNdwf4N7Qk7Jox1dwn2VLgMXeci7wFPCOFz4HODoLth2DaznxNrDUzyegBPgrsAL4C9AhC7a1ArYAbQNhDZ5nuJfOBmA/ruz02nj5g2tNMd175t4ByhvYrpW4sl7/OXvYi3uJ9/suBt4Ezs9CnsX97YBJXp4tB85pSLu88N8B10fFbbA8S6ARDfKc2TAMhmEYeUSuFO8YhmEYITDRNwzDyCNM9A3DMPIIE33DMIw8wkTfMAwjjzDRNwzDyCNM9A3DMPKI/w8B2sdePfoazgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PELM_s Result\n",
      "Accuracy : 0.9032258064516129\n",
      "AUC : 0.9009546041935887\n",
      "Sensitivity : 0.9596273291925466\n",
      "Specificity : 0.8422818791946308\n",
      "F1 : 0.911504424778761\n",
      "MCC : 0.8102784823781962\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Plot the training accuracy model transformer\n",
    "\n",
    "accuracy = model_transformer_train.history['acc']\n",
    "val_accuracy = model_transformer_train.history['val_acc']\n",
    "loss = model_transformer_train.history['loss']\n",
    "val_loss = model_transformer_train.history['val_loss']\n",
    "epochs = range(len(accuracy))\n",
    "plt.plot(epochs, accuracy, 'bo', label='Training accuracy')\n",
    "plt.plot(epochs, val_accuracy, 'b', label='Validation accuracy')\n",
    "plt.title('Transformer Training and validation accuracy')\n",
    "plt.legend()\n",
    "plt.figure()\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Transformer Training and validation loss')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "y_pred = np.argmax(model_transformer.predict(valid_X), axis=1)\n",
    "y_true = np.argmax(valid_Y, axis = 1)\n",
    "\n",
    "tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()\n",
    "f1 = f1_score(y_true, y_pred, labels=None, pos_label=1, average='binary', sample_weight=None)\n",
    "auc = roc_auc_score(y_true, y_pred, average='macro', sample_weight=None, max_fpr=None)\n",
    "sensi = tp/(tp+fn)\n",
    "specificity = tn/(tn+fp)\n",
    "accu = (tn + tp)/(tn + tp + fn + fp)\n",
    "mcc = ((tp*tn)-(fp*fn))/np.sqrt((tp+fp)*(tp+fn)*(tn+fp)*(tn+fn))\n",
    "print('{} Result'.format(string_name))\n",
    "print('Accuracy :', accu)\n",
    "print('AUC :', auc)\n",
    "print('Sensitivity :', sensi)\n",
    "print('Specificity :', specificity)\n",
    "print('F1 :', f1)\n",
    "print('MCC :', mcc)\n",
    "print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}