failing-RNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# An example where a conventional RNN fails, due to a lack of symmetry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "import matplotlib.pyplot as plt\n",
    "import sys\n",
    "from os import path\n",
    "from keras.utils.np_utils import to_categorical   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simulate Data\n",
    "$m$ sequences each of length $\\ell$ and binary class $y$.  \n",
    "class $y\\sim \\text{Bernoulli}(0.5)$  \n",
    "\n",
    "$$x=(x_1,\\ldots, x_\\ell)$$\n",
    "$x_i \\sim \\text{Bernoulli}(0.5)$ (iid)\n",
    "Falls $y_i=1$, setze\n",
    "$$x[t..t+10) \\leftarrow 1$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ell = 100\n",
    "m = 5000\n",
    "siglen = 10\n",
    "s=2 # alphabet size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(13)\n",
    "\n",
    "# development test data\n",
    "def simulate_data(m, t, oneHot=True, standardize=False):\n",
    "    x = np.random.randint(2, size = m*ell).reshape(m, ell)\n",
    "    y = np.random.randint(2, size = m)\n",
    "    for i in range(m):\n",
    "        if y[i]:\n",
    "            x[i, t:t + siglen] = 1\n",
    "    if oneHot:\n",
    "        x = to_categorical(x, num_classes=2)\n",
    "    if standardize: # is better for SimpleRNN, oneHot or not\n",
    "        x = x - 0.5 # standardize\n",
    "    return x, y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Obtain the training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "regenerate = True\n",
    "fname = \"rnn-fail-data.npz\"\n",
    "\n",
    "if regenerate or not path.exists(fname):\n",
    "    # simulate new data\n",
    "    x, y = simulate_data(3, t=20)\n",
    "    np.savez(fname, x=x, y=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y [1 1 1] \n",
      "x (3, 100, 2) [[1. 0.]\n",
      " [1. 0.]\n",
      " [1. 0.]\n",
      " [1. 0.]\n",
      " [1. 0.]\n",
      " [1. 0.]\n",
      " [1. 0.]\n",
      " [0. 1.]\n",
      " [1. 0.]\n",
      " [1. 0.]]\n"
     ]
    }
   ],
   "source": [
    "npzfile = np.load(fname)\n",
    "x = npzfile['x']\n",
    "y = npzfile['y']\n",
    "print (\"y\", y, \"\\nx\", x.shape, x[0,0:10,:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from HMMCell import HMMCell, HMMLayer, TransitionMatrixChainInitializer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 4, 4), dtype=float32, numpy=\n",
       "array([[[2., 2., 0., 0.],\n",
       "        [2., 2., 2., 0.],\n",
       "        [0., 2., 2., 2.],\n",
       "        [0., 0., 2., 2.]],\n",
       "\n",
       "       [[2., 2., 0., 0.],\n",
       "        [2., 2., 2., 0.],\n",
       "        [0., 2., 2., 2.],\n",
       "        [0., 0., 2., 2.]]], dtype=float32)>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ini = TransitionMatrixChainInitializer(2.0)\n",
    "ini([2,4,4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 24), dtype=float32, numpy=\n",
       "array([[-3.2055378e-04,  3.2448769e-04,  6.8964861e-02,  9.7857356e-02,\n",
       "         9.6753560e-02,  9.4772890e-02,  9.5709525e-02,  9.8255672e-02,\n",
       "         9.4166905e-02,  9.3973182e-02,  9.4205320e-02,  9.6159019e-02,\n",
       "         6.9181740e-02,  6.7550585e-02,  9.3670122e-02,  9.4463035e-02,\n",
       "         9.5842540e-02,  9.8553643e-02,  9.7892098e-02,  9.3036875e-02,\n",
       "         9.4120882e-02,  9.9455453e-02,  9.6613996e-02,  6.8800777e-02],\n",
       "       [-1.1615157e-03,  1.1164546e-03,  6.8668537e-02,  9.7381487e-02,\n",
       "         9.6543975e-02,  9.4569050e-02,  9.5683083e-02,  9.8028757e-02,\n",
       "         9.4520882e-02,  9.4395436e-02,  9.4784632e-02,  9.6238218e-02,\n",
       "         6.9185935e-02,  6.8131670e-02,  9.4483100e-02,  9.4915330e-02,\n",
       "         9.5678747e-02,  9.8003015e-02,  9.7637810e-02,  9.3155891e-02,\n",
       "         9.4234496e-02,  9.8961376e-02,  9.6231565e-02,  6.8566971e-02],\n",
       "       [-9.7751617e-06,  1.2516975e-06,  6.9288574e-02,  9.4259784e-02,\n",
       "         9.5204204e-02,  9.6273452e-02,  9.6270971e-02,  9.3590535e-02,\n",
       "         9.7532757e-02,  9.6139088e-02,  9.6948892e-02,  9.5340177e-02,\n",
       "         6.9151603e-02,  6.9779627e-02,  9.7024731e-02,  9.6349098e-02,\n",
       "         9.6117571e-02,  9.4477601e-02,  9.4125710e-02,  9.7454175e-02,\n",
       "         9.7040243e-02,  9.2713922e-02,  9.6107967e-02,  6.8809368e-02]],\n",
       "      dtype=float32)>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seqLayer = HMMLayer(2, 11)\n",
    "output = seqLayer(x)\n",
    "output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model(bidirectional = False, seqModelType = \"SimpleRNN\", RNNunits = 32):\n",
    "    model = keras.Sequential()\n",
    "    model.add(layers.InputLayer(input_shape=(None,s)))\n",
    "\n",
    "    if seqModelType == \"HMM\":\n",
    "        seqLayer = HMMLayer(10, 15) # (10,15) is better than (5,11)\n",
    "    elif seqModelType == \"LSTM\":\n",
    "        seqLayer = layers.LSTM(RNNunits)\n",
    "    elif seqModelType == \"GRU\":\n",
    "        seqLayer = layers.GRU(RNNunits)\n",
    "    elif seqModelType == \"SimpleRNN\":\n",
    "        seqLayer = layers.SimpleRNN(RNNunits)\n",
    "    else:\n",
    "        sys.exit(\"unknown sequence model type \" + seqModelType)\n",
    "\n",
    "    if bidirectional:\n",
    "        seqLayer = layers.Bidirectional(seqLayer)\n",
    "    \n",
    "    model.add(seqLayer)\n",
    "    model.add(layers.Dense(1, activation='sigmoid'))\n",
    "    lr = 1e-3\n",
    "    #if seqModelType == \"HMM\":\n",
    "    #    lr = 1e-2\n",
    "    print (f\"lr={lr}\")\n",
    "    model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = lr),\n",
    "                  loss = tf.keras.losses.BinaryCrossentropy(), metrics = [\"accuracy\"])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lr=0.001\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "simple_rnn (SimpleRNN)       (None, 32)                1120      \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 1)                 33        \n",
      "=================================================================\n",
      "Total params: 1,153\n",
      "Trainable params: 1,153\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "lr=0.001\n",
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "hmm_layer_1 (HMMLayer)       (None, 160)               2700      \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 161       \n",
      "=================================================================\n",
      "Total params: 2,861\n",
      "Trainable params: 2,861\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "transition matrices A:\n",
      " [[[0.37775 0.37775 0.01881 ... 0.01881 0.01881 0.01881]\n",
      "  [0.27798 0.27798 0.27798 ... 0.01384 0.01384 0.01384]\n",
      "  [0.01384 0.27798 0.27798 ... 0.01384 0.01384 0.01384]\n",
      "  ...\n",
      "  [0.01384 0.01384 0.01384 ... 0.27798 0.27798 0.01384]\n",
      "  [0.01384 0.01384 0.01384 ... 0.27798 0.27798 0.27798]\n",
      "  [0.01881 0.01881 0.01881 ... 0.01881 0.37775 0.37775]]\n",
      "\n",
      " [[0.37775 0.37775 0.01881 ... 0.01881 0.01881 0.01881]\n",
      "  [0.27798 0.27798 0.27798 ... 0.01384 0.01384 0.01384]\n",
      "  [0.01384 0.27798 0.27798 ... 0.01384 0.01384 0.01384]\n",
      "  ...\n",
      "  [0.01384 0.01384 0.01384 ... 0.27798 0.27798 0.01384]\n",
      "  [0.01384 0.01384 0.01384 ... 0.27798 0.27798 0.27798]\n",
      "  [0.01881 0.01881 0.01881 ... 0.01881 0.37775 0.37775]]\n",
      "\n",
      " [[0.37775 0.37775 0.01881 ... 0.01881 0.01881 0.01881]\n",
      "  [0.27798 0.27798 0.27798 ... 0.01384 0.01384 0.01384]\n",
      "  [0.01384 0.27798 0.27798 ... 0.01384 0.01384 0.01384]\n",
      "  ...\n",
      "  [0.01384 0.01384 0.01384 ... 0.27798 0.27798 0.01384]\n",
      "  [0.01384 0.01384 0.01384 ... 0.27798 0.27798 0.27798]\n",
      "  [0.01881 0.01881 0.01881 ... 0.01881 0.37775 0.37775]]\n",
      "\n",
      " ...\n",
      "\n",
      " [[0.37775 0.37775 0.01881 ... 0.01881 0.01881 0.01881]\n",
      "  [0.27798 0.27798 0.27798 ... 0.01384 0.01384 0.01384]\n",
      "  [0.01384 0.27798 0.27798 ... 0.01384 0.01384 0.01384]\n",
      "  ...\n",
      "  [0.01384 0.01384 0.01384 ... 0.27798 0.27798 0.01384]\n",
      "  [0.01384 0.01384 0.01384 ... 0.27798 0.27798 0.27798]\n",
      "  [0.01881 0.01881 0.01881 ... 0.01881 0.37775 0.37775]]\n",
      "\n",
      " [[0.37775 0.37775 0.01881 ... 0.01881 0.01881 0.01881]\n",
      "  [0.27798 0.27798 0.27798 ... 0.01384 0.01384 0.01384]\n",
      "  [0.01384 0.27798 0.27798 ... 0.01384 0.01384 0.01384]\n",
      "  ...\n",
      "  [0.01384 0.01384 0.01384 ... 0.27798 0.27798 0.01384]\n",
      "  [0.01384 0.01384 0.01384 ... 0.27798 0.27798 0.27798]\n",
      "  [0.01881 0.01881 0.01881 ... 0.01881 0.37775 0.37775]]\n",
      "\n",
      " [[0.37775 0.37775 0.01881 ... 0.01881 0.01881 0.01881]\n",
      "  [0.27798 0.27798 0.27798 ... 0.01384 0.01384 0.01384]\n",
      "  [0.01384 0.27798 0.27798 ... 0.01384 0.01384 0.01384]\n",
      "  ...\n",
      "  [0.01384 0.01384 0.01384 ... 0.27798 0.27798 0.01384]\n",
      "  [0.01384 0.01384 0.01384 ... 0.27798 0.27798 0.27798]\n",
      "  [0.01881 0.01881 0.01881 ... 0.01881 0.37775 0.37775]]]\n"
     ]
    }
   ],
   "source": [
    "model = get_model(bidirectional = False, seqModelType = \"SimpleRNN\", RNNunits = 32)\n",
    "model.summary()\n",
    "model = get_model(bidirectional = False, seqModelType = \"HMM\")\n",
    "model(x)\n",
    "model.summary()\n",
    "W = model.get_layer(index=0).C\n",
    "W.print_pars()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ad hoc fixed prediction method\n",
    "$$ \\hat{y} = 1 :\\Leftrightarrow \\exists j: x[j..j+10) = 1$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def adhoc_pred(x):\n",
    "    x = np.squeeze(x+.5).astype('int')\n",
    "    A = np.cumsum(x, axis=1)\n",
    "    B = np.zeros_like(A)\n",
    "    B[:,siglen:] = A[:,:-siglen]\n",
    "    yhat = np.any(A-B >= siglen, axis=1).astype('int')\n",
    "    return yhat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def adhoc_acc(x_test, y_test):\n",
    "    yhat = adhoc_pred(x_test)\n",
    "    correct = (yhat == y_test).astype('int')\n",
    "    return correct.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from time import time\n",
    "\n",
    "# ts = np.array(range(ell - 15, -1, -10))\n",
    "#ts = np.array(range(ell - 15, 20, -20))\n",
    "ts = np.array([50])\n",
    "\n",
    "len(ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HMM dir= False \tlr=0.001\n",
      "Epoch 1/40\n",
      "313/313 [==============================] - 10s 30ms/step - loss: 0.6920 - accuracy: 0.5212 - val_loss: 0.6815 - val_accuracy: 0.6300\n",
      "Epoch 2/40\n",
      "313/313 [==============================] - 10s 31ms/step - loss: 0.6650 - accuracy: 0.6674 - val_loss: 0.6331 - val_accuracy: 0.6600\n",
      "Epoch 3/40\n",
      "313/313 [==============================] - 11s 35ms/step - loss: 0.6138 - accuracy: 0.7175 - val_loss: 0.6036 - val_accuracy: 0.6700\n",
      "Epoch 4/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.5876 - accuracy: 0.7166 - val_loss: 0.5689 - val_accuracy: 0.7000\n",
      "Epoch 5/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.5634 - accuracy: 0.7348 - val_loss: 0.5659 - val_accuracy: 0.7000\n",
      "Epoch 6/40\n",
      "313/313 [==============================] - 11s 36ms/step - loss: 0.5452 - accuracy: 0.7527 - val_loss: 0.5407 - val_accuracy: 0.7100\n",
      "Epoch 7/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.5351 - accuracy: 0.7534 - val_loss: 0.5285 - val_accuracy: 0.7300\n",
      "Epoch 8/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.5238 - accuracy: 0.7626 - val_loss: 0.5130 - val_accuracy: 0.7500\n",
      "Epoch 9/40\n",
      "313/313 [==============================] - 11s 36ms/step - loss: 0.5044 - accuracy: 0.7883 - val_loss: 0.4962 - val_accuracy: 0.7800\n",
      "Epoch 10/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.4864 - accuracy: 0.8023 - val_loss: 0.4704 - val_accuracy: 0.8000\n",
      "Epoch 11/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.4737 - accuracy: 0.8242 - val_loss: 0.4632 - val_accuracy: 0.8000\n",
      "Epoch 12/40\n",
      "313/313 [==============================] - 11s 36ms/step - loss: 0.4530 - accuracy: 0.8350 - val_loss: 0.4393 - val_accuracy: 0.8200\n",
      "Epoch 13/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.4468 - accuracy: 0.8352 - val_loss: 0.4206 - val_accuracy: 0.8400\n",
      "Epoch 14/40\n",
      "313/313 [==============================] - 11s 36ms/step - loss: 0.4299 - accuracy: 0.8536 - val_loss: 0.4007 - val_accuracy: 0.8600\n",
      "Epoch 15/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.4196 - accuracy: 0.8469 - val_loss: 0.3897 - val_accuracy: 0.8800\n",
      "Epoch 16/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.4103 - accuracy: 0.8568 - val_loss: 0.3946 - val_accuracy: 0.8600\n",
      "Epoch 17/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.3994 - accuracy: 0.8577 - val_loss: 0.3735 - val_accuracy: 0.9000\n",
      "Epoch 18/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.3925 - accuracy: 0.8669 - val_loss: 0.3593 - val_accuracy: 0.9000\n",
      "Epoch 19/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.3855 - accuracy: 0.8766 - val_loss: 0.3515 - val_accuracy: 0.9000\n",
      "Epoch 20/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.3837 - accuracy: 0.8721 - val_loss: 0.3540 - val_accuracy: 0.9100\n",
      "Epoch 21/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.3681 - accuracy: 0.8801 - val_loss: 0.3432 - val_accuracy: 0.9100\n",
      "Epoch 22/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.3563 - accuracy: 0.8842 - val_loss: 0.3371 - val_accuracy: 0.9100\n",
      "Epoch 23/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.3533 - accuracy: 0.8965 - val_loss: 0.3207 - val_accuracy: 0.9100\n",
      "Epoch 24/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.3462 - accuracy: 0.8919 - val_loss: 0.3184 - val_accuracy: 0.9200\n",
      "Epoch 25/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.3536 - accuracy: 0.8834 - val_loss: 0.3143 - val_accuracy: 0.9200\n",
      "Epoch 26/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.3454 - accuracy: 0.8916 - val_loss: 0.3193 - val_accuracy: 0.9200\n",
      "Epoch 27/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.3360 - accuracy: 0.8974 - val_loss: 0.3108 - val_accuracy: 0.9200\n",
      "Epoch 28/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.3367 - accuracy: 0.8991 - val_loss: 0.2978 - val_accuracy: 0.9300\n",
      "Epoch 29/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.3237 - accuracy: 0.9026 - val_loss: 0.2891 - val_accuracy: 0.9300\n",
      "Epoch 30/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.3307 - accuracy: 0.9024 - val_loss: 0.2881 - val_accuracy: 0.9300\n",
      "Epoch 31/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.3126 - accuracy: 0.9132 - val_loss: 0.2896 - val_accuracy: 0.9200\n",
      "Epoch 32/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.3079 - accuracy: 0.9108 - val_loss: 0.2941 - val_accuracy: 0.9100\n",
      "Epoch 33/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.3171 - accuracy: 0.8990 - val_loss: 0.2746 - val_accuracy: 0.9300\n",
      "Epoch 34/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.3071 - accuracy: 0.9196 - val_loss: 0.2853 - val_accuracy: 0.9100\n",
      "Epoch 35/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.2990 - accuracy: 0.9236 - val_loss: 0.2754 - val_accuracy: 0.9200\n",
      "Epoch 36/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.2916 - accuracy: 0.9256 - val_loss: 0.2685 - val_accuracy: 0.9300\n",
      "Epoch 37/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.2913 - accuracy: 0.9206 - val_loss: 0.2665 - val_accuracy: 0.9300\n",
      "Epoch 38/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.2902 - accuracy: 0.9220 - val_loss: 0.2740 - val_accuracy: 0.9100\n",
      "Epoch 39/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.2895 - accuracy: 0.9222 - val_loss: 0.2584 - val_accuracy: 0.9400\n",
      "Epoch 40/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.2761 - accuracy: 0.9262 - val_loss: 0.2527 - val_accuracy: 0.9500\n",
      "transition matrices A:\n",
      " [[[0.69449 0.13322 0.02067 ... 0.0141  0.02231 0.01676]\n",
      "  [0.75097 0.04247 0.16079 ... 0.00324 0.00979 0.00296]\n",
      "  [0.02341 0.1672  0.44709 ... 0.0166  0.0306  0.01665]\n",
      "  ...\n",
      "  [0.01092 0.00541 0.02273 ... 0.06761 0.6629  0.00595]\n",
      "  [0.01002 0.00534 0.01999 ... 0.08581 0.70375 0.11535]\n",
      "  [0.0189  0.00729 0.02869 ... 0.00684 0.69981 0.14693]]\n",
      "\n",
      " [[0.4363  0.30541 0.01164 ... 0.04066 0.03453 0.02255]\n",
      "  [0.3554  0.27285 0.17104 ... 0.02814 0.02486 0.01756]\n",
      "  [0.01972 0.30447 0.27602 ... 0.06238 0.0473  0.02675]\n",
      "  ...\n",
      "  [0.00018 0.00016 0.00018 ... 0.4684  0.00082 0.00005]\n",
      "  [0.00061 0.00031 0.00058 ... 0.90655 0.0596  0.00385]\n",
      "  [0.00205 0.00126 0.0019  ... 0.03365 0.90916 0.01773]]\n",
      "\n",
      " [[0.79175 0.07424 0.01477 ... 0.00837 0.01049 0.01411]\n",
      "  [0.20712 0.10351 0.52312 ... 0.00528 0.01393 0.02109]\n",
      "  [0.01373 0.15138 0.51403 ... 0.01441 0.01656 0.02668]\n",
      "  ...\n",
      "  [0.00556 0.0046  0.00863 ... 0.05408 0.05385 0.00576]\n",
      "  [0.00588 0.00459 0.07431 ... 0.06938 0.1425  0.62187]\n",
      "  [0.00516 0.00435 0.02347 ... 0.00377 0.11078 0.78297]]\n",
      "\n",
      " ...\n",
      "\n",
      " [[0.1374  0.46631 0.03708 ... 0.00823 0.00831 0.01126]\n",
      "  [0.09956 0.24389 0.47483 ... 0.00551 0.00533 0.00608]\n",
      "  [0.0021  0.04622 0.82323 ... 0.00215 0.00207 0.00222]\n",
      "  ...\n",
      "  [0.00724 0.01582 0.0179  ... 0.11338 0.10895 0.02991]\n",
      "  [0.00646 0.01382 0.01562 ... 0.09931 0.09899 0.63824]\n",
      "  [0.00374 0.01259 0.01505 ... 0.00484 0.09584 0.75941]]\n",
      "\n",
      " [[0.07216 0.84455 0.00436 ... 0.00162 0.00951 0.0081 ]\n",
      "  [0.08919 0.73374 0.08067 ... 0.00338 0.01094 0.00955]\n",
      "  [0.004   0.75818 0.08385 ... 0.00233 0.00933 0.00813]\n",
      "  ...\n",
      "  [0.00302 0.00389 0.00324 ... 0.05216 0.12448 0.00462]\n",
      "  [0.00759 0.00943 0.00808 ... 0.16255 0.34269 0.27009]\n",
      "  [0.00725 0.00892 0.00791 ... 0.00739 0.35499 0.20644]]\n",
      "\n",
      " [[0.43852 0.27669 0.01065 ... 0.00877 0.00851 0.01219]\n",
      "  [0.33374 0.22738 0.17364 ... 0.00783 0.00822 0.01177]\n",
      "  [0.01012 0.15335 0.08703 ... 0.00385 0.00488 0.00881]\n",
      "  ...\n",
      "  [0.00886 0.00733 0.00601 ... 0.02846 0.0671  0.00425]\n",
      "  [0.00679 0.00621 0.00525 ... 0.02602 0.04724 0.05752]\n",
      "  [0.00667 0.00737 0.00595 ... 0.00168 0.03926 0.05567]]]\n",
      "j= 0 \tt= 50 \tr= 0 \tresults= [0.2697819173336029, 0.9350000023841858]\n",
      "lr=0.001\n",
      "Epoch 1/40\n",
      "313/313 [==============================] - 13s 39ms/step - loss: 0.6914 - accuracy: 0.5179 - val_loss: 0.6745 - val_accuracy: 0.5500\n",
      "Epoch 2/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.6616 - accuracy: 0.6564 - val_loss: 0.6208 - val_accuracy: 0.7200\n",
      "Epoch 3/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.6074 - accuracy: 0.7124 - val_loss: 0.5843 - val_accuracy: 0.7300\n",
      "Epoch 4/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.5833 - accuracy: 0.7199 - val_loss: 0.5567 - val_accuracy: 0.7200\n",
      "Epoch 5/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.5700 - accuracy: 0.7215 - val_loss: 0.5380 - val_accuracy: 0.7300\n",
      "Epoch 6/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.5472 - accuracy: 0.7423 - val_loss: 0.5229 - val_accuracy: 0.7500\n",
      "Epoch 7/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.5407 - accuracy: 0.7410 - val_loss: 0.5080 - val_accuracy: 0.7600\n",
      "Epoch 8/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.5219 - accuracy: 0.7545 - val_loss: 0.4973 - val_accuracy: 0.7600\n",
      "Epoch 9/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.5143 - accuracy: 0.7640 - val_loss: 0.4849 - val_accuracy: 0.8000\n",
      "Epoch 10/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.4932 - accuracy: 0.7832 - val_loss: 0.4732 - val_accuracy: 0.8000\n",
      "Epoch 11/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.4825 - accuracy: 0.7968 - val_loss: 0.4622 - val_accuracy: 0.8200\n",
      "Epoch 12/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.4760 - accuracy: 0.8130 - val_loss: 0.4494 - val_accuracy: 0.8300\n",
      "Epoch 13/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.4575 - accuracy: 0.8210 - val_loss: 0.4378 - val_accuracy: 0.8500\n",
      "Epoch 14/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.4440 - accuracy: 0.8292 - val_loss: 0.4292 - val_accuracy: 0.8600\n",
      "Epoch 15/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.4361 - accuracy: 0.8389 - val_loss: 0.4178 - val_accuracy: 0.9000\n",
      "Epoch 16/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.4261 - accuracy: 0.8472 - val_loss: 0.4108 - val_accuracy: 0.8900\n",
      "Epoch 17/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.4200 - accuracy: 0.8507 - val_loss: 0.3974 - val_accuracy: 0.9100\n",
      "Epoch 18/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.4060 - accuracy: 0.8547 - val_loss: 0.3887 - val_accuracy: 0.9200\n",
      "Epoch 19/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.4006 - accuracy: 0.8561 - val_loss: 0.3823 - val_accuracy: 0.9200\n",
      "Epoch 20/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3912 - accuracy: 0.8678 - val_loss: 0.3760 - val_accuracy: 0.9100\n",
      "Epoch 21/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3933 - accuracy: 0.8641 - val_loss: 0.3694 - val_accuracy: 0.9200\n",
      "Epoch 22/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3778 - accuracy: 0.8778 - val_loss: 0.3590 - val_accuracy: 0.9200\n",
      "Epoch 23/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3733 - accuracy: 0.8752 - val_loss: 0.3556 - val_accuracy: 0.9100\n",
      "Epoch 24/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3685 - accuracy: 0.8764 - val_loss: 0.3457 - val_accuracy: 0.9300\n",
      "Epoch 25/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3510 - accuracy: 0.8897 - val_loss: 0.3404 - val_accuracy: 0.9200\n",
      "Epoch 26/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3493 - accuracy: 0.8929 - val_loss: 0.3335 - val_accuracy: 0.9400\n",
      "Epoch 27/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3560 - accuracy: 0.8829 - val_loss: 0.3278 - val_accuracy: 0.9400\n",
      "Epoch 28/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3406 - accuracy: 0.8950 - val_loss: 0.3228 - val_accuracy: 0.9400\n",
      "Epoch 29/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3361 - accuracy: 0.8920 - val_loss: 0.3224 - val_accuracy: 0.9300\n",
      "Epoch 30/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3360 - accuracy: 0.8998 - val_loss: 0.3122 - val_accuracy: 0.9600\n",
      "Epoch 31/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3271 - accuracy: 0.9051 - val_loss: 0.3076 - val_accuracy: 0.9600\n",
      "Epoch 32/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3319 - accuracy: 0.8987 - val_loss: 0.3032 - val_accuracy: 0.9700\n",
      "Epoch 33/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3243 - accuracy: 0.9051 - val_loss: 0.2977 - val_accuracy: 0.9700\n",
      "Epoch 34/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.3197 - accuracy: 0.9038 - val_loss: 0.3007 - val_accuracy: 0.9500\n",
      "Epoch 35/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.3056 - accuracy: 0.9193 - val_loss: 0.2910 - val_accuracy: 0.9500\n",
      "Epoch 36/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3028 - accuracy: 0.9187 - val_loss: 0.2879 - val_accuracy: 0.9500\n",
      "Epoch 37/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3058 - accuracy: 0.9120 - val_loss: 0.2844 - val_accuracy: 0.9600\n",
      "Epoch 38/40\n",
      "313/313 [==============================] - 12s 40ms/step - loss: 0.2911 - accuracy: 0.9234 - val_loss: 0.2785 - val_accuracy: 0.9700\n",
      "Epoch 39/40\n",
      "313/313 [==============================] - 11s 34ms/step - loss: 0.2894 - accuracy: 0.9194 - val_loss: 0.2743 - val_accuracy: 0.9600\n",
      "Epoch 40/40\n",
      "313/313 [==============================] - 11s 34ms/step - loss: 0.2950 - accuracy: 0.9209 - val_loss: 0.2732 - val_accuracy: 0.9600\n",
      "transition matrices A:\n",
      " [[[0.24778 0.27487 0.09266 ... 0.02375 0.02356 0.02494]\n",
      "  [0.11948 0.09782 0.62033 ... 0.01142 0.01134 0.01193]\n",
      "  [0.00963 0.17122 0.26763 ... 0.01944 0.01928 0.02037]\n",
      "  ...\n",
      "  [0.00679 0.00667 0.00684 ... 0.34556 0.35442 0.01967]\n",
      "  [0.00539 0.00531 0.00542 ... 0.28076 0.29246 0.35816]\n",
      "  [0.00742 0.00726 0.00738 ... 0.02022 0.41629 0.47016]]\n",
      "\n",
      " [[0.2591  0.21788 0.1648  ... 0.05182 0.01346 0.02634]\n",
      "  [0.03621 0.02795 0.88554 ... 0.00755 0.00192 0.00304]\n",
      "  [0.00466 0.06797 0.68785 ... 0.0165  0.00546 0.00705]\n",
      "  ...\n",
      "  [0.00369 0.00373 0.00486 ... 0.66268 0.08406 0.00295]\n",
      "  [0.00425 0.00435 0.00977 ... 0.72344 0.08733 0.08193]\n",
      "  [0.01053 0.01111 0.0256  ... 0.0584  0.22436 0.43519]]\n",
      "\n",
      " [[0.03468 0.03981 0.02975 ... 0.00377 0.00733 0.00818]\n",
      "  [0.03721 0.02773 0.12736 ... 0.00204 0.00368 0.00405]\n",
      "  [0.00061 0.01072 0.01798 ... 0.00066 0.00107 0.00117]\n",
      "  ...\n",
      "  [0.02681 0.02773 0.02481 ... 0.16366 0.32262 0.01932]\n",
      "  [0.01512 0.01551 0.01537 ... 0.07688 0.30835 0.42452]\n",
      "  [0.0147  0.01546 0.01609 ... 0.00441 0.31391 0.48803]]\n",
      "\n",
      " ...\n",
      "\n",
      " [[0.86084 0.13256 0.00023 ... 0.00089 0.00018 0.00077]\n",
      "  [0.70797 0.28154 0.00729 ... 0.0002  0.00011 0.00148]\n",
      "  [0.00481 0.12846 0.8231  ... 0.00006 0.00006 0.01061]\n",
      "  ...\n",
      "  [0.01409 0.01318 0.03054 ... 0.07331 0.51678 0.03389]\n",
      "  [0.00308 0.00274 0.03694 ... 0.02294 0.02562 0.77474]\n",
      "  [0.01214 0.01333 0.21645 ... 0.00011 0.00126 0.74732]]\n",
      "\n",
      " [[0.26339 0.32836 0.03177 ... 0.05819 0.0121  0.0139 ]\n",
      "  [0.16475 0.18158 0.39224 ... 0.05105 0.00806 0.00719]\n",
      "  [0.00712 0.14984 0.30682 ... 0.01081 0.00706 0.00532]\n",
      "  ...\n",
      "  [0.00658 0.00698 0.01457 ... 0.54845 0.13502 0.00466]\n",
      "  [0.00303 0.00324 0.00764 ... 0.82798 0.0641  0.03845]\n",
      "  [0.01385 0.01574 0.04206 ... 0.15059 0.28485 0.21937]]\n",
      "\n",
      " [[0.12456 0.64669 0.01431 ... 0.00197 0.00281 0.00361]\n",
      "  [0.00624 0.88408 0.1066  ... 0.00029 0.00051 0.00063]\n",
      "  [0.00096 0.85978 0.12873 ... 0.00046 0.00077 0.0009 ]\n",
      "  ...\n",
      "  [0.01569 0.01015 0.00812 ... 0.0808  0.09656 0.00878]\n",
      "  [0.00957 0.00868 0.00765 ... 0.14379 0.26357 0.40751]\n",
      "  [0.0104  0.00919 0.00815 ... 0.00786 0.26232 0.53351]]]\n",
      "j= 0 \tt= 50 \tr= 1 \tresults= [0.2779819369316101, 0.9279999732971191]\n",
      "lr=0.001\n",
      "Epoch 1/40\n",
      "313/313 [==============================] - 12s 36ms/step - loss: 0.6921 - accuracy: 0.5010 - val_loss: 0.6771 - val_accuracy: 0.6100\n",
      "Epoch 2/40\n",
      "313/313 [==============================] - 11s 35ms/step - loss: 0.6627 - accuracy: 0.6838 - val_loss: 0.6402 - val_accuracy: 0.6800\n",
      "Epoch 3/40\n",
      "313/313 [==============================] - 11s 35ms/step - loss: 0.6160 - accuracy: 0.7067 - val_loss: 0.6167 - val_accuracy: 0.6700\n",
      "Epoch 4/40\n",
      "313/313 [==============================] - 11s 35ms/step - loss: 0.5914 - accuracy: 0.7215 - val_loss: 0.6013 - val_accuracy: 0.6900\n",
      "Epoch 5/40\n",
      "313/313 [==============================] - 11s 35ms/step - loss: 0.5730 - accuracy: 0.7200 - val_loss: 0.5866 - val_accuracy: 0.6800\n",
      "Epoch 6/40\n",
      "313/313 [==============================] - 11s 35ms/step - loss: 0.5581 - accuracy: 0.7374 - val_loss: 0.5730 - val_accuracy: 0.6900\n",
      "Epoch 7/40\n",
      "313/313 [==============================] - 11s 35ms/step - loss: 0.5426 - accuracy: 0.7544 - val_loss: 0.5577 - val_accuracy: 0.7000\n",
      "Epoch 8/40\n",
      "313/313 [==============================] - 11s 35ms/step - loss: 0.5256 - accuracy: 0.7692 - val_loss: 0.5426 - val_accuracy: 0.7400\n",
      "Epoch 9/40\n",
      "313/313 [==============================] - 11s 35ms/step - loss: 0.5117 - accuracy: 0.7817 - val_loss: 0.5263 - val_accuracy: 0.7500\n",
      "Epoch 10/40\n",
      "313/313 [==============================] - 11s 35ms/step - loss: 0.4973 - accuracy: 0.7976 - val_loss: 0.5092 - val_accuracy: 0.7700\n",
      "Epoch 11/40\n",
      "313/313 [==============================] - 11s 35ms/step - loss: 0.4865 - accuracy: 0.8147 - val_loss: 0.4925 - val_accuracy: 0.7800\n",
      "Epoch 12/40\n",
      "313/313 [==============================] - 11s 35ms/step - loss: 0.4686 - accuracy: 0.8258 - val_loss: 0.4775 - val_accuracy: 0.8000\n",
      "Epoch 13/40\n",
      "313/313 [==============================] - 11s 35ms/step - loss: 0.4575 - accuracy: 0.8306 - val_loss: 0.4629 - val_accuracy: 0.8000\n",
      "Epoch 14/40\n",
      "313/313 [==============================] - 12s 39ms/step - loss: 0.4436 - accuracy: 0.8400 - val_loss: 0.4497 - val_accuracy: 0.8100\n",
      "Epoch 15/40\n",
      "313/313 [==============================] - 13s 40ms/step - loss: 0.4352 - accuracy: 0.8466 - val_loss: 0.4369 - val_accuracy: 0.8200\n",
      "Epoch 16/40\n",
      "313/313 [==============================] - 13s 40ms/step - loss: 0.4195 - accuracy: 0.8604 - val_loss: 0.4272 - val_accuracy: 0.8300\n",
      "Epoch 17/40\n",
      "313/313 [==============================] - 13s 41ms/step - loss: 0.4093 - accuracy: 0.8643 - val_loss: 0.4168 - val_accuracy: 0.8400\n",
      "Epoch 18/40\n",
      "313/313 [==============================] - 15s 47ms/step - loss: 0.4003 - accuracy: 0.8591 - val_loss: 0.4078 - val_accuracy: 0.8400\n",
      "Epoch 19/40\n",
      "313/313 [==============================] - 13s 41ms/step - loss: 0.3906 - accuracy: 0.8785 - val_loss: 0.3995 - val_accuracy: 0.8300\n",
      "Epoch 20/40\n",
      "313/313 [==============================] - 13s 40ms/step - loss: 0.3890 - accuracy: 0.8690 - val_loss: 0.3912 - val_accuracy: 0.8300\n",
      "Epoch 21/40\n",
      "313/313 [==============================] - 12s 39ms/step - loss: 0.3822 - accuracy: 0.8793 - val_loss: 0.3859 - val_accuracy: 0.8600\n",
      "Epoch 22/40\n",
      "313/313 [==============================] - 13s 40ms/step - loss: 0.3687 - accuracy: 0.8940 - val_loss: 0.3788 - val_accuracy: 0.8600\n",
      "Epoch 23/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.3686 - accuracy: 0.8818 - val_loss: 0.3718 - val_accuracy: 0.8500\n",
      "Epoch 24/40\n",
      "313/313 [==============================] - 11s 36ms/step - loss: 0.3628 - accuracy: 0.8877 - val_loss: 0.3652 - val_accuracy: 0.8600\n",
      "Epoch 25/40\n",
      "313/313 [==============================] - 12s 39ms/step - loss: 0.3529 - accuracy: 0.8945 - val_loss: 0.3589 - val_accuracy: 0.8600\n",
      "Epoch 26/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.3488 - accuracy: 0.9057 - val_loss: 0.3527 - val_accuracy: 0.8800\n",
      "Epoch 27/40\n",
      "313/313 [==============================] - 11s 36ms/step - loss: 0.3409 - accuracy: 0.8996 - val_loss: 0.3475 - val_accuracy: 0.8700\n",
      "Epoch 28/40\n",
      "313/313 [==============================] - 11s 36ms/step - loss: 0.3384 - accuracy: 0.9100 - val_loss: 0.3416 - val_accuracy: 0.8900\n",
      "Epoch 29/40\n",
      "313/313 [==============================] - 12s 38ms/step - loss: 0.3251 - accuracy: 0.9013 - val_loss: 0.3380 - val_accuracy: 0.8800\n",
      "Epoch 30/40\n",
      "313/313 [==============================] - 12s 37ms/step - loss: 0.3231 - accuracy: 0.9090 - val_loss: 0.3330 - val_accuracy: 0.8800\n",
      "Epoch 31/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.3189 - accuracy: 0.9115 - val_loss: 0.3270 - val_accuracy: 0.8900\n",
      "Epoch 32/40\n",
      "313/313 [==============================] - 11s 36ms/step - loss: 0.3106 - accuracy: 0.9188 - val_loss: 0.3221 - val_accuracy: 0.9000\n",
      "Epoch 33/40\n",
      "313/313 [==============================] - 11s 37ms/step - loss: 0.3146 - accuracy: 0.9170 - val_loss: 0.3207 - val_accuracy: 0.8900\n",
      "Epoch 34/40\n",
      "313/313 [==============================] - 12s 39ms/step - loss: 0.3130 - accuracy: 0.9104 - val_loss: 0.3180 - val_accuracy: 0.9000\n",
      "Epoch 35/40\n",
      "313/313 [==============================] - 13s 42ms/step - loss: 0.3017 - accuracy: 0.9233 - val_loss: 0.3103 - val_accuracy: 0.8800\n",
      "Epoch 36/40\n",
      "313/313 [==============================] - 12s 39ms/step - loss: 0.2952 - accuracy: 0.9217 - val_loss: 0.3082 - val_accuracy: 0.9100\n",
      "Epoch 37/40\n",
      "313/313 [==============================] - 12s 39ms/step - loss: 0.2958 - accuracy: 0.9273 - val_loss: 0.3016 - val_accuracy: 0.9000\n",
      "Epoch 38/40\n",
      "313/313 [==============================] - 12s 39ms/step - loss: 0.2918 - accuracy: 0.9130 - val_loss: 0.2979 - val_accuracy: 0.9100\n",
      "Epoch 39/40\n",
      "313/313 [==============================] - 12s 39ms/step - loss: 0.2840 - accuracy: 0.9254 - val_loss: 0.2937 - val_accuracy: 0.9000\n",
      "Epoch 40/40\n",
      "313/313 [==============================] - 12s 39ms/step - loss: 0.2780 - accuracy: 0.9324 - val_loss: 0.2933 - val_accuracy: 0.9100\n",
      "transition matrices A:\n",
      " [[[0.32455 0.58521 0.03335 ... 0.00505 0.00355 0.00389]\n",
      "  [0.17488 0.36208 0.42814 ... 0.0032  0.00215 0.00222]\n",
      "  [0.00825 0.41482 0.50036 ... 0.00288 0.00177 0.00195]\n",
      "  ...\n",
      "  [0.02133 0.02441 0.0247  ... 0.14839 0.39658 0.0261 ]\n",
      "  [0.01855 0.02184 0.02221 ... 0.13268 0.16517 0.4778 ]\n",
      "  [0.02669 0.03648 0.03693 ... 0.00829 0.09673 0.66226]]\n",
      "\n",
      " [[0.09044 0.03033 0.0355  ... 0.21395 0.11495 0.16427]\n",
      "  [0.0718  0.03649 0.29829 ... 0.14713 0.09825 0.14118]\n",
      "  [0.00184 0.02687 0.03529 ... 0.0169  0.14511 0.16135]\n",
      "  ...\n",
      "  [0.00024 0.00011 0.00048 ... 0.18057 0.71931 0.05895]\n",
      "  [0.00006 0.00005 0.00008 ... 0.00293 0.17087 0.81304]\n",
      "  [0.00008 0.00007 0.00012 ... 0.00018 0.15978 0.69886]]\n",
      "\n",
      " [[0.02267 0.03733 0.01627 ... 0.10563 0.17722 0.08047]\n",
      "  [0.03977 0.03801 0.22389 ... 0.07072 0.13065 0.08668]\n",
      "  [0.00113 0.02274 0.01211 ... 0.0466  0.10294 0.00247]\n",
      "  ...\n",
      "  [0.00014 0.00015 0.00009 ... 0.04444 0.00315 0.00013]\n",
      "  [0.00058 0.00062 0.0003  ... 0.76418 0.0181  0.01469]\n",
      "  [0.00073 0.00101 0.00054 ... 0.03292 0.83343 0.02159]]\n",
      "\n",
      " ...\n",
      "\n",
      " [[0.80362 0.08249 0.02018 ... 0.00496 0.02055 0.0052 ]\n",
      "  [0.17138 0.06777 0.63875 ... 0.00449 0.02949 0.0053 ]\n",
      "  [0.00721 0.06239 0.73814 ... 0.00475 0.02205 0.00514]\n",
      "  ...\n",
      "  [0.00418 0.00266 0.0112  ... 0.04369 0.24213 0.00314]\n",
      "  [0.01441 0.00712 0.02894 ... 0.14233 0.53181 0.16906]\n",
      "  [0.01446 0.00849 0.03314 ... 0.00822 0.63904 0.16782]]\n",
      "\n",
      " [[0.02523 0.87633 0.06709 ... 0.00031 0.00059 0.00036]\n",
      "  [0.00413 0.02701 0.9596  ... 0.00007 0.00013 0.00007]\n",
      "  [0.00014 0.00925 0.58789 ... 0.00003 0.00007 0.00003]\n",
      "  ...\n",
      "  [0.08484 0.23718 0.3632  ... 0.02329 0.03408 0.00109]\n",
      "  [0.08908 0.22804 0.36131 ... 0.02182 0.04293 0.01791]\n",
      "  [0.09426 0.20681 0.32065 ... 0.0018  0.05218 0.02756]]\n",
      "\n",
      " [[0.05332 0.02016 0.00573 ... 0.01168 0.00839 0.00997]\n",
      "  [0.03755 0.01783 0.0629  ... 0.00648 0.0047  0.00513]\n",
      "  [0.00181 0.02576 0.03831 ... 0.00499 0.00401 0.00424]\n",
      "  ...\n",
      "  [0.00683 0.00708 0.00924 ... 0.2127  0.11335 0.00462]\n",
      "  [0.00788 0.00867 0.0109  ... 0.55423 0.144   0.12409]\n",
      "  [0.01312 0.01667 0.01794 ... 0.031   0.35139 0.3293 ]]]\n",
      "j= 0 \tt= 50 \tr= 2 \tresults= [0.2907837927341461, 0.9340000152587891]\n",
      "\n",
      " [[0.935      0.92799997 0.93400002]]\n",
      "time [s]: 1405.5908632278442\n",
      "SimpleRNN dir= False \tlr=0.001\n",
      "j= 0 \tt= 50 \tr= 0 \tresults= [0.6941809058189392, 0.49900001287460327]\n",
      "lr=0.001\n",
      "j= 0 \tt= 50 \tr= 1 \tresults= [0.6936482191085815, 0.5120000243186951]\n",
      "lr=0.001\n",
      "j= 0 \tt= 50 \tr= 2 \tresults= [0.6956847310066223, 0.4699999988079071]\n",
      "\n",
      " [[0.49900001 0.51200002 0.47      ]]\n",
      "time [s]: 307.9049725532532\n"
     ]
    }
   ],
   "source": [
    "num_reps = 3 # 30\n",
    "num_epochs = 40 # 100\n",
    "modelnames = [\"HMM\", \"SimpleRNN\"] #, \"SimpleRNN\", \"LSTM\", \"GRU\"] #, \"ad hoc\"\n",
    "directions = [False] # , True] # bidirectional?\n",
    "test_accs = { (name, direction) : np.empty((len(ts), num_reps)) for name in modelnames for direction in directions}\n",
    "\n",
    "for modelname in modelnames:\n",
    "    start_time = time()\n",
    "    for direction in directions:\n",
    "        print (modelname, \"dir=\", direction, \"\\t\", end = \"\")\n",
    "        for j in range(len(ts)):\n",
    "            t = ts[j]\n",
    "            accs = test_accs[(modelname, direction)]\n",
    "            for r in range(num_reps):\n",
    "                accuracy = 0.0\n",
    "                x_test, y_test = simulate_data(1000, t)\n",
    "                if modelname == \"ad hoc\":\n",
    "                    accuracy = adhoc_acc(x_test, y_test)\n",
    "                else:\n",
    "                    model = get_model(bidirectional = direction, seqModelType = modelname, RNNunits = 32)\n",
    "                    x_train, y_train = simulate_data(m, t)\n",
    "                    x_val, y_val = simulate_data(100, t)\n",
    "                    # model.get_layer(index=0).C.print_pars()\n",
    "                    model.fit(x_train, y_train,\n",
    "                          validation_data = (x_val, y_val), batch_size = 16, epochs = num_epochs,\n",
    "                              verbose=1 if modelname==\"HMM\" else 0)\n",
    "                    if modelname == \"HMM\":\n",
    "                        # print transition and emission matrices\n",
    "                        # pass\n",
    "                        M = model.get_layer(index=0)\n",
    "                        M.C.print_pars()\n",
    "                        #HMML = M(x_test[0:4])\n",
    "                        #print(\"HMML=\", HMML)\n",
    "                        #print(\"probs=\", model(x_test[0:4]))\n",
    "                    results = model.evaluate(x_test, y_test, batch_size = 16, verbose=0)\n",
    "                    print (\"j=\", j, \"\\tt=\", t, \"\\tr=\", r, \"\\tresults=\", results)\n",
    "                    accuracy = results[1]\n",
    "                accs[j, r] = accuracy\n",
    "        print (\"\\n\", accs)\n",
    "    end_time = time()\n",
    "    seconds_elapsed = end_time - start_time\n",
    "    print (\"time [s]:\", seconds_elapsed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plot Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy was averaged over 3 repetitions\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0gAAAFNCAYAAAA6tU9BAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3debhdZX33//eHECTIDEELCUM1DGGKErE+TrGIgMqgRQWtCooUBYderYITYq0VHx5rVdD8+FGM+igRFREQBcQqWgeGGhmlpIjkAFZAQUAQA9/nj7VCdw5n2Bn23sk579d17euctda91/6uew9nf86611qpKiRJkiRJsM6gC5AkSZKkNYUBSZIkSZJaBiRJkiRJahmQJEmSJKllQJIkSZKklgFJkiRJklqTNiAl+cckdyX59aBrWRMkeXaSm5Lcn+SQAdZxS5IXjrJsXpKhjunrkswbZ33bJ6kk6/a6vlVc72uSXDzG8u8lOaqbtv2sazU+zoIk/9jrx1EjyRFJfthlW58bSdKkstYEpPaL6YPtF/j/TvLZJBuu5LpmAn8HzK6qJ6/eStda/wCcWlUbVtW5q2OFqzucDFdVu1bV93qx7n6rqi9W1YtWd9sVMdLz1avHkiRJWlOtNQGpdWBVbQg8HXgG8L4VXUH75W874O6q+s1K3n8i2g64btBFaNVM4NenJElSX6xtAQmAqroN+BawG0CSTZL8a5I7ktzWDp+b0i47Ism/J/l4kt8C3wMuAbZu90YtaNsd1A7ZuqcdzrTLssdr914dn+Rq4IEk67bz3pnk6iQPtI//pCTfSnJfku8k2axjHV9J8usk9ya5LMmuHcsWJDktyTfb+/40yVM6lu+a5JIkv233nr2nnb9OkhOS/FeSu5OcnWTz0fotyZuSLG7Xc16Srdv5/wX8OXB+2ydPGOG+Wyf5WpI7k/wyyds6lu2d5Mokv2/r++d20WXtz3va9T4ryVOSfLet964kX0yy6bCHe0aS65P8rt1TuP4o2/PYcLcxaljmNUlubR/zvR3rGLMPk7w2ya/aZe9lDOkYBtdOLzeMqd07c0yaoYy/a5/zjNJ23yS/aF8vpwIZZ73HJrkJuKmdt3PHa+bGJK/saD8tycfa7bo3yQ+TTGPk52v4Y/2vJFe097siyf8atv0fSvN+uy/JxUm27Fg+6ntgnH59apLvt/e7K8mXO5aNtZ1btK/z3ye5vK3th+2yx+0tG+H5e0OSG9rn6qIk23XzXLbL39Te9772tfz0dv6o76MRtntBkk+n+Uy5v+3XJyf5l/Yxf5HkaR3td2m34Z40n2UHjdYXwFOGPdao/Tis3ZZJLmgf47dJfpBkrfw7IknSaNbKP2xphsi9GPhZO+tzwFLgqcDTgBcBR3Xc5ZnAzcBWwL7AAcDt7XCyI5LsCJwFvAOYDlxIExbW61jH4cBLgE2ramk776/a9e0IHEgT2t4DbEnTt51ffr4FzGpr+A/gi8M263Dgg8BmwGLgw+22bgR8B/g2sHW7jZe293kbcAjw/HbZ74DTRumzvwQ+ArwS+DPgV8BCgKp6CnAr7R66qvrjsPuuA5wP/BzYBtgHeEeS/domnwA+UVUb03zxOrud/7z256bten9M80X/I229uwAzgZOGlfsaYL92XTvS3Z7C0WpY5jnATm3tJ+Z/AvCofZhkNvAZ4LXtsi2AGV3UMpaX0uz93JPmudhveIM2VHyNZru3BP4LePY46z2E5nU+O8kTaf4J8CWa19vhwKc7Asn/AfYC/hewOfAu4FFGfr4669oc+CbwSZq++Gfgm0m26Gj2auDI9nHXA/6+Y9l474HRfAi4mOa9MQP4VFvPeNt5GvAQzev9De2tK2mOw3sP8HKaz4Qf0HxGdBrxuUzyCprX9OuAjYGDgLu7eB+N5JX8z+vgj8CPafpuS+CrNM8BSaa267647Yu3Al9MstN4fdFFP3b6O2Co7ZMntX1UY9QvSdLap6rWihtwC3A/cA/Nl/tPA9No/kj/EZjW0fZw4N/a348Abh22rnnAUMf0+4GzO6bXAW4D5nU89htGqOc1HdNfAz7TMf1W4NxRtmVTmi8Vm7TTC4AzOpa/GPhFx7b8bJT13ADs0zH9Z8CfgHVHaPuvwP/umN6wbbt9x/a8cJTHeeYIffhu4LPt75fRhLsth7XZvt3Ox9XT0eaQzu1r6zhmWF/81yjP22M1d1HDjI55lwOHjdeHwInAwo5lTwQeHqOfvgcc1TF9BPDDjukCntMxfTZwwvC2NF+sf9LRLjRfSo8aY71/2TH9KuAHw2r7/4AP0Ly2HwT2HKH+xz1fw+p6LXD5sPv8GDiiY/vf17HsLcC3V+A98I+jtP08cHrnc9jFdk5pn8edO5b9U8e2jLStjz1/NGHujR3L1gH+AGzXxXN5EfD2FX0fjdB+AfD/d0y/FbihY3p34J729+cCvwbW6Vh+Fk1QG68vRu3H4c8NzbGK3wCeOtp72ps3b968eVvbb2vbHqRDqmrTqtquqt5SVQ/SHDszFbijHfZxD80f96067rdknPVuTRO6AKiqR9v7bDPOOv674/cHR5jeECDJlCQnpxnG9XuaL/bQ/Bd4mc6z6f1h2X1p9rD81yh1bwd8vWO7bwAeoQmNww3fxvuBu1l+G0ezHc2QxHs6Hus9HY/zRpo9Pb9oh129dLQVJdkqycI0QyF/D/xflu8HWL6vf9XWPp7xahitf8fqw607a6mqB2j6bFWMVken4Y9bjP8a7ly+HfDMYc/Xa4An0/T1+oz+mhrLcq+h1q9Y/jU04vZ1+R4YzbtoQuLl7dCxZXs/xtrO6TQhd/hrqVvbAZ/oWO9v2xrG3VZGf8+O9z4aSVefMbSvmfaza5llz814fTFWPw53Cs0e7ouT3JzkhDFqlyRprTQRDuheQrMHacv6n6Fvw403BOR2mv/GAtAeSzCTZi9St+sYy6uBg4EX0nwx3IRmKFfGuM8yS2j2Io227A1V9e9drOd2mi9CwGPDarZg+W0cq4ZfVtWskRZW1U3A4e0QopcDX22HXY3UZx9p5+9RVXe3Q5lOHdZmZsfv27a1j2mMGsYzah8muYNmGOCy6Q1o+mw0DwAbdEyv7BkS76CjDzpej2Pp7OslwPerat/hjdr+eYhmGOLPx1jHSJZ7DbW2pRn+OZ6Vfg9U1a+BNwEkeQ7wnSSXMfZ2TqEZdjsT+EVHrcs80P7cAPh9+3vn87UE+HBVdTsMsNMShh3j0zF/1PfRKrodmJlknY6QtC3wn8CdjN0Xo/bjcFV1H80wu79rh+D9W5IrqurSce4qSdJaY23bg/Q4VXUHzbj7jyXZOM1B909J8vwVWM3ZwEuS7NOO5f87mtD1o9VU5kbt+u6m+UL2Tytw3wuAJyd5R5InJNkoyTPbZfOBD6c9eDzJ9CQHj7KeLwFHJpmT5iQM/wT8tKpu6aKGy4HfpzlRxbR2b8BuSZ7RPu5fJ5nefjG7p73PIzRfzB6lOQHEMhvRDpVMsg3wzhEe79gkM9pjXt4DfHmENssZo4bxjNWHXwVemuQ57fFo/8DY75lFwMuTbJDkqTR7tVbGN4Fdk7w8zUkE3saKha0LgB3TnGBiant7RpJd2v45E/jnNCcMmJLmZAxPYOTnq9OF7XpfneZEJa8CZrePN56Vfg8keUWSZcd+/Y4myD0yznY+ApwDnNQ+H7OB1y9bZ1XdSfPPgb9u++ANLB9q5gPvXnYcTpoTwbyiy5LPAP4+yV5pPLV9fY35PlpFP6UJfe9q+2EezXGRC8frC8box+EPkuSl7faEJlg+QnfvM0mS1hprfUBqvY7mgPDrab5AfZXmWJKuVNWNwF/THPx9F80XiwOr6uHVVN/naYa03NbW+JMVqO0+mhNBHEgzpOcm4AXt4k8A59EMd7mvXe8zR1nPpTTHWn2NZg/FU4DDuqzhkfbx5wC/pOmjM2j2AgDsD1yX5P62psOq6qGq+gPNySb+vR268xc0xwk9HbiXJgicM8JDfokm9N7c3rq5SOWINXRxv1H7sKquA45t67mD5rU1NMp6AD5Oc4zSf9OcOGRl9j5QVXcBrwBOpgkUs4Bu9hIuu/99NCcqOYxmz8KvgY8Cy85O+PfANcAVNEPHPkpz7MpIz1fneu+mOTHB37V1vQt4aVvveFb6PUBzIoSfts/teTTH9/yyi+08jmYI2q9pjqP57LD1vokmoN8N7ErHP0Sq6uvtuhamGRJ4Lc3JXcZVVV+h6ccvAfcB5wKbd/E+WmntZ9VBbY130Ryj+bqqWrbHaNS+6KIfO82iOWnM/TTHn326Jsi1yCRJWibN4Q2SNLElOYLmJAzPGXQtkiRpzTVR9iBJkiRJ0iozIEmSJqUkZyb5TZJrR1meJJ9Mc4Htq9Ne8FeSNLEZkCRNClW1wOF1GmYBzfGLozmA5rirWcDRNBeOliRNcAYkSdKkVFWX0ZyoZDQHA5+vxk+ATZN0fQIgSdLayYAkSdLItmH5C+wO0d3FtSVJa7G17kKxW265ZW2//faDLkOSJrWrrrrqrqqaPug6emykCxmPeOrXJEfTDMPjiU984l4777xzL+uSJI1jVf5OrXUBafvtt+fKK68cdBmSNKkl+dWga+iDIWBmx/QMmmtFPU5VnQ6cDjB37tzy75QkDdaq/J1yiJ0kSSM7D3hdeza7vwDurao7Bl2UJKm31ro9SJIkrQ5JzgLmAVsmGQI+AEwFqKr5wIXAi4HFwB+AIwdTqSSpnwxIkqRJqaoOH2d5Acf2qRxJ0hrCgCRprfOnP/2JoaEhHnrooUGXMuGtv/76zJgxg6lTpw66FEmS+sKAJGmtMzQ0xEYbbcT2229PMtKJxrQ6VBV33303Q0ND7LDDDoMuR5KkvvAkDZLWOg899BBbbLGF4ajHkrDFFlu4p06SNKkYkCStlQxH/WE/S5ImGwOS1Efn/uw2nn3yd9nhhG/y7JO/y7k/u23QJWklbbjhhstNL1iwgOOOOw6Ak046iSQsXrz4seUf//jHSfLYddy23357nvvc5y63jjlz5rDbbrv1uHJJkjQWA5LUJ+f+7Dbefc413HbPgxRw2z0P8u5zrjEkTVC77747CxcufGz6q1/9KrNnz16uzX333ceSJUsAuOGGG/panyRJGpkBSeqTUy66kQf/9Mhy8x780yOcctGNA6po8hjEnrtDDjmEb3zjGwDcfPPNbLLJJkyfPn25Nq985Sv58pe/DMBZZ53F4YePedZpSZLUBwYkqU9uv+fBFZqv1aNXe+4efPBB5syZ89jtxBNPXG75xhtvzMyZM7n22ms566yzeNWrXvW4dRx66KGcc845AJx//vkceOCBq1STJEladZ7mW+qTrTedxm0jhKGtN502gGomjg+efx3X3/77UZf/7NZ7ePiRR5eb9+CfHuFdX72asy6/dcT7zN56Yz5w4K5jPu60adNYtGjRY9MLFix47PiiZQ477DAWLlzIRRddxKWXXspnP/vZ5ZZvvvnmbLbZZixcuJBddtmFDTbYYMzHlCRJveceJKlP3rnfTkybOmW5edOmTuGd++00oIomh+HhaLz5q9OBBx7IF77wBbbddls23njjEdu86lWv4thjj3V4nSRJawj3IEl9csjTtgGaY5Fuv+dBtt50Gu/cb6fH5mvljLen59knf3fEPXfbbDqNL//Ns3pVFtDsZfroRz/KjjvuOGqbl73sZdxxxx3st99+3H777T2tR5Ikjc+AJPXRIU/bxkDUZ+/cbyfefc41y50go5977g477LAxl2+00UYcf/zxfalFkiSNL1U16BpWyNy5c2v4OH9Jk8sNN9zALrvs0nX7c392m3vuVsFI/Z3kqqqaO6CS1mj+nZKkwVuVv1PuQZI04bnnTpIkdcuTNEiSJElSy4AkSZIkSS0DkiRJkiS1DEiSJEmS1DIgSZIkSVLLgCRJK+nDH/4wu+66K3vssQdz5szhpz/9KUcddRTXX3/9aln/hhtuOG6bKVOmMGfOHHbbbTcOPPBA7rnnHgBuueUWkvCpT33qsbbHHXccCxYsAOCII45gm2224Y9//CMAd911F9tvv/1qqVuSpLWZAUmSVsKPf/xjLrjgAv7jP/6Dq6++mu985zvMnDmTM844g9mzZ/etjmnTprFo0SKuvfZaNt98c0477bTHlm211VZ84hOf4OGHHx7xvlOmTOHMM8/sV6mSJK0VDEiSJr6rz4aP7wYnbdr8vPrsVV7lHXfcwZZbbskTnvAEALbccku23npr5s2bx7KLhG644YYcf/zx7LXXXrzwhS/k8ssvZ968efz5n/855513HgALFizg4IMPZv/992ennXbigx/84IiPd8opp/CMZzyDPfbYgw984AMjtnnWs57Fbbfd9tj09OnT2Wefffjc5z43Yvt3vOMdfPzjH2fp0qUr3Q+SJE00PQ1ISfZPcmOSxUlOGGH5Zkm+nuTqJJcn2a2X9UiahK4+G85/G9y7BKjm5/lvW+WQ9KIXvYglS5aw44478pa3vIXvf//7j2vzwAMPMG/ePK666io22mgj3ve+93HJJZfw9a9/nRNPPPGxdpdffjlf/OIXWbRoEV/5ylceC1jLXHzxxdx0001cfvnlLFq0iKuuuorLLrtsuTaPPPIIl156KQcddNBy80844QQ+9rGP8cgjjzyuvm233ZbnPOc5fOELX1iVrpAkaUJZt1crTjIFOA3YFxgCrkhyXlV1Ds5/D7Coql6WZOe2/T69qknSBPStE+DX14y+fOgKeOSPy8/704PwjePgqpH3rPDk3eGAk8d82A033JCrrrqKH/zgB/zbv/0br3rVqzj55OXvs95667H//vsDsPvuu/OEJzyBqVOnsvvuu3PLLbc81m7fffdliy22AODlL385P/zhD5k7d+5jyy+++GIuvvhinva0pwFw//33c9NNN/G85z2PBx98kDlz5nDLLbew1157se+++y5Xww477MDee+/Nl770pRG34z3veQ8HHXQQL3nJS8bcXkmSJoueBSRgb2BxVd0MkGQhcDDQGZBmAx8BqKpfJNk+yZOq6r97WJekyWR4OBpv/gqYMmUK8+bNY968eey+++6PG8o2depUkgCwzjrrPDYcb5111lluWNuyNqNNVxXvfve7+Zu/+ZvH1bDsGKR7772Xl770pZx22mm87W1vW67Ne97zHg499FCe97znPe7+T33qU5kzZw5nn73qww4lSZoIehmQtgGWdEwPAc8c1ubnwMuBHybZG9gOmAEYkCR1Z5w9PXx8t3Z43TCbzIQjv7nSD3vjjTeyzjrrMGvWLAAWLVrEdtttx7XXXrvC67rkkkv47W9/y7Rp0zj33HMfd+KE/fbbj/e///285jWvYcMNN+S2225j6tSpbLXVVv+zOZtswic/+UkOPvhg3vzmNy93/5133pnZs2dzwQUXsPfeez/u8d/73ve6B0mSpFYvj0HKCPNq2PTJwGZJFgFvBX4GPO5o4SRHJ7kyyZV33nnn6q9U0sS1z4kwddry86ZOa+avgvvvv5/Xv/71zJ49mz322IPrr7+ek046aaXW9ZznPIfXvva1zJkzh7/6q79abngdNMc7vfrVr+ZZz3oWu+++O4ceeij33Xff49bztKc9jT333JOFCxc+btl73/tehoaGRnz8XXfdlac//ekrVbskSRNNqoZnltW04uRZwElVtV87/W6AqvrIKO0D/BLYo6p+P9p6586dW8MPYJY0udxwww3ssssu3d/h6rPh0n+Ae4dgkxlNONrjlb0rcAUsWLCAK6+8klNPPXXQpYxqpP5OclVVzR3lLpOaf6ckafBW5e9UL4fYXQHMSrIDcBtwGPDqzgZJNgX+UFUPA0cBl40VjiRppezxyjUmEEmSpDVbzwJSVS1NchxwETAFOLOqrktyTLt8PrAL8Pkkj9CcvOGNvapHktZERxxxBEccccSgy5AkSa1e7kGiqi4ELhw2b37H7z8GZvWyBkmSJEnqVk8vFCtJvdKr4ye1PPtZkjTZGJAkrXXWX3997r77br+891hVcffdd7P++usPuhRJkvqmp0PsJKkXZsyYwdDQEJ72v/fWX399ZsyYMegyJEnqGwOSpLXO1KlT2WGHHQZdhiRJmoAcYidJkiRJLQOSJEmSJLUMSJIkSZLUMiBJkiRJUsuAJEmSJEktA5IkSZIktQxIkiRJktQyIEmSJElSy4AkSZIkSS0DkiRJkiS1DEiSJEmS1DIgSZImrST7J7kxyeIkJ4ywfJMk5yf5eZLrkhw5iDolSf1jQJIkTUpJpgCnAQcAs4HDk8we1uxY4Pqq2hOYB3wsyXp9LVSS1FcGJEnSZLU3sLiqbq6qh4GFwMHD2hSwUZIAGwK/BZb2t0xJUj8ZkCRJk9U2wJKO6aF2XqdTgV2A24FrgLdX1aPDV5Tk6CRXJrnyzjvv7FW9kqQ+MCBJkiarjDCvhk3vBywCtgbmAKcm2fhxd6o6varmVtXc6dOnr/5KJUl9Y0CSJE1WQ8DMjukZNHuKOh0JnFONxcAvgZ37VJ8kaQAMSJKkyeoKYFaSHdoTLxwGnDesza3APgBJngTsBNzc1yolSX217qALkCRpEKpqaZLjgIuAKcCZVXVdkmPa5fOBDwELklxDMyTv+Kq6a2BFS5J6zoAkSZq0qupC4MJh8+Z3/H478KJ+1yVJGhyH2EmSJElSy4AkSZIkSS0DkiRJkiS1DEiSJEmS1DIgSZIkSVLLgCRJkiRJLQOSJEmSJLUMSJIkSZLUMiBJkiRJUsuAJEmSJEktA5IkSZIktQxIkiRJktQyIEmSJElSy4AkSZIkSS0DkiRJkiS1DEiSJEmS1DIgSZIkSVLLgCRJkiRJLQOSJEmSJLUMSJIkSZLUMiBJkiRJUsuAJEmSJEktA5IkSZIktXoakJLsn+TGJIuTnDDC8k2SnJ/k50muS3JkL+uRJEmSpLH0LCAlmQKcBhwAzAYOTzJ7WLNjgeurak9gHvCxJOv1qiZJkiRJGksv9yDtDSyuqpur6mFgIXDwsDYFbJQkwIbAb4GlPaxJkiRJkkbVy4C0DbCkY3qondfpVGAX4HbgGuDtVfVoD2uSJEmSpFH1MiBlhHk1bHo/YBGwNTAHODXJxo9bUXJ0kiuTXHnnnXeu/kolSZIkid4GpCFgZsf0DJo9RZ2OBM6pxmLgl8DOw1dUVadX1dyqmjt9+vSeFSxJkiRpcutlQLoCmJVkh/bEC4cB5w1rcyuwD0CSJwE7ATf3sCZJkiRJGtW6vVpxVS1NchxwETAFOLOqrktyTLt8PvAhYEGSa2iG5B1fVXf1qiZJkiRJGkvPAhJAVV0IXDhs3vyO328HXtTLGiRJkiSpWz29UKwkSZIkrU0MSJIkSZLUMiBJkiRJUsuAJEmSJEktA5IkSZIktQxIkiRJktQyIEmSJElSy4AkSZIkSS0DkiRJkiS1DEiSJEmS1DIgSZIkSVLLgCRJkiRJLQOSJEmSJLUMSJKkSSvJ/kluTLI4yQmjtJmXZFGS65J8v981SpL6a91BFyBJ0iAkmQKcBuwLDAFXJDmvqq7vaLMp8Glg/6q6NclWg6lWktQv7kGSJE1WewOLq+rmqnoYWAgcPKzNq4FzqupWgKr6TZ9rlCT1mQFJkjRZbQMs6Zgeaud12hHYLMn3klyV5HV9q06SNBAOsZMkTVYZYV4Nm14X2AvYB5gG/DjJT6rqP5dbUXI0cDTAtttu24NSJUn94h4kSdJkNQTM7JieAdw+QptvV9UDVXUXcBmw5/AVVdXpVTW3quZOnz69ZwVLknrPgCRJmqyuAGYl2SHJesBhwHnD2nwDeG6SdZNsADwTuKHPdUqS+sghdpKkSamqliY5DrgImAKcWVXXJTmmXT6/qm5I8m3gauBR4IyqunZwVUuSes2AJEmatKrqQuDCYfPmD5s+BTiln3VJkgbHIXaSJEmS1DIgSZIkSVKrq4CU5GtJXpLEQCVJkiRpwuo28HyG5mriNyU5OcnOPaxJkiRJkgaiq4BUVd+pqtcATwduAS5J8qMkRyaZ2ssCJUmSJKlfuh4yl2QL4AjgKOBnwCdoAtMlPalMkiRJkvqsq9N8JzkH2Bn4AnBgVd3RLvpykit7VZwkSZIk9VO310E6taq+O9KCqpq7GuuRJEmSpIHpdojdLkk2XTaRZLMkb+lRTZIkSZI0EN0GpDdV1T3LJqrqd8CbelOSJEmSJA1GtwFpnSRZNpFkCrBeb0qSJEmSpMHo9hiki4Czk8wHCjgG+HbPqpIkSZKkAeg2IB0P/A3wZiDAxcAZvSpKkiRJkgahq4BUVY8Cn2lvkiRJkjQhdXsdpFnAR4DZwPrL5lfVn/eoLkmSJEnqu25P0vBZmr1HS4EXAJ+nuWisJEmSJE0Y3QakaVV1KZCq+lVVnQT8Ze/KkiRJkqT+6/YkDQ8lWQe4KclxwG3AVr0rS5IkSZL6r9s9SO8ANgDeBuwF/DXw+l4VJUmSJEmDMO4epPaisK+sqncC9wNH9rwqSZIkSRqAcfcgVdUjwF5J0od6JEmSJGlguj0G6WfAN5J8BXhg2cyqOqcnVUmSJEnSAHQbkDYH7mb5M9cVYECSJEmSNGF0FZCqyuOOJEmSJE14XQWkJJ+l2WO0nKp6w2qvSJIkSZIGpNvTfF8AfLO9XQpsTHNGuzEl2T/JjUkWJzlhhOXvTLKovV2b5JEkm6/IBkiSJEnS6tLtELuvdU4nOQv4zlj3aU8PfhqwLzAEXJHkvKq6vmO9pwCntO0PBP62qn67QlsgSZIkSatJt3uQhpsFbDtOm72BxVV1c1U9DCwEDh6j/eHAWStZjyRJkiStsm6PQbqP5Y9B+jVw/Dh32wZY0jE9BDxzlPVvAOwPHNdNPZIkSZLUC90OsdtoJdY90oVlH3eih9aBwL+PNrwuydHA0QDbbjvejitJkiRJWjldDbFL8rIkm3RMb5rkkHHuNgTM7JieAdw+StvDGGN4XVWdXlVzq2ru9OnTuylZkiRJklZYt8cgfaCq7l02UVX3AB8Y5z5XALOS7JBkPZoQdN7wRm3wej7wjS5rkSRJkqSe6GqIHSMHqTHvW1VLkxwHXARMAc6squuSHNMun982fRlwcVU90GUtkiRJktQT3eLA3+0AABDoSURBVAakK5P8M81puwt4K3DVeHeqqguBC4fNmz9segGwoMs6JEmSJKlnuh1i91bgYeDLwNnAg8CxvSpKkiRJkgah27PYPQCc0ONaJEmSJGmguj2L3SVJNu2Y3izJRb0rS5IkSZL6r9shdlu2Z64DoKp+B2zVm5IkSZIkaTC6DUiPJnnsCq1Jtmf0i75KkiRJ0lqp27PYvRf4YZLvt9PPA47uTUmSJEmSNBjdnqTh20nm0oSiRTQXdX2wl4VJkiRJUr91FZCSHAW8HZhBE5D+Avgx8Je9K02SJEmS+qvbY5DeDjwD+FVVvQB4GnBnz6qSJEmSpAHoNiA9VFUPASR5QlX9Atipd2VJkiRJUv91e5KGofY6SOcClyT5HXB778qSJEmSpP7rag9SVb2squ6pqpOA9wP/ChzSy8IkSeq1JPsnuTHJ4iQnjNHuGUkeSXJoP+uTJPVft3uQHlNV3x+/lSRJa7YkU4DTgH2BIeCKJOdV1fUjtPsocFH/q5Qk9Vu3xyBJkjTR7A0srqqbq+phYCFw8Ajt3gp8DfhNP4uTJA2GAUmSNFltAyzpmB5q5z0myTbAy4D5faxLkjRABiRJ0mSVEebVsOl/AY6vqkfGXFFydJIrk1x5551eBUOS1mYrfAySJEkTxBAws2N6Bo8/Q+tcYGESgC2BFydZWlXndjaqqtOB0wHmzp07PGRJktYiBiRJ0mR1BTAryQ7AbcBhwKs7G1TVDst+T7IAuGB4OJIkTSwGJEnSpFRVS5McR3N2uinAmVV1XZJj2uUedyRJk5ABSZI0aVXVhcCFw+aNGIyq6oh+1CRJGixP0iBJkiRJLQOSJEmSJLUMSJIkSZLUMiBJkiRJUsuAJEmSJEktA5IkSZIktQxIkiRJktQyIEmSJElSy4AkSZIkSS0DkiRJkiS1DEiSJEmS1DIgSZIkSVLLgCRJkiRJLQOSJEmSJLUMSJIkSZLUMiBJkiRJUsuAJEmSJEktA5IkSZIktQxIkiRJktQyIEmSJElSy4AkSZIkSS0DkiRJkiS1DEiSJEmS1DIgSZIkSVLLgCRJkiRJLQOSJEmSJLUMSJIkSZLUMiBJkiRJUqunASnJ/kluTLI4yQmjtJmXZFGS65J8v5f1SJIkSdJY1u3VipNMAU4D9gWGgCuSnFdV13e02RT4NLB/Vd2aZKte1SNJkiRJ4+nlHqS9gcVVdXNVPQwsBA4e1ubVwDlVdStAVf2mh/VIkiRJ0ph6GZC2AZZ0TA+18zrtCGyW5HtJrkryuh7WI0mSJElj6tkQOyAjzKsRHn8vYB9gGvDjJD+pqv9cbkXJ0cDRANtuu20PSpUkSZKk3u5BGgJmdkzPAG4foc23q+qBqroLuAzYc/iKqur0qppbVXOnT5/es4IlSZIkTW69DEhXALOS7JBkPeAw4Lxhbb4BPDfJukk2AJ4J3NDDmiRJkiRpVD0bYldVS5McB1wETAHOrKrrkhzTLp9fVTck+TZwNfAocEZVXdurmiRJkiRpLL08BomquhC4cNi8+cOmTwFO6WUdkiRJktSNnl4oVpIkSZLWJgYkSZIkSWoZkCRJkiSpZUCSJEmSpJYBSZIkSZJaBiRJkiRJahmQJEmSJKllQJIkSZKklgFJkiRJkloGJEnSpJVk/yQ3Jlmc5IQRlr8mydXt7UdJ9hxEnZKk/jEgSZImpSRTgNOAA4DZwOFJZg9r9kvg+VW1B/Ah4PT+VilJ6jcDkiRpstobWFxVN1fVw8BC4ODOBlX1o6r6XTv5E2BGn2uUJPWZAUmSNFltAyzpmB5q543mjcC3RlqQ5OgkVya58s4771yNJUqS+s2AJEmarDLCvBqxYfICmoB0/EjLq+r0qppbVXOnT5++GkuUJPXbuoMuQJKkARkCZnZMzwBuH94oyR7AGcABVXV3n2qTJA2Ie5AkSZPVFcCsJDskWQ84DDivs0GSbYFzgNdW1X8OoEZJUp+5B0mSNClV1dIkxwEXAVOAM6vquiTHtMvnAycCWwCfTgKwtKrmDqpmSVLvGZAkSZNWVV0IXDhs3vyO348Cjup3XZKkwXGInSRJkiS1DEiSJEmS1DIgSZIkSVLLgCRJkiRJLQOSJEmSJLUMSJIkSZLUMiBJkiRJUsuAJEmSJEktA5IkSZIktQxIkiRJktQyIEmSJElSy4AkSZIkSS0DkiRJkiS1DEiSJEmS1DIgSZIkSVLLgCRJkiRJLQOSJEmSJLUMSJIkSZLUMiBJkiRJUsuAJEmSJEktA5IkSZIktQxIkiRJktQyIEmSJElSy4AkSZIkSS0DkiRJkiS1DEiSJEmS1DIgSZIkSVLLgCRJkiRJLQOSJEmSJLV6GpCS7J/kxiSLk5wwwvJ5Se5Nsqi9ndjLeiRJkiRpLOv2asVJpgCnAfsCQ8AVSc6rquuHNf1BVb20V3VIkiRJUrd6uQdpb2BxVd1cVQ8DC4GDe/h4kiRJkrRKehmQtgGWdEwPtfOGe1aSnyf5VpJde1iPJEmSJI2plwEpI8yrYdP/AWxXVXsCnwLOHXFFydFJrkxy5Z133rmay5T66Oqz4eO7wUmbNj+vPnvQFUmSJKlDLwPSEDCzY3oGcHtng6r6fVXd3/5+ITA1yZbDV1RVp1fV3KqaO3369B6WLPXQ1WfD+W+De5cA1fw8/22GJEmSpDVILwPSFcCsJDskWQ84DDivs0GSJydJ+/vebT1397AmaXAu/Qf404PLz/vTg818SZIkrRF6dha7qlqa5DjgImAKcGZVXZfkmHb5fOBQ4M1JlgIPAodV1fBheNLEcO/Qis2XJElS3/UsIMFjw+YuHDZvfsfvpwKn9rIGaY2xyYx2eN0I8yVJkrRG6OmFYiV12OdEmDpt+XlTpzXzJUmStEYwIEn9sscr4cBPwiYzgTQ/D/xkM1+SJElrhJ4OsZM0zB6vNBBJkiStwdyDJEmSJEktA5IkSZIktQxIkiRJktQyIEmSJElSy4AkSZq0kuyf5MYki5OcMMLyJPlku/zqJE8fRJ2SpP4xIEmSJqUkU4DTgAOA2cDhSWYPa3YAMKu9HQ18pq9FSpL6zoAkSZqs9gYWV9XNVfUwsBA4eFibg4HPV+MnwKZJ/qzfhUqS+seAJEmarLYBlnRMD7XzVrSNJGkCWesuFHvVVVfdleRXg65jFW0J3DXoIgZosm8/2AeTffth7e+D7QZdwGqQEebVSrQhydE0Q/AA/pjk2lWsbaJa21/3vWTfjM6+GZ19M7qdVvaOa11Aqqrpg65hVSW5sqrmDrqOQZns2w/2wWTffrAP1hBDwMyO6RnA7SvRhqo6HTgdfG7HYt+Mzr4ZnX0zOvtmdEmuXNn7OsROkjRZXQHMSrJDkvWAw4DzhrU5D3hdeza7vwDurao7+l2oJKl/1ro9SJIkrQ5VtTTJccBFwBTgzKq6Lskx7fL5wIXAi4HFwB+AIwdVrySpPwxIg3H6oAsYsMm+/WAfTPbtB/tgjVBVF9KEoM558zt+L+DYFVytz+3o7JvR2Tejs29GZ9+MbqX7Js1nvyRJkiTJY5AkSZIkqWVA6oMkU5L8LMkF7fScJD9JsijJlUn2HnSNvTTC9u+Z5MdJrklyfpKNB11jLyW5pd3WRcvOqJJk8ySXJLmp/bnZoOvspVH64BVJrkvyaJIJfQaeUbb/lCS/SHJ1kq8n2XTQdWrFJNk/yY1JFic5YYTlSfLJdvnVSZ4+iDoHoYu+eU3bJ1cn+VGSPQdR5yCM1zcd7Z6R5JEkh/azvkHqpm+SzGs/S69L8v1+1zgoXbynNmm/U/287ZtJc7xkkjOT/Ga0yyuszGexAak/3g7c0DH9v4EPVtUc4MR2eiIbvv1nACdU1e7A14F3DqSq/npBVc3pOBXnCcClVTULuLSdnuiG98G1wMuBywZYUz8N3/5LgN2qag/gP4F3D640ragkU4DTgAOA2cDhSWYPa3YAMKu9HQ18pq9FDkiXffNL4Pnt6/9DTJLjKLrsm2XtPkpzApFJoZu+af+R9GngoKraFXhF3wsdgC5fN8cC11fVnsA84GPt2TkngwXA/mMsX+HPYgNSjyWZAbyEJhQsU8CyvSabMMI1NSaKUbZ/J/7nS/ElwF/1u641wMHA59rfPwccMsBaBqKqbqiqGwddx6BU1cVVtbSd/AnN9XW09tgbWFxVN1fVw8BCmvd1p4OBz1fjJ8CmSf6s34UOwLh9U1U/qqrftZOT6fXfzesG4K3A14Df9LO4Aeumb14NnFNVtwJU1WTpn276poCNkgTYEPgtsJRJoKouo9ne0azwZ7EBqff+BXgX8GjHvHcApyRZAvwfJvZ/jkfa/muBg9rfX8HyF2GciAq4OMlVSY5u5z1p2bVU2p9bDay6/hipDyaT8bb/DcC3+lyTVs02wJKO6aF23oq2mYhWdLvfyOR5/Y/bN0m2AV4GzGdy6eZ1syOwWZLvtZ+nr+tbdYPVTd+cCuxC80/3a4C3V9WjCFbis9iA1ENJXgr8pqquGrbozcDfVtVM4G+Bf+17cX0wxva/ATg2yVXARsDDfS+uv55dVU+n2cV7bJLnDbqgAZjsfTDq9id5L81/+b44qOK0UjLCvOGnhe2mzUTU9XYneQFNQDq+pxWtObrpm38Bjq+qR/pQz5qkm75ZF9iLZmTKfsD7k+zY68LWAN30zX7AImBrYA5w6kQ/xnsFrPBnsddB6q1nAwcleTGwPrBxkv8LHEhzXA7AV1h++NlEMuL2V9VfAy8CaD/YXjLAGnuuqm5vf/4myddpdpX/d5I/q6o72t28E3qYwCh9MFmOPRp1+5O8HngpsE95zYW1zRDL7/2eweOHS3fTZiLqaruT7EHz9++Aqrq7T7UNWjd9MxdY2IyUYkvgxUmWVtW5/SlxYLp9T91VVQ8ADyS5DNiT5jjOiaybvjkSOLn9W7I4yS+BnYHL+1PiGm2FP4vdg9RDVfXuqppRVdsDhwHfbcPB7cDz22Z/Cdw0oBJ7arTtT7IVQJJ1gPcxgYcRJHliko2W/U4TDK8FzgNe3zZ7PfCNwVTYe2P0waQw2vYn2Z/mv+YHVdUfBlmjVsoVwKwkO7QHQh9G877udB7wuvYMSn8B3LtsaO0EN27fJNkWOAd4bVVN9C+3ncbtm6raoaq2b/92fhV4yyQIR9Dde+obwHOTrJtkA+CZLH8SqImqm765FdgHIMmTaI73vrmvVa65Vviz2D1Ig/Em4BNJ1gUeojmjxmRyeJJlV6Y/B/jsIIvpsScBX2//E7gu8KWq+naSK4Czk7yR5kNtIp+JZ7Q+eBnwKWA68M0ki6pqvwHW2Sujbf9i4AnAJe2yn1TVMYMrUyuiqpYmOY7mLGNTgDOr6rokx7TL5wMXAi8GFgN/oPkP74TXZd+cCGwBfLp9/S/tOMPjhNVl30xK3fRNVd2Q5NvA1TTHNp9RVRP+H25dvm4+BCxIcg3NkLLjq+qugRXdR0nOojlz35ZJhoAPAFNh5T+L46gOSZIkSWo4xE6SJEmSWgYkSZIkSWoZkCRJkiSpZUCSJEmSpJYBSZIkSZJaBiSpj5JsmuQtg65DkiRJIzMgSf21KWBAkiRJWkMZkKT+Ohl4SpJFSU4ZdDGSJElanheKlfooyfbABVW124BLkSRJ0gjcgyRJkiRJLQOSJEmSJLUMSFJ/3QdsNOgiJEmSNDIDktRHVXU38O9JrvUkDZIkSWseT9IgSZIkSS33IEmSJElSy4AkSZIkSS0DkiRJkiS1DEiSJEmS1DIgSZIkSVLLgCRJkiRJLQOSJEmSJLUMSJIkSZLU+n/nMCuGB5JfhAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1008x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#  single plot\n",
    "somemodelnames = [\"HMM\", \"SimpleRNN\"] #, \"LSTM\", \"GRU\", \"ad hoc\"]\n",
    "fig, ax = plt.subplots(1,2, figsize=(14, 5))\n",
    "for j, direction in enumerate(directions):\n",
    "    for i, model in enumerate(somemodelnames):\n",
    "        avg_test_acc = np.mean(test_accs[(model, direction)], axis = 1)\n",
    "        ax[j].plot(ts, avg_test_acc, 'o-', label = model)\n",
    "    ax[j].set_title('Performance of established ' + (\"bidirectional\" if direction else \"unidirectional\") + \" sequence models\" )\n",
    "    ax[j].set_xlabel(\"t\")\n",
    "    ax[j].set_ylabel(\"accuracy\");\n",
    "    ax[j].legend()\n",
    "print ('Accuracy was averaged over ' + str(num_reps) + ' repetitions')\n",
    "fig.savefig('failing1.pdf') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['HMM', 'SimpleRNN']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "modelnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABaYAAAR8CAYAAABmNUNvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzde7BkZ13v4e+PSSJXCcIAkgsCBkJERRmDlIggCBHBiCgS8ERRDFHxUl6jKOgRkfLGgQMaI0aNAgEOCMETidwEFJAMEpRwM0YhQwIOlwSIcDDkd/5Ya5POzp49nTD9zrDneaqmaq9ea3W/3b12KvXpd79d3R0AAAAAABjlRvt7AAAAAAAAHFyEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQDYZ6rqiKp6U1V9sqp+c3+PZ7SqunFVdVUducSxJ1TVRSPGdaCqqlOr6tX7exxf7Krq36rqPpvsf21Vfd/IMQEA7I0wDQBsKVX1qYV/V1fVpxe2H/sF3O9bqur79+VYt6gfS/If3X2L7n7S+p1VdXZV/cq6246tqqvWHdNV9eB1x50+3/7oefvUeftp64579Hz76fv0mXHQqKqTq+p9VXVFVX24qv6kqm62v8eVbPw71N136e43z/ufXlXPXbf/W7v7hSPHCQCwN8I0ALCldPfN1/4l+UCShy/c9rz9Pb5VqapD9vcYZndM8q59cD/vS/IDaxtV9SVJvivJf6w77qIkj62qxf+vPXk+n+vpALqOhtnDc359kvt09y2TfGWSL03ylC/g/gAAWEeYBgAOKlW1rap+taourqqPVNXzqurwed/N5tmIH6uqy6vqH6vqVlX1e0m+Iclz55nXv7fB/R5SVS+ZZ1deXlWvq6q7Ley/WVU9q6oumWdhvn4tYFXV/ecZ2VdU1Qeq6jHz7deapb247MHCkhE/WlX/luSd8+1/WFW7quoTVfXWqvrGdWN8yvzcP1FV51fV7efZoNdadqOqXlVVp+7hNfyWqvqnebxvqapvmG9/QZLvS/Kr8+v0zTfoTZq8NMmDquoW8/bDk7w5yUfXHff+JP+e5AHzGG6X5J5J/mZPd7y2hEZV/cp8DXywqh5aVSfOSyJ8tKp+duH4m1TVc6rqsvm1/Z2qOnRh/5Pm931Xku9f91g3qar/Nb/vH6qq/z1H9utls+trvn4+UFW1cPxJVfXW+efNrvljq+qqqvqRqrokyblLXMu3raq/ma+ht9Q0Q/fVC/vvUdPSER+vqndX1XetO/fc+dw3Z/ogY7Pn/ciqetc8jldX1THz7b9WVX+57tg/qqrfnn/+sqo6a37NL5mv+xvN+06dx/ecqvp4ktPWP253v7+71661SnJ1pkC90Rj39Lu42etw9nwtvK6mZW9eU1VH7O01rKqfTPLIXPM79uL59g9V1X3n434myQ/M+9eugc//t2S+Hn59vmY+XFVnrv2eLVwPj5uv9d1V9fML4/qmqnr7/P59qKp+a7P3DwBgM8I0AHCw+fkkD05y3yRHJvnvJM+Y9z0+ySFJjkhymyRPTPLZ7v7ZJOcnefw88/pnr3Ovk3OS3CXJ7ZO8J8mfL+x7VpJjMwXuL0vyK0m6qr4yyV8n+Z0kt05yryQXXo/n87D5nK+bt9+c5Kvn+3p5khcvRNRfyjTr+MFJDk9ySpLPzON8zFrYrKo7JPmmJC9a/2BVddskr0jy9PkxTs8UM2/Z3ScleUmS35hfpzdej+ex3pVJzkvyPfP2yUnO2sOxZ837k+SxSV6c5Ko9HLvmjpne+9tnei5nzo/1NUkelOQ3F0Lhr8+3f3Wm1/r+SX4hSeYQ+GNJviXT+/vt6x7nGZmus69Ocrckd80GIXRJe7q+Xp8pnt534djHJHn+/PNm13ySbEty73l8J+7lsZLkjCS7k9wu0zW0OLP9S5O8KsmfZPodOjnJmfN1vnbux+ZzfzTJD+3pyVbVPZL8WabX97bz8zynpg90np/kxKq6yXzsoZnev7Xn/LwkVyS5c5LjM133/2Ph7u+X5IJ5jNf5oGm+zwdW1RXz/Tw00+/wZj7/u7jE65B5PL+cZHuSf838Gm92bnc/K9f+HfvexQF098uS/H6SP5/3H7/BOJ+Q5FFJvjnJMZle299f2L8tyY5MIf6hmX4X7jzve3aSp3X3l87nvmwvrwkAwB4J0wDAweYJSU7r7ku7+zOZouP3zVH2vzNFort091XdfX53X7nMnc7H/3l3f2rhfo+fZ1Memiku/UR3f6i7P9fdb+zuz2WKU6/o7pfM97G7u99xPZ7Pb3b35d396XkcZ3X3x7v7v5M8LVM8XotKj5+f+0XdfXV3v727L0/yxiSda8LmY5K8srs/tsHjnZjkgu5+0TzeP0uyK9cNspt50jwD9vKqujzJW/dw3FlJTq6q22QK+v93D8e9OMkJNa0BvFnAXvRfSX6nu69KcnamUPq73X1ld789yb9lisnJFLuf0t0f6e4PJ3lqromcj0ryx939nu7+VKb3Pcnnl3T4oSQ/Nb9HV2SK4I9eYnzXstn11d2d5IVJTpof98syxfW1NYU3u+bXPLm7/6u7P72Xa/nGSb4zya/Ox/5zpgi85hFJ3tndz5uv8/MzfZDxyIVzf2U+94J15653UpK/6u6/6+7PZrqeb5NkR3e/L8l7M82kT5ITkny4uy+oqjtmCs8/Mz+nyzJF5cXX/eLu/uN5jJ/ew2v+mnkpj6MzhdsPbDLW5Nq/i3t8HRaOf1l3v3l+jX85yQOravuS534hHpvp2n9/d38iyZMyLYezeD08pbs/Mz/2ezJ9MJNM/428a1Xdurs/2d3/uI/GBAAchIRpAOCgMYeXozLN8F2Lom/P9P9Et840Q/H1Sf7P/GfsT6uqbUve9yFV9bs1L5ORKebUfL9fnmkm9sUbnHpUpgh6Q12ybhy/VFXvnWd6fjzJjZPcZn7uR2z0WHPYPCvXLEPx/Un+Yg+Pd4dMy2csev9838v6ze4+fO1fphmtG3ltpqh+WpKXznHyOrr7k0lek2kN4EO7+21LjGF3d189/7wWJj+8sP/TSW4+v263z7Wf8+LzvUOu/R4sHneHJIcmuXDhentZphmq18terq9kmin8vXMM/94kf9/dly1xzSfJ1d196ZKPdfv5510Lw1t8/ndMcr91Hzw8MtPvwNq5e3q91rvWtTZ/kPPBXPPaPz9zjM/0Ycpa5L5jput+98IYnpnpw4eNxryp7r4kyeuS/OVeDl32dbjO8fOHQJ/K9JyXOfcLsf53+P1JbpLpLzmS5HPd/ZGF/f+V5Obzzz+QKVK/r6aljh6yj8YEAByEfDEHAHDQ6O6uqg8m+e5N4uWTkzx5/tP18zItq/G8TDOKN/O4TMslPCDTzMrbJbksU4i7LNPSEnfONMtz0SWZlnfYyJVJbrqwffuNntbaD1X1bUl+ItNs2XfPj/3JJLXw3O+S6QsD1zsryflVdUamkLmn2cmXJnngutuOzhQM96nuvrqmdat/Icl99nL4WUnOzbRcyb4cQ1fVhzLFwrWov/h8L8v0emVhXxb2XZVpBv76tbGvr82ur3T3P1XVRzO994/JtATGXq/5eTb6+mt7s8f60Hz8EblmBvHi878kyd9298Ozzjxjuufj/2O++ej1xy24NAtrUM8fEh2Ra177FyZ56rzkysMzzfxdG8Onktxq/tBlI3v7fV7vkEy/O5tZvM89vg4LPv+6zbPcb57pdd7buXsb+972X+t1zfQefDrTEivbN73j7ndnmm2/LdMM9JdW1a329KERAMBmzJgGAA42pyd5elUdlXz+y9gePv/8oKo6rqYvSftEpqj4ufm8D+eaJTE2cotM6zV/NMnNMi33kCSZl9U4K8kzq+p2NX352H3nuHNWkodV1SPm27dX1dqfzV+Q5HvmJRSOTfKDe3lut8j0p/a7kxyW5H9mmjm65rlJnlZVd67J19X8JXjdfXGSdyX50yQv3CQ0nZNpDd3vmWfWnpwpbL1yL2O7oX43yYOWWDLgVZli6ukrGMMLkjylqm49r7H9pFwze/ZFSR5fVXetqptn+mAjyeff9zMzve+3mV/zo+YPEK6vPV5f68b5c5mWPXnpwu17vOav72PNy068Ismvz9flPTKF8DUvy3R9fF9VHVpVh1XVN1bVXdede5P5On/sJuN4YZJHVNX9aloO57R5TDvnsXwwyT9mivD/Ml/D6e5/T/KWJL9dVbeoqhtV1TFVdd+NHmQjVXVyVR05/3ynTL9Lr1n2/M1eh4VjTqyqe9f0ZZhPTfK67v7PJc7d23+LPpzkTvNs+Y28IMnPVdXRNX3p4VOTPH+TiP958+ty63n2+hWZIvjVezkNAGBDwjQAcLD57SSvTvLaqvpkkjcl+fp53xGZvjDwk0nemWkG7toXAD4j03rHH6+q397gfv8kUxD+UJJ/SfL36/b/ZKYZt2/PFNd+I9NM5n/LtG7zL2daemNnkq9aGOsh8/2ekb0vJfCKJG+YH+fiJB+Zz13z9EwzoV+bKbyfnuRLFvb/eaZ1lfe0jEfmNZa/M1Oc/WimL4h82LxW9T43r+v82iWO+1x3v3pex3lfe3KmaH9hpg8L/iHTe5Pu/qtM780bMy15cd66c3860wzVnZlC3iszfanc9bW36yuZlrZ4YJK/Wfc6bHbN35DHekKm5SB2Z/qw4wVJ/l+SdPfHkzwk06zryzI996dmWtJk7dzbZYqnf5Tpg5ANzetX//B83O75uZ04rwu++JwflGu+9HDNSZm+4PM9mWYCvzDXXspjb74myVur6spM7+07Mn0J41KWeB2S6ff56Zl+T++e+Usklzj3jCTfMC/zcfYGD392pr+0+FhVvWmD/X+Y6YOLN2X6b8XHkvzMkk/tYUneO19Hv5XkUeveDwCApdUSH4wDAHAQqKoHJ/mD7r4h4ZSDVFU9M8mNu/sJ+3ssXyzmoPzO7t5o5jsAwEHBGtMAAKSqDss0q/uM/T0WDmzz8h2daRb5fZKcnGu+hBAAAJaysqU8qurMqvrPqnrnHvZXVT2rqi6qqn+uqs3+nBAAgBWpqntmWkbkFkmes5+Hw4HvlpmWjbky03IUT+3uVa0xDgDAFrWypTyq6n6Zvg37rO6+xwb7H5rpW+MfmuTeSZ7Z3fdeyWAAAAAAADhgrGzGdHe/IdMXaezJiZmidXf3W5IcXlVfvqrxAAAAAABwYFhZmF7CEUkuWdjeNd8GAAAAAMAWtj+//LA2uG3DdUWq6pQkpyTJzW52s3sde+yxqxwXAAAAAABfoLe97W0f6e7tG+3bn2F6V5KjFraPTHLpRgd29xmZvyF+x44dvXPnztWPDgAAAACAG6yq3r+nfftzKY9zkpxck29MckV3X7YfxwMAAAAAwAArmzFdVS9Icv8kt6mqXUmekuTQJOnu05Ocm+ShSS5K8l9JHreqsQAAAAAAcOBYWZju7pP2sr+T/PiqHh8AAAAAgAPT/lzKAwAAAACAg5AwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMNRKw3RVnVBV762qi6rqtA3236qq/qqq/rmq3lpV91jleAAAAAAA2P9WFqaraluS5yT59iTHJTmpqo5bd9gvJ7mgu78myclJnrmq8QAAAAAAcGBY5Yzp45Nc1N0Xd/dnk5yd5MR1xxyX5DVJ0t3vSfIVVXW7FY4JAAAAAID9bJVh+ogklyxs75pvW/SOJN+dJFV1fJI7Jjly/R1V1SlVtbOqdu7evXtFwwUAAAAAYIRVhuna4LZet/30JLeqqguS/ESStye56jondZ/R3Tu6e8f27dv3/UgBAAAAABjmkBXe964kRy1sH5nk0sUDuvsTSR6XJFVVSf59/gcAAAAAwBa1yhnT5yc5pqruVFWHJXl0knMWD6iqw+d9SfL4JG+YYzUAAAAAAFvUymZMd/dVVfXEJOcl2ZbkzO6+sKpOnfefnuTuSc6qqs8leVeSH17VeAAAAAAAODCscimPdPe5Sc5dd9vpCz+/OckxqxwDAAAAAAAHllUu5QEAAAAAANchTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAwlTAMAAAAAMJQwDQAAAADAUMI0AAAAAABDCdMAAAAAAAy10jBdVSdU1Xur6qKqOm2D/besqldU1Tuq6sKqetwqxwMAAAAAwP63sjBdVduSPCfJtyc5LslJVXXcusN+PMm7uvtrk9w/ye9V1WGrGhMAAAAAAPvfKmdMH5/kou6+uLs/m+TsJCeuO6aT3KKqKsnNk3wsyVUrHBMAAAAAAPvZKsP0EUkuWdjeNd+26NlJ7p7k0iT/kuSnuvvq9XdUVadU1c6q2rl79+5VjRcAAAAAgAFWGaZrg9t63fZDklyQ5A5J7pnk2VX1pdc5qfuM7t7R3Tu2b9++70cKAAAAAMAwqwzTu5IctbB9ZKaZ0Ysel+SlPbkoyb8nOXaFYwIAAAAAYD9bZZg+P8kxVXWn+QsNH53knHXHfCDJA5Okqm6X5G5JLl7hmAAAAAAA2M8OWdUdd/dVVfXEJOcl2ZbkzO6+sKpOnfefnuQ3kvxZVf1LpqU/frG7P7KqMQEAAAAAsP+tLEwnSXefm+TcdbedvvDzpUkevMoxAAAAAABwYFnlUh4AAAAAAHAdwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMtFaar6iVV9R1VJWQDAAAAAPAFWTY0/2GSxyT516p6elUdu8IxAQAAAACwhS0Vprv71d392CRfn+Q/kryqqt5UVY+rqkNXOUAAAAAAALaWpZfmqKpbJ/nBJI9P8vYkz8wUql+1kpEBAAAAALAlHbLMQVX10iTHJvmLJA/v7svmXS+sqp2rGhwAAAAAAFvPUmE6ybO7+7Ub7ejuHftwPAAAAAAAbHHLLuVx96o6fG2jqm5VVT+2ojEBAAAAALCFLRumf6S7L1/b6O6PJ/mR1QwJAAAAAICtbNkwfaOqqrWNqtqW5LDVDAkAAAAAgK1s2TWmz0vyoqo6PUknOTXJK1c2KgAAAAAAtqxlw/QvJnlCkh9NUkn+NslzVzUoAAAAAAC2rqXCdHdfneQP538AAAAAAHCDLRWmq+qYJL+V5LgkN167vbvvvKJxAQAAAACwRS375Yd/mmm29FVJHpDkrCR/sapBAQAAAACwdS0bpm/S3a9JUt39/u7+tSTfurphAQAAAACwVS375YefqaobJfnXqnpikg8mue3qhgUAAAAAwFa17Izpn05y0yQ/meReSb4/yQ+salAAAAAAAGxde50xXVXbkjyqu38+yaeSPG7lowIAAAAAYMva64zp7v5ckntVVQ0YDwAAAAAAW9yya0y/PcnLq+rFSa5cu7G7X7qSUQEAAAAAsGUtG6a/LMlHk3zrwm2dRJgGAAAAAOB6WSpMd7d1pQEAAAAA2CeWCtNV9aeZZkhfS3f/0D4fEQAAAAAAW9qyS3n89cLPN07yiCSX7vvhAAAAAACw1S27lMdLFrer6gVJXr2SEQEAAAAAsKXd6Aaed0ySo/flQAAAAAAAODgsu8b0J3PtNaY/lOQXVzIiAAAAAAC2tGWX8rjFqgcCAAAAAMDBYamlPKrqEVV1y4Xtw6vqu1Y3LAAAAAAAtqpl15h+SndfsbbR3ZcnecpqhgQAAAAAwFa2bJje6LillgEBAAAAAIBFy4bpnVX1+1V1l6q6c1U9I8nbVjkwAAAAAAC2pmXD9E8k+WySFyZ5UZJPJ/nxVQ0KAAAAAICta6nlOLr7yiSnrXgsAAAAAAAcBJaaMV1Vr6qqwxe2b1VV561uWAAAAAAAbFXLLuVxm+6+fG2juz+e5LarGRIAAAAAAFvZsmH66qo6em2jqr4iSa9iQAAAAAAAbG1LrTGd5ElJ/r6qXj9v3y/JKasZEgAAAAAAW9myX374yqrakSlGX5Dk5Uk+vcqBAQAAAACwNS0Vpqvq8Ul+KsmRmcL0NyZ5c5Jv3ct5JyR5ZpJtSZ7b3U9ft//nkzx2YSx3T7K9uz92PZ4DAAAAAABfRJZdY/qnknxDkvd39wOSfF2S3ZudUFXbkjwnybcnOS7JSVV13OIx3f073X3P7r5nkl9K8npRGgAAAABga1s2TH+muz+TJFX1Jd39niR328s5xye5qLsv7u7PJjk7yYmbHH9SkhcsOR4AAAAAAL5ILRumd1XV4UleluRVVfXyJJfu5ZwjklyyeB/zbddRVTdNckKSl+xh/ylVtbOqdu7evelEbQAAAAAADnDLfvnhI+Yff62qXpfklkleuZfTaqO72sOxD0/yD3taxqO7z0hyRpLs2LFjT/cBAAAAAMAXgaXC9KLufv2Sh+5KctTC9pHZ8yzrR8cyHgAAAAAAB4Vll/K4Ic5PckxV3amqDssUn89Zf1BV3TLJtyR5+QrHAgAAAADAAeJ6z5heVndfVVVPTHJekm1JzuzuC6vq1Hn/6fOhj0jyt9195arGAgAAAADAgaO6v7iWbN6xY0fv3Llzfw8DAAAAAIBNVNXbunvHRvtWuZQHAAAAAABchzANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCaKWTsAACAASURBVCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAwlDANAAAAAMBQwjQAAAAAAEMJ0wAAAAAADCVMAwAAAAAw1ErDdFWdUFXvraqLquq0PRxz/6q6oKourKrXr3I8AAAAAADsf4es6o6raluS5yT5tiS7kpxfVed097sWjjk8yR8kOaG7P1BVt13VeAAAAAAAODCscsb08Uku6u6Lu/uzSc5OcuK6Yx6T5KXd/YEk6e7/XOF4AAAAAAA4AKwyTB+R5JKF7V3zbYvumuRWVfV3VfW2qjp5ozuqqlOqamdV7dy9e/eKhgsAAAAAwAirDNO1wW29bvuQJPdK8h1JHpLkV6vqrtc5qfuM7t7R3Tu2b9++70cKAAAAAMAwK1tjOtMM6aMWto9McukGx3yku69McmVVvSHJ1yZ53wrHBQAAAADAfrTKGdPnJzmmqu5UVYcleXSSc9Yd8/Ik31xVh1TVTZPcO8m7VzgmAAAAAAD2s5XNmO7uq6rqiUnOS7ItyZndfWFVnTrvP727311Vr0zyz0muTvLc7n7nqsYEAAAAAMD+V93rl30+sO3YsaN37ty5v4cBAAAAAMAmqupt3b1jo32rXMoDAAAAAACuQ5gGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhhKmAQAAAAAYSpgGAAAAAGAoYRoAAAAAgKGEaQAAAAAAhlppmK6qE6rqvVV1UVWdtsH++1fVFVV1wfzvyascDwAAAAAA+98hq7rjqtqW5DlJvi3JriTnV9U53f2udYe+sbsftqpxAAAAAABwYFnljOnjk1zU3Rd392eTnJ3kxBU+HgAAAAAAXwRWGaaPSHLJwvau+bb17lNV76iqv6mqr1rheAAAAAAAOACsbCmPJLXBbb1u+5+S3LG7P1VVD03ysiTHXOeOqk5JckqSHH300ft6nAAAAAAADLTKGdO7khy1sH1kkksXD+juT3T3p+afz01yaFXdZv0ddfcZ3b2ju3ds3759hUMGAAAAAGDVVhmmz09yTFXdqaoOS/LoJOcsHlBVt6+qmn8+fh7PR1c4JgAAAAAA9rOVLeXR/f/Zu/No2c6yTsC/lwREQkgYwpjI1DQhjTJd0W5sEGwRlBkZAjQ2ihgExF7K3IgTdpYgig0aWQyCyhBmUCQE6AYaAbkxAUFBwxCSDkMQkjCEIfD2H3sfUjk5594K3Prq5tznWeuuU1V7qHfvqq9W8quv3t0XVtWjk5yU5KAkL+zuD1fVcfPyE5L8bJJHVtWFSS5I8sDu3tzuAwAAAACAHaQuaznwrl27evfu3esuAwAAAACAPaiqU7p711bLVtnKAwAAAAAALkEwDQAAAADAUIJpAAAAAACGEkwDAAAAADCUYBoAAAAAgKEE0wAAAAAADCWYBgAAAABgKME0AAAAAABDCaYBAAAAABhKMA0AAAAAwFCCaQAAAAAAhhJMAwAAAAAwlGAaAAAAAIChBNMAAAAAAAwlmAYAAAAAYCjBNAAAAAAAQwmmAQAAAAAYSjANAAAAAMBQgmkAAAAAAIYSTAMAAAAAMJRgGgAAAACAoQTTAAAAAAAMJZgGAAAAAGAowTQAAAAAAEMJpgEAAAAAGEowDQAAAADAUIJpAAAAAACGEkwDAAAAADCUYBoAAAAAgKEE0wAAAAAADCWYBgAAAABgKME0AAAAAABDCaYBAAAAABhKMA0AAAAAwFCCaQAAAAAAhhJMAwAAAAAwlGAaAAAAAIChBNMAAAAAAAwlmAYAAAAAYCjBNAAAAAAAQ1V3r7uGS6WqzklyxrrrgMGukeTz6y4CGMq4hwOPcQ8HFmMeDjzGPQei63f3EVstuMwF03Agqqrd3b1r3XUA4xj3cOAx7uHAYszDgce4h4vTygMAAAAAgKEE0wAAAAAADCWYhsuG5627AGA44x4OPMY9HFiMeTjwGPewQI9pAAAAAACGMmMaAAAAAIChBNOwH6qqg6rq1Kr66/n+LavqvVV1WlXtrqrbrrtGYN/aYtzfoqreU1X/WFVvrKqrrLtGYN+pqk/O4/u0qto9P3a1qjq5qv51/nvVddcJ7DvbjPv7VdWHq+rbVbVr3TUC+842Y/4ZVfWRqvpgVb22qg5fd52wToJp2D89Nsk/L9z//SS/1d23TPIb831gZ9k87p+f5Ind/YNJXpvkcWupClilO3b3Lbt7I4x6YpK3dfdNkrxtvg/sLJvH/YeS3CfJO9dYE7A6m8f8yUlu3t0/lORfkjxpfaXB+gmmYT9TVUcm+ZlModSGTrIxW/KwJGePrgtYnW3G/U1z0f+knpzkvqPrAoa7Z5IXz7dfnORea6wFGKC7/7m7P7ruOoAxuvst3X3hfPe9SY5cZz2wboJp2P/8UZLHJ/n2wmO/muQZVXVmkmfGt6qw02w17j+U5B7z7fslOWp0UcBKdZK3VNUpVfWI+bFrdfenk2T+e821VQeswlbjHti59jbmfz7J3w6uCfYrgmnYj1TV3ZJ8rrtP2bTokUn+e3cfleS/J3nB8OKAldjDuP/5JI+qqlOSHJrkG8OLA1bpdt196yR3zTTWb7/ugoCVM+7hwLLtmK+qpyS5MMlfras42B8IpmH/crsk96iqTyZ5eZI7VdVfJvm5JK+Z13llEhc/hJ1jy3Hf3R/p7jt3922SvCzJx9ZZJLBvdffZ89/PZeojf9skn62q6yTJ/Pdz66sQ2Ne2GffADrXdmK+qn0tytyQP7u5eX4WwfoJp2I9095O6+8juvkGSByZ5e3c/JFNP6TvMq90pyb+uqURgH9tu3FfVNZOkqi6X5H8kOWGNZQL7UFUdUlWHbtxOcudM7XvekOnL6Mx/X7+eCoF9bQ/jHtiBthvzVXWXJE9Ico/u/uo6a4T9wcHrLgBYyi8meXZVHZzka0n0pIOd79iqetR8+zVJXrTOYoB96lpJXltVyfTf4y/t7jdX1fuTnFhVv5DkU5n6ywM7w3bj/t5J/leSI5L8TVWd1t0/tcY6gX1juzF/epLvS3LyvOy93X3c+sqE9Sq/GgAAAAAAYCStPAAAAAAAGEowDQAAAADAUIJpAAAAAACGEkwDAAAAADCUYBoAAAAAgKEE0wAAsEZVdXhV/fK66wAAgJEE0wAAsF6HJxFMAwBwQBFMAwDAeh2f5MZVdVpVPWPdxQAAwAjV3euuAQAADlhVdYMkf93dN19zKQAAMIwZ0wAAAAAADCWYBgAAAABgKME0AACs15eSHLruIgAAYCTBNAAArFF3/1uSd1fVh1z8EACAA4WLHwIAAAAAMJQZ0wAAAAAADCWYBgAAAABgKME0AAAAAABDCaYBAAAAABhKMA0AAAAAwFCCaQAAAAAAhhJMAwAAAAAwlGAaAAAAAIChBNMAAAAAAAwlmAYAAAAAYCjBNAAAAAAAQwmmAQAAAAAYSjANAAAAAMBQgmkAAAAAAIYSTAMAAAAAMJRgGgAAAACAoQTTAAAAAAAMJZgGAAAAAGAowTQAAAAAAEMJpgEAAAAAGEowDQAAAADAUIJpAAAAAACGEkwDAAAAADCUYBoAAAAAgKEE0wAAAAAADCWYBgAAAABgKME0AAAAAABDCaYBAAAAABhKMA0AAAAAwFCCaQAAAAAAhhJMAwAAAAAwlGAaAAAAAIChBNMAAAAAAAwlmAYAAAAAYCjBNAAAAAAAQwmmAQAAAAAYSjANAAAAAMBQgmkAAAAAAIYSTAMAAAAAMJRgGgAAAACAoQTTAAAAAAAMJZgGAAAAAGAowTQAAAAAAEMJpgEAAAAAGEowDQAAAADAUIJpAAAAAACGEkwDAAAAADCUYBoAAAAAgKEE0wAAAAAADCWYBgAAAABgKME0AAAAAABDCaYBAAAAABhKMA0AAAAAwFCCaQCAA1xVXa+q/q6qvlRVT7+U2/77qjp3RXUdV1VvXcW+Lwuq6viqev6S6763qh6y6pr2Z1X1mar6sXXXcVlWVb9QVW/cw/L/UlUfGFkTALBzCaYBgLWrqi8v/Pt2VV2wcP/B38N+D/iwbkm/nOST3X1odz9l88KqukFVva6qPl9V51XVB6vqQUnS3f/S3YcPr/iSNb68qr4+v2e+UFVvrqp/t7D8uKrqqnrMpu0+X1U/Ot8+fl7n7gvLrzw/du1xR8NOUVWXr6r/U1XnzGPn1Kr66XXXlSRVdXRVXbj4WHe/oLvvPi+/4vzeP3Jh+Vu7+xajawUAdibBNACwdt195Y1/ST6V5O4Lj/3Vuutblao6eN01zK6f5J/2sPxlST6a5Kgk10jysCSfH1DXpfU783voyCRfTPJnm5Z/IclTqupKe9jHF5L8TlXVimo8YFTVQeuuYaRtxvO3kjw6yXW6+7Akj0lyYlVd/XvYJwDAjiCYBgD2e1V1UFU9tao+Ps9w/auqOnxedsg8W/YLVXVuVb2vqq5aVX+Q5IeTPH+eRfsHW+z34Kp6dVV9dt72f1fVTReWH1JVf1xVZ86zHd+xERRV1Y/PM7LPq6pPbcwg3jxLe7EdxcIMxEdW1ceSfGh+/E+r6qyqOr+q/n5jBu9CjU+bj/38qnp/VV27ql6wue1GVZ1cVcdtcw7vUFX/MNf73qr64fnxlyV5QJKnzufpP2/arpLsSvKi7r6gu7/Z3ad091vm5RebdTnv+zfn4/hyVb2mqq5eVSfO9b93Ywbmwvl4dFV9cp5V+vTtQuGqunlVvb2qvlhV/1xV99pqve7+apJXJrnlpkWnJvnHTOHgdt6Q5PuS3H8P6yzWtPTxzutv+TrMy/5dVb27ppYqf5vkqpue6z/P7+9z533cbpkat6j53lX1gbm+M6rqyQvL/k9VPXzT+h+teZbvnl6DeRz+cVW9paq+kuQ/7um55m0ePo+fc6rq8bXQjqP2MO7n5b+wsO3j9nLMV6uql87rfmJ+rqppjH+5Lj67/no1/Wrjqgvn64PzeX9XVR2zsO5nqurXq+rDSc7f/Lzd/e3u/lB3Xzi/r7+d6f11vW3qPH6u8xVV9aUkD9zTedgYfzV9zny6qs6uhV8F7OUcvjPJQXXRr1NuVRdvn/PO+e9H5+X3qqq7VNXpC/v/wfmcnDufo7suLHt5Vf1RVZ00v6ffXVXXX6jrOXXRTPIP1MJnLwBwYBBMAwCXBY9LcuckP5ZpNuw3k/zhvOzhSQ7OFPRcI9PsxG90968leX+Sh88zr39tm32/IcmNk1w7yUeSvHhh2R8nOTpTwH21JP8jSc8h1l8neUaSqye5TZIPX4rjudu8za3m++9J8oPzvl6f5JVVdfl52ZOS3Gs+/sOTPCLJ1+Y6HzSHXamq6ya5XZITNz9ZVV0zyRuTHD8/xwlJ3lRVh3X3sUlenXm2cXe/a3Hb7u4k70vyZ1V1/1oIWffgAZmC3R+Yj+vdSZ6b6RyekWRzu5C7ZwqRb5vk2CSXaN9SVVdJcnKSF2R6nR+a5IWLgeLCuocmeWCS0zcvy/QaPn7e31a+leRpSX6rlp/xu9Tx7ul1mPdzYqYw8OpJnpnkvy4c0w2SvG7e18Z78XUb4emldH6SB2V6P907ya9X1V3mZS/N9BpsPO9t5ud7y5KvwUOSPDXJoZnG37bPVVW3TPKsTOfuyPnfNRb2te24n7f9o0zn/sgkN9i07WYnJLl8khsm+ckkj0zyoO7+SqbPgGMX1n1gkpO6+4s1fUn0J5l+JXD1JH+R6bwvzmR+wLzPbWdBV9XJmcbtu5O8KdMXJNu5b6bxfVimsbmnz78kOSjJf0xyoyQ/k+m9u9Fre0/b3j7JtxZ+nXLqpjpuP/+96bz8dZuO6YqZPgdfl+SI+bleWVU3XFjtQZk+w66W5NNJfmt+fOMz8MaZvoB5UKZfOQAABxDBNABwWfBLSZ7Y3Wd399cyhRsPmEPZb2YKRW7c3Rd29/vnsGmv5vVf3N1fXtjvbWuayXv5TMHbY7r7M939re5+V3d/K1Ng+MbufvW8j3O6+9JcEOzp3X1ud18w1/GS7v5id38zye9lCrhuNK/78PnYT59nX57a3ecmeVeSzhQ4JVOw8+bu/sIWz3fPJKd194lzvX+e5Kwkd91i3a3cK8nfz+fnU1W1u6putYf1n9/dn5xreUuSf+7ud3T3hUlelYsC+Q3/cz4fn0jynFw8JNxw7yQf6u6/ml+L92cKee+7sM5TaroQ4/lJbp0pTLyY7n5fpnBwuy8qkmm29dcyvf7LWPZ4t30dqurfJzkmyW919ze6+21J3rzwHD+X5DVzj99vd/ebMrVfufOSNX5Hd7+tuz887+cfMgXid5gXvyrJf6qq68z3H5TklfOxLPMavKq73zfv++t7ea77J3l1d7+3u7+eKWxf/P+TPY37jW3fM2/75Gzz/zZV9X1zjU+Yx/rpmULtjeD/YmH8fMwvXajhOfOvBL7V3c/LNOP5Ngvr/+Fc4wVbn/Gku38yU1h/jyRvm7/w2c47uvtN8zm7YC/nYcPT5l80nJrkLxeOZ5ltv1sbv6541vxLipMyfXHxgIV1Tuzuf5g/216ai37F8M0kV8n0xV/P75HP7YOaAIDLEME0ALBfmwOUozLNLD13Dh5PzfTfMVfPNHvzHUleVVM7jN9bdqZrTW0ynjn/zP38TDOma97vdTLNxP74FpseleRj38NhnbmpjifV1C7hvEyzBq+Y5BrzsV9vq+eag62XZJqhmvnvX2zzfNfNNHN30RnZpp3AFs/1+e5+XHffLNPM8n9J8po9bPLZhdsXbHH/ypvWXzwfZ8z1bnb9JLffeA/M74P7ZnqdNjy9pwsx3ijTzOdLzKaePTXJY2ubPr/zuX1qkt9IcoVt9rFo2ePd0+tw3STnzOHh4rIN10/ykE3Hvytbn6s9qqrb1dSW5pz5PfffMs82nsP1k5Pcv6oulylk3OjzvsxrsPm9ve1zzbV/Z/3uPj/JefN2exv3m7c9b2PbLVx73u5TC48tvv9PSnKtqrrF/AXBTTIF7hvH/ORNx3xELj52LnbM25m/cHhjkvtU1Z6+UPjO/pY4D1vVcEaS616Kbb9b103yqU0h++bPlc8s3P5qLhoLf5vps/vPkny2qv6kqjZ/LgAAO5xgGgDYr82hx/9LcqfuPnzh3xXnwPTr3f0b3X10pp+e3y/TT/GTaUbxnjws04zTO2b62fzR8+OV6WfnF+aimcuLzsz0E/StfCXJ4sX1rr3VYW3cqKqfzNTz+N6Z2h1cLVOYWQvHvt1zvSTJz87tFo5K8jfbrHd2poBt0Q/M+75U5lmNz0pyg6o65NJuv42jNtV19hbrnJnkLZveA1fu7l/dosZPJPn1JP+rqi4RLM+z29+c5AnbFTQHiJ/JNGN9X9nT6/DpTF9GXHHTsg1nZpqZvXj8h3T3YkuHZZ2Y5BVJjurpgnx/nuk9v+FlmWbc3iHTGPi7hRr29hpsHnN7eq5PZ2ovkeQ77VoOS/Y+7udtj1rY9rCNbbfwmUy9nRfP53fe//Ns3lflojYyr12Y/Xxmkt/YVMOVunvxi5m9fc5sdnC2H9MX298S52HDJcbQEtvure69LT87Fz+nG8+918+Vnjyru2+V5IeS3CLJY/e2HQCwswimAYDLghOSHF9VRyVTr96quvt8+79U1THz7M7zMwVp35q3+2y2DpY3HJqpZcO/JTkkye9uLJjDqpckeXZVXWu+WNePzbOxX5LkbjVdFO2gqjqiqn5o3vS0TGHxFavq6EwzRPfk0Ew/az8n0+zc3840Y3rD85P8XlXdqCa3qvniZd398UztHF6U5BXd/Y1tnuMNSW5VVT87zxJ/aKYA6c3brH8x86zyY+ZjPSzJcZlaOizVMmUJT6iqw+Y+yo/OFGRu9rpMx/CAqrp8VV2hqn50nuF6CXOwfH62aOcxe1qm47jSNsuTqbXEtuH1d2FPr8O/ZJqx/9T52O6Y5C4L2744yf2q6ifm1+H759tbffGxrXkW7ZWT/Ft3f62q/lOmL3MWvT7Jf8jUz/plCzNiL9VrsMRznZjkvlX1w/MXCL+dKUDesO24n7e9T1X9SE2tOn5307bfMbf6eG2mcXRIVd04Uwj6lwurvTTTF1rH5qI2HknyvCSPqapd8/i7clXdo6r29L5ZPAc3r6o7z58HV6iqn8/Us/5de9t2wZ7Ow4anze+JW2RqUfKKJbb9XKaLH24Ol5N857ydl+0/Q9+V5HJV9avz+/knM33R98q9HdD8vtlVU6/uryT5Ri763AYADhCCaQDgsuD3k7w1ydur6kuZZnDeel52vUxB2peSfCjThcU2LgD4h0keWlVfrKrf32K/L8gUCH8m08XI/u+m5b+SqY3GqZnC69/JNJP5Y5n6BT85U+uN3ZmCvI1aD573+7xcPPzayhszXfDuY5nahnx+3nbD8ZlmQr89U9B6QqYetxtenOmCe9u18Uh3fzZTb9unzMfx6CR366lX9TKukilUPS/TBQWPSHKfJbddxt8k+UCm8/jKbHHOuvuLSX4qU9D86UyzNX830wXttvPMJE+siy4kubi/j2aaJfv9223cU5/nS9M7fI/29DrM4e8DMs3e/0KSx2fhPMxfQtw3U4/gz2dqmfDYXMr/np+f57gkz5zH0uOzKUjs7q9mer1/Igsh7aV9Dfb2XHM/5MdlCo03Zo2fl+Tr8yrbjvt521/L9BqelalNx+IM4s1+af57Rqax9Pxc1KIkmcbgQZlmXb91ocZ3Z/oc+LMk52b6AuFBWX6W9EFJnp5pTH8u08VL79vdH1py+2TPn3/JFOi+L8knMn3J8dvd/c69bTu/nr+f5JS51cctc0m/kemChudW1T0WF8xtZ+6W5GczvZ+fleQB8+fj3hyeafb8uZk+987IdLFZAOAAUr3H624AALA/m3vV/kl3b9dPeb81t624IFObh7PWXQ/rVVVXzRTKX7e7P73uei4L5l9lfKi7D153LQAAl5YZ0wAAl1Fz+4NfyTQzGy5z5rYY3z9f+O5ZSd4nlAYAODCsLJiuqhdW1eeqasufqc092v64qk6vqg9W1a23Wg8AgEuaf3b/xUw9qp+75nLgu3W/TK10zsrUlufB6y0HAIBRVtbKo6pun+TLSV7S3TffYvlPZ7oC/U8n+ZEkz+7uH1lJMQAAAAAA7DdWNmN6vuDGF/awyj0zhdbd3e9NcnhVXWdV9QAAAAAAsH9YZ4/p6yU5c+H+xs/3AAAAAADYwdZ59eba4rEt+4pU1SOSPCJJDjnkkNscffTRq6wLAAAAAIDv0SmnnPL57j5iq2XrDKbPSnLUwv0jk5y91Yrd/bzMV5vftWtX7969e/XVAQAAAADwXauqM7Zbts5WHm9I8tCa/GiS87r702usBwAAAACAAVY2Y7qqXpbkx5Nco6rOSvK0JJdPku4+Icmbkvx0ktOTfDXJw1ZVCwAAAAAA+4+VBdPdfexelneSR63q+QEAAAAA2D+ts5UHAAAAAAAHIME0AAAAAABDCaYBAAAAABhKMA0AAAAAwFCCaQAAAAAAhhJMAwAAAAAwlGAaAAAAAIChBNMAAAAAAAwlmAYAAAAAYCjBNAAAAAAAQwmmAQAAAAAYSjANAAAAAMBQgmkAAAAAAIYSTAMAAAAAMJRgGgAAAACAoQTTAAAAAAAMJZgGAAAAAGAowTQAAAAAAEMJpgEAAAAAGEowDQAAAADAUIJpAAAAAACGEkwDAAAAADCUYBoAAAAAgKEE0wAAAAAADCWYBgAAAABgKME0AAAAAABDCaYBAAAAABhKMA0AAAAAwFCCaQAAAAAAhhJMAwAAAAAwlGAaAAAAAIChBNMAAAAAAAwlmAYAAAAAYCjBNAAAAAAAQwmmAQAAAAAYSjANAAAAAMBQgmkAAAAAAIYSTAMAAAAAMJRgGgAAAACAoQTTAAAAAAAMJZgGAAAAAGAowTQAAAAAAEMJpgEAXYh4DgAAG/dJREFUAAAAGEowDQAAAADAUIJpAAAAAACGEkwDAAAAADCUYBoAAAAAgKEE0wAAAAAADCWYBgAAAABgKME0AAAAAABDCaYBAAAAABhKMA0AAAAAwFCCaQAAAAAAhhJMAwAAAAAwlGAaAAAAAIChBNMAAAAAAAwlmAYAAAAAYCjBNAAAAAAAQwmmAQAAAAAYSjANAAAAAMBQgmkAAAAAAIYSTAMAAAAAMJRgGgAAAACAoQTTAAAAAAAMJZgGAAAAAGAowTQAAAAAAEMJpgEAAAAAGEowDQAAAADAUIJpAAAAAACGEkwDAAAAADCUYBoAAAAAgKEE0wAAAAAADCWYBgAAAABgKME0AAAAAABDCaYBAAAAABhKMA0AAAAAwFCCaQAAAAAAhhJMAwAAAAAwlGAaAAAAAIChBNMAAAAAAAwlmAYAAAAAYKiVBtNVdZeq+mhVnV5VT9xi+VWr6rVV9cGq+vuquvkq6wEAAAAAYP1WFkxX1UFJnpvkrkmOSXJsVR2zabUnJzmtu38oyUOTPHtV9QAAAAAAsH9Y5Yzp2yY5vbs/3t3fSPLyJPfctM4xSd6WJN39kSQ3qKprrbAmAAAAAADWbJXB9PWSnLlw/6z5sUUfSHKfJKmq2ya5fpIjN++oqh5RVburavc555yzonIBAAAAABhhlcF0bfFYb7p/fJKrVtVpSR6T5NQkF15io+7ndfeu7t51xBFH7PtKAQAAAAAY5uAV7vusJEct3D8yydmLK3T3+UkeliRVVUk+Mf8DAAAAAGCHWuWM6fcnuUlV3bCqrpDkgUnesLhCVR0+L0uShyd55xxWAwAAAACwQ61sxnR3X1hVj05yUpKDkrywuz9cVcfNy09IcrMkL6mqbyX5pyS/sKp6AAAAAADYP6yylUe6+01J3rTpsRMWbr8nyU1WWQMAAAAAAPuXVbbyAAAAAACASxBMAwAAAAAwlGAaAAAAAIChBNMAAAAAAAwlmAYAAAAAYCjBNAAAAAAAQwmmAQAAAAAYSjANAAAAAMBQgmkAAAAAAIYSTAMAAAAAMJRgGgAAAACAoQTTAAAAAAAMJZgGAAAAAGAowTQAAAAAAEMJpgEAAAAAGEowDQAAAADAUIJpAAAAAACGEkwDAAAAADCUYBoAAAAAgKEE0wAAAAAADCWYBgAAAABgKME0AAAAAABDCaYBAAAAABhKMA0AAAAAwFCCaQAAAAAAhhJMAwAAAAAwlGAaAAAAAIChBNMAAAAAAAwlmAYAAAAAYCjBNAAAAAAAQwmmAQAAAAAYSjANAAAAAMBQgmkAAAAAAIYSTAMAAAAAMJRgGgAAAACAoQTTAAAAAAAMJZgGAAAAAGAowTQAAAAAAEMJpgEAAAAAGEowDQAAAADAUIJpAAAAAACGEkwDAAAAADCUYBoAAAAAgKEE0wAAAAAADCWYBgAAAABgKME0AAAAAABDCaYBAAAAABhKMA0AAAAAwFCCaQAAAAAAhhJMAwAAAAAwlGAaAAAAAIChBNMAAAAAAAwlmAYAAAAAYCjBNAAAAAAAQwmmAQAAAAAYSjANAAAAAMBQgmkAAAAAAIYSTAMAAAAAMJRgGgAAAACAoQTTAAAAAAAMJZgGAAAAAGAowTQAAAAAAEMJpgEAAAAAGEowDQAAAADAUIJpAAAAAACGEkwDAAAAADCUYBoAAAAAgKEE0wAAAAAADCWYBgAAAABgKME0AAAAAABDCaYBAAAAABhKMA0AAAAAwFCCaQAAAAAAhhJMAwAAAAAwlGAaAAAAAIChBNMAAAAAAAwlmAYAAAAAYCjBNAAAAAAAQwmmAQAAAAAYSjANAAAAAMBQgmkAAAAAAIZaaTBdVXepqo9W1elV9cQtlh9WVW+sqg9U1Yer6mGrrAcAAAAAgPVbWTBdVQcleW6SuyY5JsmxVXXMptUeleSfuvsWSX48yR9U1RVWVRMAAAAAAOu3yhnTt01yend/vLu/keTlSe65aZ1OcmhVVZIrJ/lCkgtXWBMAAAAAAGu2ymD6eknOXLh/1vzYouckuVmSs5P8Y5LHdve3N++oqh5RVburavc555yzqnoBAAAAABhglcF0bfFYb7r/U0lOS3LdJLdM8pyqusolNup+Xnfv6u5dRxxxxL6vFAAAAACAYVYZTJ+V5KiF+0dmmhm96GFJXtOT05N8IsnRK6wJAAAAAIA1W2Uw/f4kN6mqG84XNHxgkjdsWudTSX4iSarqWklumuTjK6wJAAAAAIA1O3hVO+7uC6vq0UlOSnJQkhd294er6rh5+QlJfifJn1fVP2Zq/fGE7v78qmoCAAAAAGD9VhZMJ0l3vynJmzY9dsLC7bOT3HmVNQAAAAAAsH9ZZSsPAAAAAAC4BME0AAAAAABDCaYBAAAAABhKMA0AAAAAwFCCaQAAAAAAhhJMAwAAAAAwlGAaAAAAAIChBNMAAAAAAAwlmAYAAAAAYCjBNAAAAAAAQwmmAQAAAAAYSjANAAAAAMBQgmkAAAAAAIYSTAMAAAAAMJRgGgAAAACAoQTTAAAAAAAMJZgGAAAAAGAowTQAAAAAAEMJpgEAAAAAGEowDQAAAADAUIJpAAAAAACGEkwDAAAAADCUYBoAAAAAgKEE0wAAAAAADCWYBgAAAABgKME0AAAAAABDCaYBAAAAABhKMA0AAAAAwFCCaQAAAAAAhhJMAwAAAAAwlGAaAAAAAIChBNMAAAAAAAwlmAYAAAAAYCjBNAAAAAAAQwmmAQAAAAAYSjANAAAAAMBQgmkAAAAAAIYSTAMAAAAAMJRgGgAAAACAoQTTAAAAAAAMJZgGAAAAAGAowTQAAAAAAEMtFUxX1aur6meqSpANAAAAAMD3ZNmg+U+TPCjJv1bV8VV19AprAgAAAABgB1sqmO7ut3b3g5PcOsknk5xcVX9XVQ+rqsuvskAAAAAAAHaWpVtzVNXVk/y3JA9PcmqSZ2cKqk9eSWUAAAAAAOxIBy+zUlW9JsnRSf4iyd27+9PzoldU1e5VFQcAAAAAwM6zVDCd5Dnd/fatFnT3rn1YDwAAAAAAO9yyrTxuVlWHb9ypqqtW1S+vqCYAAAAAAHawZYPpX+zuczfudPcXk/ziakoCAAAAAGAnWzaYvlxV1cadqjooyRVWUxIAAAAAADvZsj2mT0pyYlWdkKSTHJfkzSurCgAAAACAHWvZYPoJSX4pySOTVJK3JHn+qooCAAAAAGDnWiqY7u5vJ/nT+R8AAAAAAHzXlgqmq+omSf5nkmOSXHHj8e6+0YrqAgAAAABgh1r24ocvyjRb+sIkd0zykiR/saqiAAAAAADYuZYNpr+/u9+WpLr7jO7+zSR3Wl1ZAAAAAADsVMte/PBrVXW5JP9aVY9O8v+SXHN1ZQEAAAAAsFMtO2P6V5NcKcmvJLlNkock+blVFQUAAAAAwM611xnTVXVQkvt39+OSfDnJw1ZeFQAAAAAAO9ZeZ0x397eS3KaqakA9AAAAAADscMv2mD41yeur6pVJvrLxYHe/ZiVVAQAAAACwYy0bTF8tyb8ludPCY51EMA0AAAAAwKWyVDDd3fpKAwAAAACwTywVTFfVizLNkL6Y7v75fV4RAAAAAAA72rKtPP564fYVk9w7ydn7vhwAAAAAAHa6ZVt5vHrxflW9LMlbV1IRAAAAAAA72uW+y+1ukuQH9mUhAAAAAAAcGJbtMf2lXLzH9GeSPGElFQEAAAAAsKMt28rj0FUXAgAAAADAgWGpVh5Vde+qOmzh/uFVda/VlQUAAAAAwE61bI/pp3X3eRt3uvvcJE9bTUkAAAAAAOxkywbTW623VBsQAAAAAABYtGwwvbuqnlVVN66qG1XVHyY5ZZWFAQAAAACwMy0bTD8myTeSvCLJiUkuSPKoVRUFAAAAAMDOtVQ7ju7+SpInrrgWAAAAAAAOAEvNmK6qk6vq8IX7V62qk1ZXFgAAAAAAO9WyrTyu0d3nbtzp7i8mueZqSgIAAAAAYCdbNpj+dlX9wMadqrpBkl5FQQAAAAAA7GxL9ZhO8pQk/7eq3jHfv32SR6ymJAAAAAAAdrJlL3745qralSmMPi3J65NcsMrCAAAAAADYmZYKpqvq4Ukem+TITMH0jyZ5T5I77WW7uyR5dpKDkjy/u4/ftPxxSR68UMvNkhzR3V+4FMcAAAAAAMBlyLI9ph+b5IeTnNHdd0xyqyTn7GmDqjooyXOT3DXJMUmOrapjFtfp7md09y27+5ZJnpTkHUJpAAAAAICdbdlg+mvd/bUkqarv6+6PJLnpXra5bZLTu/vj3f2NJC9Pcs89rH9skpctWQ8AAAAAAJdRywbTZ1XV4Ulel+Tkqnp9krP3ss31kpy5uI/5sUuoqisluUuSV2+z/BFVtbuqdp9zzh4nagMAAAAAsJ9b9uKH955v/mZV/e8khyV58142q612tc26d0/y7u3aeHT385I8L0l27dq13T4AAAAAALgMWCqYXtTd71hy1bOSHLVw/8hsP8v6gdHGAwAAAADggLBsK4/vxvuT3KSqblhVV8gUPr9h80pVdViSOyR5/QprAQAAAABgP3GpZ0wvq7svrKpHJzkpyUFJXtjdH66q4+blJ8yr3jvJW7r7K6uqBQAAAACA/Ud1X7ZaNu/atat379697jIAAAAAANiDqjqlu3dttWyVrTwAAAAAAOASBNMAAAAAAAwlmAYAAAAAYCjBNAAAAAAAQwmmAQAAAAAYSjANAAAAAMBQgmkAAAAAAIYSTAMAAAAAMJRgGgAAAACAoQTTAAAAAAAMJZgGAAAAAGAowTQAAAAAAEMJpgEAAAAAGEowDQAAAADAUIJpAAAAAACGEkwDAAAAADCUYBoAAAAAgKEE0wAAAAAADCWYBgAAAABgKME0AAAAAABDCaYBAAAAABhKMA0AAAAAwFCCaQAAAAAAhhJMAwAAAAAwlGAaAAAAAIChBNMAAAAAAAwlmAYAAAAAYCjBNAAAAAAAQwmmAQAAAAAYSjANAAAAAMBQgmkAAAAAAIYSTAMAAAAAMJRgGgAAAACAoQTTAAAAAAAMJZgGAAAAAGAowTQAAAAAAEMJpgEAAAAAGEowDQAAAADAUIJpAAAAAACGEkwDAAAAADCUYBoAAPj/7d1fiGaHWcfx3+OuFZRoWrqp0k3aIqkXVhvssCJFrYW20YbEf4X4BwsVQ2oL2gs1RS14J8YLRSoh1ErBP0EhsWtr/4QWrYiBndW1Jk3TrqUmS8SkvVBblRDyeLFvcJzMRIU5z5nMfD4Q3ve85+yb5+aB5DtnzwAAwChhGgAAAACAUcI0AAAAAACjhGkAAAAAAEYJ0wAAAAAAjBKmAQAAAAAYJUwDAAAAADBKmAYAAAAAYJQwDQAAAADAKGEaAAAAAIBRwjQAAAAAAKOEaQAAAAAARgnTAAAAAACMEqYBAAAAABglTAMAAAAAMEqYBgAAAABglDANAAAAAMAoYRoAAAAAgFHCNAAAAAAAo4RpAAAAAABGCdMAAAAAAIwSpgEAAAAAGCVMAwAAAAAwSpgGAAAAAGCUMA0AAAAAwChhGgAAAACAUcI0AAAAAACjhGkAAAAAAEYJ0wAAAAAAjBKmAQAAAAAYJUwDAAAAADBKmAYAAAAAYJQwDQAAAADAKGEaAAAAAIBRwjQAAAAAAKOEaQAAAAAARgnTAAAAAACMEqYBAAAAABglTAMAAAAAMEqYBgAAAABglDANAAAAAMCoRcN0VV1fVQ9V1cWqum2fa15TVReq6oGq+osl5wEAAAAAYH0nl/riqjqR5N1JXpfkUpJzVXW2uz+145ork/x2kuu7++GqumqpeQAAAAAAOByWvGP6TJKL3f257n4iyV1Jbtp1zY8mubu7H06S7n5swXkAAAAAADgElgzTL07yyI7jS5vPdnp5kudX1Z9X1fmq+om9vqiqbqmq7arafvzxxxcaFwAAAACACUuG6drjs951fDLJq5K8MckbkvxyVb38GX+o+87u3ururVOnTh38pAAAAAAAjFnsGdO5fIf01TuOTyd5dI9rvtDdX07y5ar6RJJXJvnMgnMBAAAAALCiJe+YPpfk2qp6WVU9L8nNSc7uuub9Sb6zqk5W1Vcn+fYkDy44EwAAAAAAK1vsjunufrKq3p7kI0lOJHlvdz9QVbduzt/R3Q9W1YeTfDLJU0ne0933LzUTAAAAAADrq+7dj30+3La2tnp7e3vtMQAAAAAAeBZVdb67t/Y6t+SjPAAAAAAA4BmEaQAAAAAARgnTAAAAAACMEqYBAAAAABglTAMAAAAAMEqYBgAAAABglDANAAAAAMAoYRoAAAAAgFHCNAAAAAAAo4RpAAAAAABGCdMAAAAAAIwSpgEAAAAAGCVMAwAAAAAwSpgGAAAAAGCUMA0AAAAAwChhGgAAAACAUcI0AAAAAACjhGkAAAAAAEYJ0wAAAAAAjBKmAQAAAAAYJUwDAAAAADBKmAYAAAAAYJQwDQAAAADAKGEaAAAAAIBRwjQAAAAAAKOEaQAAAAAARgnTAAAAAACMEqYBAAAAABglTAMAAAAAMEqYBgAAAABglDANAAAAAMAoYRoAAAAAgFHCNAAAAAAAo4RpAAAAAABGCdMAAAAAAIwSpgEAAAAAGCVMAwAAAAAwSpgGAAAAAGCUMA0AAAAAwChhGgAAAACAUcI0AAAAAACjhGkAAAAAAEYJ0wAAAAAAjBKmAQAAAAAYJUwDAAAAADBKmAYAAAAAYJQwDQAAAADAKGEaAAAAAIBRwjQAAAAAAKOEaQAAAAAARgnTAAAAAACMEqYBAAAAABglTAMAAAAAMEqYBgAAAABglDANAAAAAMAoYRoAAAAAgFHCNAAAAAAAo4RpAAAAAABGCdMAAAAAAIwSpgEAAAAAGCVMAwAAAAAwSpgGAAAAAGCUMA0AAAAAwChhGgAAAACAUcI0AAAAAACjhGkAAAAAAEYJ0wAAAAAAjBKmAQAAAAAYJUwDAAAAADBKmAYAAAAAYJQwDQAAAADAKGEaAAAAAIBRwjQAAAAAAKOEaQAAAAAARgnTAAAAAACMEqYBAAAAABglTAMAAAAAMEqYBgAAAABglDANAAAAAMAoYRoAAAAAgFGLhumqur6qHqqqi1V12x7nX1NV/1JVFzb/vGvJeQAAAAAAWN/Jpb64qk4keXeS1yW5lORcVZ3t7k/tuvQvu/uGpeYAAAAAAOBwWfKO6TNJLnb357r7iSR3JblpwX8fAAAAAADPAUuG6RcneWTH8aXNZ7t9R1X9XVV9qKq+ecF5AAAAAAA4BBZ7lEeS2uOz3nX8N0le0t1fqqrvS/InSa59xhdV3ZLkliS55pprDnpOAAAAAAAGLXnH9KUkV+84Pp3k0Z0XdPe/dveXNu//LMlXVtULd39Rd9/Z3VvdvXXq1KkFRwYAAAAAYGlLhulzSa6tqpdV1fOS3Jzk7M4Lqurrq6o2789s5vnigjMBAAAAALCyxR7l0d1PVtXbk3wkyYkk7+3uB6rq1s35O5L8cJK3VtWTSf4jyc3dvftxHwAAAAAAHCH1XOvAW1tbvb29vfYYAAAAAAA8i6o6391be51b8lEeAAAAAADwDMI0AAAAAACjhGkAAAAAAEYJ0wAAAAAAjBKmAQAAAAAYJUwDAAAAADBKmAYAAAAAYJQwDQAAAADAKGEaAAAAAIBRwjQAAAAAAKOEaQAAAAAARgnTAAAAAACMEqYBAAAAABglTAMAAAAAMEqYBgAAAABglDANAAAAAMAoYRoAAAAAgFHCNAAAAAAAo4RpAAAAAABGCdMAAAAAAIwSpgEAAAAAGCVMAwAAAAAwSpgGAAAAAGCUMA0AAAAAwChhGgAAAACAUcI0AAAAAACjhGkAAAAAAEYJ0wAAAAAAjBKmAQAAAAAYJUwDAAAAADBKmAYAAAAAYJQwDQAAAADAKGEaAAAAAIBRwjQAAAAAAKOEaQAAAAAARgnTAAAAAACMEqYBAAAAABglTAMAAAAAMEqYBgAAAABglDANAAAAAMCo6u61Z/h/qarHk/zj2nPAsBcm+cLaQwCj7D0cP/Yejhc7D8ePvec4ekl3n9rrxHMuTMNxVFXb3b219hzAHHsPx4+9h+PFzsPxY+/hf/IoDwAAAAAARgnTAAAAAACMEqbhueHOtQcAxtl7OH7sPRwvdh6OH3sPO3jGNAAAAAAAo9wxDQAAAADAKGEaDqGqOlFVf1tVH9gcX1dV91XVhararqoza88IHKw99v6VVfXXVfX3VfWnVfW1a88IHJyq+vxmvy9U1fbmsxdU1b1V9dnN6/PXnhM4OPvs/Zuq6oGqeqqqttaeETg4++z87VX16ar6ZFXdU1VXrj0nrEmYhsPpZ5I8uOP415L8Sndfl+Rdm2PgaNm99+9Jclt3f0uSe5L83CpTAUv6nu6+rrufjlG3JflYd1+b5GObY+Bo2b339yf5wSSfWHEmYDm7d/7eJK/o7m9N8pkk71xvNFifMA2HTFWdTvLGXI5ST+skT98t+XVJHp2eC1jOPnv/Tfnv/0m9N8kPTc8FjLspyfs279+X5PtXnAUY0N0PdvdDa88BzOjuj3b3k5vD+5KcXnMeWJswDYfPbyT5+SRP7fjsZ5PcXlWPJPn1+KkqHDV77f39SW7cvH9TkqunhwIW1Uk+WlXnq+qWzWcv6u5/SpLN61WrTQcsYa+9B46u/23n35LkQ8MzwaEiTMMhUlU3JHmsu8/vOvXWJO/o7quTvCPJ74wPByziWfb+LUneVlXnk1yR5Inx4YAlvbq7vy3J9+byrn/X2gMBi7P3cLzsu/NV9YtJnkzy+2sNB4eBMA2Hy6uT3FhVn09yV5LXVtXvJXlzkrs31/xxEr/8EI6OPfe+uz/d3a/v7lcl+cMk/7DmkMDB6u5HN6+P5fJz5M8k+eeq+oYk2bw+tt6EwEHbZ++BI2q/na+qNye5IcmPdXevNyGsT5iGQ6S739ndp7v7pUluTvLx7v7xXH6m9HdvLnttks+uNCJwwPbb+6q6Kkmq6iuS/FKSO1YcEzhAVfU1VXXF0++TvD6XH99zNpd/GJ3N6/vXmRA4aM+y98ARtN/OV9X1SX4hyY3d/e9rzgiHwcm1BwD+T34qyW9W1ckk/5nEM+ng6PuRqnrb5v3dSX53zWGAA/WiJPdUVXL5v8f/oLs/XFXnkvxRVf1kkodz+fnywNGw397/QJLfSnIqyQer6kJ3v2HFOYGDsd/OX0zyVUnu3Zy7r7tvXW9MWFf5WwMAAAAAAEzyKA8AAAAAAEYJ0wAAAAAAjBKmAQAAAAAYJUwDAAAAADBKmAYAAAAAYJQwDQAAK6qqK6vqp9eeAwAAJgnTAACwriuTCNMAABwrwjQAAKzrV5N8Y1VdqKrb1x4GAAAmVHevPQMAABxbVfXSJB/o7lesPAoAAIxxxzQAAAAAAKOEaQAAAAAARgnTAACwrn9LcsXaQwAAwCRhGgAAVtTdX0zyV1V1v19+CADAceGXHwIAAAAAMMod0wAAAAAAjBKmAQAAAAAYJUwDAAAAADBKmAYAAAAAYJQwDQAAAADAKGEaAAAAAIBRwjQAAAAAAKOEaQAAAAAARv0X34Qo/GkLJvIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1800x1440 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# one plot each\n",
    "fig, ax = plt.subplots(len(modelnames), len(directions), squeeze=False, figsize=(25,20))\n",
    "for i, model in enumerate(modelnames):\n",
    "    for j, direction in enumerate(directions):\n",
    "        avg_test_acc = np.mean(test_accs[(model, direction)], axis = 1)\n",
    "        ax[i,j].plot(ts, avg_test_acc)\n",
    "        ax[i,j].set_ylim([.45, 1])\n",
    "        ax[i,j].set_title('Test accuracy of ' + model + ' model ' + (\"(Bidirectional)\" if direction else \"\") + ' averaged over ' + str(num_reps) + ' repetitions')\n",
    "        ax[i,j].set_xlabel(\"t\")\n",
    "        ax[i,j].set_ylabel(\"accuracy\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig.savefig('failing.pdf') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}