03.Family_Classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os,sys\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras\n",
    "from tensorflow.keras.datasets import *\n",
    "from tensorflow.keras.models import *\n",
    "from tensorflow.keras.layers import *\n",
    "from tensorflow.keras.preprocessing.text import *\n",
    "from tqdm import tqdm\n",
    "from collections import Counter\n",
    "import pandas as pd\n",
    "import shutil\n",
    "import pickle\n",
    "import gc, math\n",
    "from math import log, floor\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.metrics.pairwise import *\n",
    "# os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\n",
    "# The GPU id to use, usually either \"0\" or \"1\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_words = 332 #dict size (tri-gram總數量)\n",
    "max_length = 106 #第一轉折點 tri-gram num (最靠近outlier數字的斜率變化轉折點) 超過此數量的hkl拿掉\n",
    "min_length = 33 #最小的轉折點(第三個轉折點) #Q1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14/14 [00:00<00:00, 28.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2416 2416\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "root_dir = \"./data/Hooklog/trace_picked_fam/\"\n",
    "fam_dirs = next(os.walk(root_dir))[1]\n",
    "trainX = []\n",
    "trainY = []\n",
    "for fam in tqdm(fam_dirs):\n",
    "    in_directory = root_dir + fam + '/'\n",
    "    hl_list = next(os.walk(in_directory))[2] # get all filenames in the in_directory\n",
    "    hl_list = [os.path.join(in_directory, f) for f in hl_list] # filepathname list\n",
    "    #hl_list = list(filter(lambda f: f.endswith(\".enc.npy\"), hl_list)) # in case some non-hooklog file in the folder\n",
    "    hl_list = list(filter(lambda f: f.endswith(\".norm.npy\"), hl_list)) #load norm data\n",
    "    for npy in hl_list:\n",
    "        tri_enc = np.load(npy)\n",
    "        if len(tri_enc)>max_length:\n",
    "            tri_enc = tri_enc[:max_length]\n",
    "        if len(tri_enc)<min_length:\n",
    "            continue\n",
    "        trainY.append(fam)\n",
    "        trainX.append(tri_enc)\n",
    "print(len(trainY),len(trainX))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0]], dtype=int32)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 8,  8,  8, ..., 10, 10, 10])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fam_code = {}\n",
    "for i,v in enumerate(list(set(trainY))):\n",
    "    fam_code[v] = i\n",
    "# fam_code = pickle.load(file=open(root_dir+\"fam_dict_enc.pkl\",'rb'))\n",
    "train_Y = []\n",
    "for fam in trainY:\n",
    "    train_Y.append(fam_code[fam])\n",
    "train_Y = np.array(train_Y)\n",
    "pickle.dump(obj=fam_code,file=open(root_dir+\"fam_dict_enc.pkl\",'wb'))\n",
    "train_Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainX = [list(x) for x in trainX]\n",
    "# trainX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2416, 106) (2416,)\n"
     ]
    }
   ],
   "source": [
    "train_X = tf.keras.preprocessing.sequence.pad_sequences(trainX,maxlen=max_length,padding='post',value=0,dtype='float32')\n",
    "print(train_X.shape, train_Y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_valid_set(X_all, Y_all, percentage):\n",
    "    all_data_size = len(X_all)\n",
    "    valid_data_size = int(floor(all_data_size * percentage))\n",
    "\n",
    "    X_all, Y_all = _shuffle(X_all, Y_all)\n",
    "\n",
    "    X_train, Y_train = X_all[0:valid_data_size], Y_all[0:valid_data_size]\n",
    "    X_valid, Y_valid = X_all[valid_data_size:], Y_all[valid_data_size:]\n",
    "\n",
    "    return X_train, Y_train, X_valid, Y_valid\n",
    "def _shuffle(X, Y):\n",
    "    randomize = np.arange(len(X))\n",
    "    np.random.shuffle(randomize)\n",
    "#     print(X.shape, Y.shape)\n",
    "    return (X[randomize], Y[randomize])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2174, 106) (2174,) (242, 106) (242,)\n",
      "{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}\n"
     ]
    }
   ],
   "source": [
    "X_train, y_train, X_test, y_test = split_valid_set(train_X, train_Y, 0.9)\n",
    "print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)\n",
    "assert len(set(y_test)) == len(set(y_train))\n",
    "print(set(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_X = np.expand_dims(train_X,axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2174, 106, 1) (242, 106, 1)\n"
     ]
    }
   ],
   "source": [
    "X_train = np.expand_dims(X_train,axis=-1)\n",
    "X_test = np.expand_dims(X_test,axis=-1)\n",
    "print(X_train.shape,X_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**for Mike ML data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 242/242 [00:00<00:00, 2667.16it/s]\n",
      "100%|██████████| 2174/2174 [00:00<00:00, 3331.94it/s]\n"
     ]
    }
   ],
   "source": [
    "def convert_onehot(X_train=X_train):\n",
    "    \"\"\"\n",
    "    input: padded numpy array encode by int\n",
    "    output: one hot list(samples) of list(tri-gram terms)\n",
    "    \"\"\"\n",
    "    X_train_li = []\n",
    "    for arr_npy in tqdm(X_train):\n",
    "        arr_li = []\n",
    "        for element in arr_npy:\n",
    "            init_npy = np.zeros((max_words,))\n",
    "            one_hot = init_npy\n",
    "            if element ==0:\n",
    "                arr_li.append(init_npy)\n",
    "                continue\n",
    "            one_hot[element] = 1\n",
    "            arr_li.append(one_hot)\n",
    "        X_train_li.append(arr_li)\n",
    "    return X_train_li\n",
    "X_test_li = convert_onehot(X_test)\n",
    "X_train_li = convert_onehot(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "242it [00:02, 80.99it/s]\n",
      "2174it [00:35, 60.72it/s]\n"
     ]
    }
   ],
   "source": [
    "def conver2df(X_test_li=X_test_li, y_test=y_test):\n",
    "    \"\"\"\n",
    "    Input: output of convert_onehot function &　it;s corresponding label\n",
    "    Output: df\n",
    "    \"\"\"\n",
    "    col = ['word_'+str(i) for i in range(max_length)]\n",
    "    col.append('LABEL')\n",
    "    df = pd.DataFrame(columns=col)\n",
    "    for one_hot_li, label in tqdm(zip(X_test_li, y_test)):\n",
    "        temp = [list(element) for element in one_hot_li]\n",
    "        temp.extend([label])\n",
    "        temp = pd.Series(temp,col)\n",
    "        df = df.append(temp,ignore_index=True)\n",
    "    return df\n",
    "test_df = conver2df(X_test_li,y_test)\n",
    "train_df = conver2df(X_train_li,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word_0</th>\n",
       "      <th>word_1</th>\n",
       "      <th>word_2</th>\n",
       "      <th>word_3</th>\n",
       "      <th>word_4</th>\n",
       "      <th>word_5</th>\n",
       "      <th>word_6</th>\n",
       "      <th>word_7</th>\n",
       "      <th>word_8</th>\n",
       "      <th>word_9</th>\n",
       "      <th>...</th>\n",
       "      <th>word_97</th>\n",
       "      <th>word_98</th>\n",
       "      <th>word_99</th>\n",
       "      <th>word_100</th>\n",
       "      <th>word_101</th>\n",
       "      <th>word_102</th>\n",
       "      <th>word_103</th>\n",
       "      <th>word_104</th>\n",
       "      <th>word_105</th>\n",
       "      <th>LABEL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...</td>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, ...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238</th>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...</td>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>239</th>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...</td>\n",
       "      <td>[0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>242 rows × 107 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                word_0  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "4    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "239  [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                                word_1  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "4    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "239  [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                                word_2  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "4    [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "239  [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                                word_3  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...   \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "4    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...   \n",
       "239  [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                                word_4  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...   \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "4    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "239  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                                word_5  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...   \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "4    [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...   \n",
       "239  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...   \n",
       "\n",
       "                                                word_6  \\\n",
       "0    [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...   \n",
       "4    [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "239  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                                word_7  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "4    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "239  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                                word_8  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...   \n",
       "4    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...   \n",
       "239  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                                word_9  ...  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  ...   \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, ...  ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  ...   \n",
       "4    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  ...   \n",
       "..                                                 ...  ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...  ...   \n",
       "238  [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...  ...   \n",
       "239  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...  ...   \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  ...   \n",
       "\n",
       "                                               word_97  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...   \n",
       "4    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...   \n",
       "239  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                               word_98  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, ...   \n",
       "4    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...   \n",
       "239  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                               word_99  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...   \n",
       "4    [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...   \n",
       "239  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                              word_100  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...   \n",
       "4    [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "239  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                              word_101  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1    [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...   \n",
       "4    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...   \n",
       "239  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                              word_102  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...   \n",
       "4    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "239  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                              word_103  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1    [0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...   \n",
       "4    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...   \n",
       "239  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                              word_104  \\\n",
       "0    [0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...   \n",
       "4    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "..                                                 ...   \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "238  [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, ...   \n",
       "239  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                              word_105 LABEL  \n",
       "0    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...     2  \n",
       "1    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...     3  \n",
       "2    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...     3  \n",
       "3    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, ...     4  \n",
       "4    [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...     5  \n",
       "..                                                 ...   ...  \n",
       "237  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...     0  \n",
       "238  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...     2  \n",
       "239  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...     3  \n",
       "240  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...     9  \n",
       "241  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...     3  \n",
       "\n",
       "[242 rows x 107 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.to_csv(\"./data/Hooklog/trace_picked_fam/Bi-testing_onehot.csv\",index=False)\n",
    "train_df.to_csv(\"./data/Hooklog/trace_picked_fam/Bi-training_onehot.csv\",index=False)\n",
    "test_df = pd.read_csv(\"./data/Hooklog/trace_picked_fam/Bi-testing_onehot.csv\")\n",
    "test_df #check saving"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "hyper-parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_length = train_X.shape[-1]\n",
    "cat_num = train_Y.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_dim = 128#validXr.shape[-1] \n",
    "max_length = X_test.shape[-1] # max sequence length\n",
    "cat_num = y_test.shape[0]\n",
    "do = 0.1\n",
    "vocabulary_size = max_words+1\n",
    "hidden_dim = 64\n",
    "EPOCHS=2000\n",
    "# traina = True #改\n",
    "batch_size = 128 #改"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# init_w = tensorflow.keras.initializers.Constant(value=0.9) #portyion=0.6, w=0.9, b = 0.8-0.85 (0.83從0開始)\n",
    "init_w = tensorflow.keras.initializers.RandomNormal(mean=0.9,stddev=0.06)\n",
    "# init_b = tensorflow.keras.initializers.Constant(value=0.84) #w=1 ; b=0.499, portion=1\n",
    "init_b = tensorflow.keras.initializers.RandomNormal(mean=0.7,stddev=0.2)\n",
    "def onezero(x):\n",
    "    portion = 1#0.6#0.6 #0.6~1\n",
    "    z = tf.where(x>=1.0, x - x + 1.0, x)\n",
    "    y = tf.where(z<=0.0, z - z + 0.0, portion*z)\n",
    "    return y\n",
    "\n",
    "train_ds = tf.data.Dataset.from_tensor_slices((X_train,y_train)).shuffle(X_train.shape[0],reshuffle_each_iteration=True).batch(batch_size)\n",
    "valid_ds = tf.data.Dataset.from_tensor_slices((X_test,y_test)).batch(batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Total Loss: 2.6321074962615967, Accuracy Rate:  9.85%,            Test_Total_Loss: 2.6222646236419678, Test_Accuracy_Rate: 10.80%\n",
      "Epoch 2, Total Loss: 2.620380401611328, Accuracy Rate: 12.21%,            Test_Total_Loss: 2.6071951389312744, Test_Accuracy_Rate: 12.04%\n",
      "Epoch 3, Total Loss: 2.6324777603149414, Accuracy Rate: 11.59%,            Test_Total_Loss: 2.614319324493408, Test_Accuracy_Rate: 11.18%\n",
      "Epoch 4, Total Loss: 2.605168104171753, Accuracy Rate: 14.07%,            Test_Total_Loss: 2.597059726715088, Test_Accuracy_Rate: 13.66%\n",
      "Epoch 5, Total Loss: 2.592358350753784, Accuracy Rate: 13.25%,            Test_Total_Loss: 2.5813372135162354, Test_Accuracy_Rate: 13.49%\n",
      "Epoch 6, Total Loss: 2.5672852993011475, Accuracy Rate: 15.81%,            Test_Total_Loss: 2.542290687561035, Test_Accuracy_Rate: 14.53%\n",
      "Epoch 7, Total Loss: 2.5320024490356445, Accuracy Rate: 14.49%,            Test_Total_Loss: 2.5214712619781494, Test_Accuracy_Rate: 14.32%\n",
      "Epoch 8, Total Loss: 2.5178937911987305, Accuracy Rate: 14.82%,            Test_Total_Loss: 2.5096235275268555, Test_Accuracy_Rate: 16.06%\n",
      "Epoch 9, Total Loss: 2.5134143829345703, Accuracy Rate: 15.65%,            Test_Total_Loss: 2.4987778663635254, Test_Accuracy_Rate: 15.19%\n",
      "Epoch 10, Total Loss: 2.5013701915740967, Accuracy Rate: 16.10%,            Test_Total_Loss: 2.4906933307647705, Test_Accuracy_Rate: 17.14%\n",
      "Epoch 11, Total Loss: 2.492358684539795, Accuracy Rate: 17.92%,            Test_Total_Loss: 2.4809834957122803, Test_Accuracy_Rate: 18.83%\n",
      "Epoch 12, Total Loss: 2.482325553894043, Accuracy Rate: 18.75%,            Test_Total_Loss: 2.4695141315460205, Test_Accuracy_Rate: 18.96%\n",
      "Epoch 13, Total Loss: 2.4677398204803467, Accuracy Rate: 18.96%,            Test_Total_Loss: 2.469226598739624, Test_Accuracy_Rate: 17.14%\n",
      "Epoch 14, Total Loss: 2.4480156898498535, Accuracy Rate: 18.63%,            Test_Total_Loss: 2.439602851867676, Test_Accuracy_Rate: 19.50%\n",
      "Epoch 15, Total Loss: 2.434391975402832, Accuracy Rate: 18.67%,            Test_Total_Loss: 2.4274024963378906, Test_Accuracy_Rate: 19.29%\n",
      "Epoch 16, Total Loss: 2.425117254257202, Accuracy Rate: 19.16%,            Test_Total_Loss: 2.416712760925293, Test_Accuracy_Rate: 19.95%\n",
      "Epoch 17, Total Loss: 2.4163191318511963, Accuracy Rate: 19.78%,            Test_Total_Loss: 2.4049484729766846, Test_Accuracy_Rate: 19.37%\n",
      "Epoch 18, Total Loss: 2.4075510501861572, Accuracy Rate: 19.45%,            Test_Total_Loss: 2.415950059890747, Test_Accuracy_Rate: 19.12%\n",
      "Epoch 19, Total Loss: 2.3971428871154785, Accuracy Rate: 20.86%,            Test_Total_Loss: 2.3789048194885254, Test_Accuracy_Rate: 20.78%\n",
      "Epoch 20, Total Loss: 2.389824390411377, Accuracy Rate: 20.74%,            Test_Total_Loss: 2.3767006397247314, Test_Accuracy_Rate: 20.65%\n",
      "Epoch 21, Total Loss: 2.38183331489563, Accuracy Rate: 20.61%,            Test_Total_Loss: 2.373878240585327, Test_Accuracy_Rate: 22.76%\n",
      "Epoch 22, Total Loss: 2.36971378326416, Accuracy Rate: 22.10%,            Test_Total_Loss: 2.35567307472229, Test_Accuracy_Rate: 22.06%\n",
      "Epoch 23, Total Loss: 2.357774019241333, Accuracy Rate: 22.97%,            Test_Total_Loss: 2.344102621078491, Test_Accuracy_Rate: 24.92%\n",
      "Epoch 24, Total Loss: 2.3492259979248047, Accuracy Rate: 24.42%,            Test_Total_Loss: 2.33015775680542, Test_Accuracy_Rate: 26.12%\n",
      "Epoch 25, Total Loss: 2.3451931476593018, Accuracy Rate: 24.63%,            Test_Total_Loss: 2.3609912395477295, Test_Accuracy_Rate: 25.62%\n",
      "Epoch 26, Total Loss: 2.3315601348876953, Accuracy Rate: 24.34%,            Test_Total_Loss: 2.3047847747802734, Test_Accuracy_Rate: 26.78%\n",
      "Epoch 27, Total Loss: 2.3165111541748047, Accuracy Rate: 25.25%,            Test_Total_Loss: 2.3367502689361572, Test_Accuracy_Rate: 25.54%\n",
      "Epoch 28, Total Loss: 2.3137974739074707, Accuracy Rate: 25.91%,            Test_Total_Loss: 2.286869764328003, Test_Accuracy_Rate: 27.65%\n",
      "Epoch 29, Total Loss: 2.308590888977051, Accuracy Rate: 25.54%,            Test_Total_Loss: 2.344247341156006, Test_Accuracy_Rate: 23.97%\n",
      "Epoch 30, Total Loss: 2.2992827892303467, Accuracy Rate: 25.79%,            Test_Total_Loss: 2.2802374362945557, Test_Accuracy_Rate: 26.03%\n",
      "Epoch 31, Total Loss: 2.288262128829956, Accuracy Rate: 26.45%,            Test_Total_Loss: 2.2736127376556396, Test_Accuracy_Rate: 27.65%\n",
      "Epoch 32, Total Loss: 2.2779526710510254, Accuracy Rate: 26.95%,            Test_Total_Loss: 2.2800710201263428, Test_Accuracy_Rate: 26.37%\n",
      "Epoch 33, Total Loss: 2.275047779083252, Accuracy Rate: 26.49%,            Test_Total_Loss: 2.2506377696990967, Test_Accuracy_Rate: 27.48%\n",
      "Epoch 34, Total Loss: 2.2579550743103027, Accuracy Rate: 28.10%,            Test_Total_Loss: 2.233520030975342, Test_Accuracy_Rate: 29.01%\n",
      "Epoch 35, Total Loss: 2.250443458557129, Accuracy Rate: 27.81%,            Test_Total_Loss: 2.2277023792266846, Test_Accuracy_Rate: 28.44%\n",
      "Epoch 36, Total Loss: 2.266047477722168, Accuracy Rate: 26.66%,            Test_Total_Loss: 2.3054990768432617, Test_Accuracy_Rate: 25.83%\n",
      "Epoch 37, Total Loss: 2.2484018802642822, Accuracy Rate: 28.10%,            Test_Total_Loss: 2.2272918224334717, Test_Accuracy_Rate: 28.85%\n",
      "Epoch 38, Total Loss: 2.240781784057617, Accuracy Rate: 28.27%,            Test_Total_Loss: 2.2155442237854004, Test_Accuracy_Rate: 28.68%\n",
      "Epoch 39, Total Loss: 2.22580885887146, Accuracy Rate: 27.98%,            Test_Total_Loss: 2.2030327320098877, Test_Accuracy_Rate: 29.64%\n",
      "Epoch 40, Total Loss: 2.2149763107299805, Accuracy Rate: 28.89%,            Test_Total_Loss: 2.202183246612549, Test_Accuracy_Rate: 28.52%\n",
      "Epoch 41, Total Loss: 2.216496229171753, Accuracy Rate: 28.48%,            Test_Total_Loss: 2.2000317573547363, Test_Accuracy_Rate: 29.72%\n",
      "Epoch 42, Total Loss: 2.211146593093872, Accuracy Rate: 29.06%,            Test_Total_Loss: 2.1901628971099854, Test_Accuracy_Rate: 31.04%\n",
      "Epoch 43, Total Loss: 2.196192741394043, Accuracy Rate: 29.55%,            Test_Total_Loss: 2.1851301193237305, Test_Accuracy_Rate: 30.01%\n",
      "Epoch 44, Total Loss: 2.1950032711029053, Accuracy Rate: 29.35%,            Test_Total_Loss: 2.2447595596313477, Test_Accuracy_Rate: 27.94%\n",
      "Epoch 45, Total Loss: 2.1854734420776367, Accuracy Rate: 30.34%,            Test_Total_Loss: 2.16355037689209, Test_Accuracy_Rate: 30.71%\n",
      "Epoch 46, Total Loss: 2.193732500076294, Accuracy Rate: 29.93%,            Test_Total_Loss: 2.2694947719573975, Test_Accuracy_Rate: 28.23%\n",
      "Epoch 47, Total Loss: 2.179858922958374, Accuracy Rate: 29.43%,            Test_Total_Loss: 2.179950475692749, Test_Accuracy_Rate: 29.35%\n",
      "Epoch 48, Total Loss: 2.1599626541137695, Accuracy Rate: 30.71%,            Test_Total_Loss: 2.1479973793029785, Test_Accuracy_Rate: 30.88%\n",
      "Epoch 49, Total Loss: 2.169358491897583, Accuracy Rate: 30.09%,            Test_Total_Loss: 2.1463916301727295, Test_Accuracy_Rate: 31.13%\n",
      "Epoch 50, Total Loss: 2.158830404281616, Accuracy Rate: 30.63%,            Test_Total_Loss: 2.1293442249298096, Test_Accuracy_Rate: 31.62%\n",
      "Epoch 51, Total Loss: 2.157139539718628, Accuracy Rate: 30.34%,            Test_Total_Loss: 2.148094892501831, Test_Accuracy_Rate: 30.42%\n",
      "Epoch 52, Total Loss: 2.1471107006073, Accuracy Rate: 30.38%,            Test_Total_Loss: 2.113766670227051, Test_Accuracy_Rate: 31.79%\n",
      "Epoch 53, Total Loss: 2.1311185359954834, Accuracy Rate: 31.29%,            Test_Total_Loss: 2.1224524974823, Test_Accuracy_Rate: 32.04%\n",
      "Epoch 54, Total Loss: 2.133551597595215, Accuracy Rate: 31.83%,            Test_Total_Loss: 2.1415011882781982, Test_Accuracy_Rate: 32.12%\n",
      "Epoch 55, Total Loss: 2.1244630813598633, Accuracy Rate: 31.83%,            Test_Total_Loss: 2.1085379123687744, Test_Accuracy_Rate: 31.33%\n",
      "Epoch 56, Total Loss: 2.110858678817749, Accuracy Rate: 32.00%,            Test_Total_Loss: 2.111231803894043, Test_Accuracy_Rate: 32.41%\n",
      "Epoch 57, Total Loss: 2.119804620742798, Accuracy Rate: 32.12%,            Test_Total_Loss: 2.0902538299560547, Test_Accuracy_Rate: 33.44%\n",
      "Epoch 58, Total Loss: 2.105496883392334, Accuracy Rate: 32.70%,            Test_Total_Loss: 2.145188093185425, Test_Accuracy_Rate: 30.59%\n",
      "Epoch 59, Total Loss: 2.094348192214966, Accuracy Rate: 32.95%,            Test_Total_Loss: 2.0897884368896484, Test_Accuracy_Rate: 32.95%\n",
      "Epoch 60, Total Loss: 2.094513177871704, Accuracy Rate: 32.86%,            Test_Total_Loss: 2.086782932281494, Test_Accuracy_Rate: 31.95%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 61, Total Loss: 2.075007438659668, Accuracy Rate: 33.20%,            Test_Total_Loss: 2.058689594268799, Test_Accuracy_Rate: 35.35%\n",
      "Epoch 62, Total Loss: 2.067067861557007, Accuracy Rate: 33.73%,            Test_Total_Loss: 2.064682722091675, Test_Accuracy_Rate: 33.77%\n",
      "Epoch 63, Total Loss: 2.062241315841675, Accuracy Rate: 33.44%,            Test_Total_Loss: 2.053793430328369, Test_Accuracy_Rate: 33.73%\n",
      "Epoch 64, Total Loss: 2.0644454956054688, Accuracy Rate: 33.03%,            Test_Total_Loss: 2.1141021251678467, Test_Accuracy_Rate: 33.49%\n",
      "Epoch 65, Total Loss: 2.061877965927124, Accuracy Rate: 34.40%,            Test_Total_Loss: 2.0341010093688965, Test_Accuracy_Rate: 34.40%\n",
      "Epoch 66, Total Loss: 2.0397214889526367, Accuracy Rate: 34.06%,            Test_Total_Loss: 2.0387449264526367, Test_Accuracy_Rate: 34.85%\n",
      "Epoch 67, Total Loss: 2.0322370529174805, Accuracy Rate: 34.98%,            Test_Total_Loss: 2.059960126876831, Test_Accuracy_Rate: 34.69%\n",
      "Epoch 68, Total Loss: 2.0312721729278564, Accuracy Rate: 34.81%,            Test_Total_Loss: 2.0226991176605225, Test_Accuracy_Rate: 35.60%\n",
      "Epoch 69, Total Loss: 2.019404888153076, Accuracy Rate: 35.22%,            Test_Total_Loss: 1.9863064289093018, Test_Accuracy_Rate: 37.04%\n",
      "Epoch 70, Total Loss: 2.0209128856658936, Accuracy Rate: 35.64%,            Test_Total_Loss: 2.0968518257141113, Test_Accuracy_Rate: 33.03%\n",
      "Epoch 71, Total Loss: 2.028271436691284, Accuracy Rate: 34.69%,            Test_Total_Loss: 1.9884697198867798, Test_Accuracy_Rate: 37.09%\n",
      "Epoch 72, Total Loss: 2.010620594024658, Accuracy Rate: 35.51%,            Test_Total_Loss: 1.9769495725631714, Test_Accuracy_Rate: 37.46%\n",
      "Epoch 73, Total Loss: 2.0040442943573, Accuracy Rate: 36.30%,            Test_Total_Loss: 1.9828147888183594, Test_Accuracy_Rate: 36.30%\n",
      "Epoch 74, Total Loss: 2.0043540000915527, Accuracy Rate: 35.97%,            Test_Total_Loss: 1.9748303890228271, Test_Accuracy_Rate: 36.13%\n",
      "Epoch 75, Total Loss: 1.9915988445281982, Accuracy Rate: 35.76%,            Test_Total_Loss: 1.9522185325622559, Test_Accuracy_Rate: 38.70%\n",
      "Epoch 76, Total Loss: 1.9727813005447388, Accuracy Rate: 36.80%,            Test_Total_Loss: 1.9457690715789795, Test_Accuracy_Rate: 38.04%\n",
      "Epoch 77, Total Loss: 1.9701863527297974, Accuracy Rate: 36.22%,            Test_Total_Loss: 1.9337698221206665, Test_Accuracy_Rate: 38.49%\n",
      "Epoch 78, Total Loss: 1.97185480594635, Accuracy Rate: 37.25%,            Test_Total_Loss: 1.9454357624053955, Test_Accuracy_Rate: 38.20%\n",
      "Epoch 79, Total Loss: 1.9588936567306519, Accuracy Rate: 37.96%,            Test_Total_Loss: 1.9534668922424316, Test_Accuracy_Rate: 38.41%\n",
      "Epoch 80, Total Loss: 1.9504667520523071, Accuracy Rate: 37.17%,            Test_Total_Loss: 1.9781566858291626, Test_Accuracy_Rate: 35.68%\n",
      "Epoch 81, Total Loss: 1.9490506649017334, Accuracy Rate: 37.04%,            Test_Total_Loss: 2.1334686279296875, Test_Accuracy_Rate: 31.71%\n",
      "Epoch 82, Total Loss: 1.9714150428771973, Accuracy Rate: 36.92%,            Test_Total_Loss: 1.9191477298736572, Test_Accuracy_Rate: 39.20%\n",
      "Epoch 83, Total Loss: 1.9316000938415527, Accuracy Rate: 38.87%,            Test_Total_Loss: 2.0152549743652344, Test_Accuracy_Rate: 36.18%\n",
      "Epoch 84, Total Loss: 1.9359151124954224, Accuracy Rate: 37.87%,            Test_Total_Loss: 1.9323636293411255, Test_Accuracy_Rate: 38.00%\n",
      "Epoch 85, Total Loss: 1.9225847721099854, Accuracy Rate: 38.37%,            Test_Total_Loss: 1.8831006288528442, Test_Accuracy_Rate: 38.53%\n",
      "Epoch 86, Total Loss: 1.924178123474121, Accuracy Rate: 38.04%,            Test_Total_Loss: 1.8935295343399048, Test_Accuracy_Rate: 38.66%\n",
      "Epoch 87, Total Loss: 1.9195841550827026, Accuracy Rate: 38.00%,            Test_Total_Loss: 1.894606351852417, Test_Accuracy_Rate: 40.27%\n",
      "Epoch 88, Total Loss: 1.9174567461013794, Accuracy Rate: 38.78%,            Test_Total_Loss: 1.886088490486145, Test_Accuracy_Rate: 39.61%\n",
      "Epoch 89, Total Loss: 1.9140498638153076, Accuracy Rate: 39.11%,            Test_Total_Loss: 1.862537145614624, Test_Accuracy_Rate: 40.73%\n",
      "Epoch 90, Total Loss: 1.9117481708526611, Accuracy Rate: 39.36%,            Test_Total_Loss: 1.8983893394470215, Test_Accuracy_Rate: 39.98%\n",
      "Epoch 91, Total Loss: 1.8783279657363892, Accuracy Rate: 39.86%,            Test_Total_Loss: 1.858215570449829, Test_Accuracy_Rate: 39.40%\n",
      "Epoch 92, Total Loss: 1.894771933555603, Accuracy Rate: 39.36%,            Test_Total_Loss: 1.8694196939468384, Test_Accuracy_Rate: 40.60%\n",
      "Epoch 93, Total Loss: 1.8912327289581299, Accuracy Rate: 39.57%,            Test_Total_Loss: 1.8403431177139282, Test_Accuracy_Rate: 41.02%\n",
      "Epoch 94, Total Loss: 1.8751778602600098, Accuracy Rate: 40.73%,            Test_Total_Loss: 1.8417812585830688, Test_Accuracy_Rate: 40.85%\n",
      "Epoch 95, Total Loss: 1.870522141456604, Accuracy Rate: 40.23%,            Test_Total_Loss: 1.8334310054779053, Test_Accuracy_Rate: 40.60%\n",
      "Epoch 96, Total Loss: 1.8599721193313599, Accuracy Rate: 40.40%,            Test_Total_Loss: 1.8183614015579224, Test_Accuracy_Rate: 42.05%\n",
      "Epoch 97, Total Loss: 1.8537591695785522, Accuracy Rate: 40.23%,            Test_Total_Loss: 1.8245183229446411, Test_Accuracy_Rate: 41.18%\n",
      "Epoch 98, Total Loss: 1.8386768102645874, Accuracy Rate: 40.94%,            Test_Total_Loss: 1.8192068338394165, Test_Accuracy_Rate: 41.02%\n",
      "Epoch 99, Total Loss: 1.8482164144515991, Accuracy Rate: 39.74%,            Test_Total_Loss: 1.8016204833984375, Test_Accuracy_Rate: 41.56%\n",
      "Epoch 100, Total Loss: 1.844152569770813, Accuracy Rate: 40.60%,            Test_Total_Loss: 1.7875152826309204, Test_Accuracy_Rate: 42.26%\n",
      "Epoch 101, Total Loss: 1.8147225379943848, Accuracy Rate: 41.23%,            Test_Total_Loss: 1.8224745988845825, Test_Accuracy_Rate: 41.93%\n",
      "Epoch 102, Total Loss: 1.8334907293319702, Accuracy Rate: 41.80%,            Test_Total_Loss: 1.7838650941848755, Test_Accuracy_Rate: 42.67%\n",
      "Epoch 103, Total Loss: 1.8232933282852173, Accuracy Rate: 41.85%,            Test_Total_Loss: 1.8282101154327393, Test_Accuracy_Rate: 40.89%\n",
      "Epoch 104, Total Loss: 1.8391225337982178, Accuracy Rate: 40.81%,            Test_Total_Loss: 1.777063012123108, Test_Accuracy_Rate: 42.88%\n",
      "Epoch 105, Total Loss: 1.810410976409912, Accuracy Rate: 41.97%,            Test_Total_Loss: 1.8758043050765991, Test_Accuracy_Rate: 40.23%\n",
      "Epoch 106, Total Loss: 1.8044642210006714, Accuracy Rate: 42.59%,            Test_Total_Loss: 1.7564116716384888, Test_Accuracy_Rate: 43.42%\n",
      "Epoch 107, Total Loss: 1.7958446741104126, Accuracy Rate: 42.96%,            Test_Total_Loss: 1.7713783979415894, Test_Accuracy_Rate: 43.71%\n",
      "Epoch 108, Total Loss: 1.7887920141220093, Accuracy Rate: 43.05%,            Test_Total_Loss: 1.7817003726959229, Test_Accuracy_Rate: 43.71%\n",
      "Epoch 109, Total Loss: 1.7771450281143188, Accuracy Rate: 42.22%,            Test_Total_Loss: 1.8220255374908447, Test_Accuracy_Rate: 41.23%\n",
      "Epoch 110, Total Loss: 1.7977116107940674, Accuracy Rate: 41.68%,            Test_Total_Loss: 1.7870410680770874, Test_Accuracy_Rate: 43.92%\n",
      "Epoch 111, Total Loss: 1.7899563312530518, Accuracy Rate: 42.09%,            Test_Total_Loss: 1.7357902526855469, Test_Accuracy_Rate: 44.00%\n",
      "Epoch 112, Total Loss: 1.7556384801864624, Accuracy Rate: 43.50%,            Test_Total_Loss: 1.7164697647094727, Test_Accuracy_Rate: 44.62%\n",
      "Epoch 113, Total Loss: 1.7641299962997437, Accuracy Rate: 43.83%,            Test_Total_Loss: 1.7101894617080688, Test_Accuracy_Rate: 45.49%\n",
      "Epoch 114, Total Loss: 1.747668981552124, Accuracy Rate: 43.75%,            Test_Total_Loss: 1.717207431793213, Test_Accuracy_Rate: 44.62%\n",
      "Epoch 115, Total Loss: 1.7507233619689941, Accuracy Rate: 43.46%,            Test_Total_Loss: 1.6913291215896606, Test_Accuracy_Rate: 45.86%\n",
      "Epoch 116, Total Loss: 1.7408678531646729, Accuracy Rate: 43.75%,            Test_Total_Loss: 1.7399418354034424, Test_Accuracy_Rate: 43.54%\n",
      "Epoch 117, Total Loss: 1.7221899032592773, Accuracy Rate: 44.95%,            Test_Total_Loss: 1.7541507482528687, Test_Accuracy_Rate: 44.12%\n",
      "Epoch 118, Total Loss: 1.7363632917404175, Accuracy Rate: 44.41%,            Test_Total_Loss: 1.6832348108291626, Test_Accuracy_Rate: 45.94%\n",
      "Epoch 119, Total Loss: 1.7118054628372192, Accuracy Rate: 45.45%,            Test_Total_Loss: 1.6857331991195679, Test_Accuracy_Rate: 45.61%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 120, Total Loss: 1.7107726335525513, Accuracy Rate: 44.74%,            Test_Total_Loss: 1.6713680028915405, Test_Accuracy_Rate: 46.65%\n",
      "Epoch 121, Total Loss: 1.718066930770874, Accuracy Rate: 44.33%,            Test_Total_Loss: 1.6641285419464111, Test_Accuracy_Rate: 46.27%\n",
      "Epoch 122, Total Loss: 1.696630597114563, Accuracy Rate: 46.23%,            Test_Total_Loss: 1.67356538772583, Test_Accuracy_Rate: 45.74%\n",
      "Epoch 123, Total Loss: 1.6792237758636475, Accuracy Rate: 45.65%,            Test_Total_Loss: 1.6663469076156616, Test_Accuracy_Rate: 46.07%\n",
      "Epoch 124, Total Loss: 1.6972556114196777, Accuracy Rate: 44.78%,            Test_Total_Loss: 1.680889368057251, Test_Accuracy_Rate: 46.15%\n",
      "Epoch 125, Total Loss: 1.696319818496704, Accuracy Rate: 44.87%,            Test_Total_Loss: 1.6602802276611328, Test_Accuracy_Rate: 46.23%\n",
      "Epoch 126, Total Loss: 1.7153617143630981, Accuracy Rate: 44.16%,            Test_Total_Loss: 1.6510720252990723, Test_Accuracy_Rate: 46.61%\n",
      "Epoch 127, Total Loss: 1.6534874439239502, Accuracy Rate: 46.32%,            Test_Total_Loss: 1.6170231103897095, Test_Accuracy_Rate: 47.27%\n",
      "Epoch 128, Total Loss: 1.6744850873947144, Accuracy Rate: 45.82%,            Test_Total_Loss: 1.7384824752807617, Test_Accuracy_Rate: 44.78%\n",
      "Epoch 129, Total Loss: 1.666870355606079, Accuracy Rate: 46.73%,            Test_Total_Loss: 1.602324366569519, Test_Accuracy_Rate: 48.18%\n",
      "Epoch 130, Total Loss: 1.6450436115264893, Accuracy Rate: 45.78%,            Test_Total_Loss: 1.6072518825531006, Test_Accuracy_Rate: 48.22%\n",
      "Epoch 131, Total Loss: 1.6623190641403198, Accuracy Rate: 46.15%,            Test_Total_Loss: 1.6018762588500977, Test_Accuracy_Rate: 48.22%\n",
      "Epoch 132, Total Loss: 1.625930905342102, Accuracy Rate: 47.60%,            Test_Total_Loss: 1.617124319076538, Test_Accuracy_Rate: 48.10%\n",
      "Epoch 133, Total Loss: 1.6177998781204224, Accuracy Rate: 47.97%,            Test_Total_Loss: 1.5833349227905273, Test_Accuracy_Rate: 49.01%\n",
      "Epoch 134, Total Loss: 1.6246341466903687, Accuracy Rate: 47.85%,            Test_Total_Loss: 1.5784218311309814, Test_Accuracy_Rate: 48.84%\n",
      "Epoch 135, Total Loss: 1.6179859638214111, Accuracy Rate: 47.76%,            Test_Total_Loss: 1.6854583024978638, Test_Accuracy_Rate: 45.99%\n",
      "Epoch 136, Total Loss: 1.6075196266174316, Accuracy Rate: 47.89%,            Test_Total_Loss: 1.5967246294021606, Test_Accuracy_Rate: 49.34%\n",
      "Epoch 137, Total Loss: 1.5926440954208374, Accuracy Rate: 48.26%,            Test_Total_Loss: 1.581591010093689, Test_Accuracy_Rate: 49.67%\n",
      "Epoch 138, Total Loss: 1.6266347169876099, Accuracy Rate: 47.76%,            Test_Total_Loss: 1.5500375032424927, Test_Accuracy_Rate: 50.17%\n",
      "Epoch 139, Total Loss: 1.6565334796905518, Accuracy Rate: 46.40%,            Test_Total_Loss: 1.5429856777191162, Test_Accuracy_Rate: 50.21%\n",
      "Epoch 140, Total Loss: 1.5918958187103271, Accuracy Rate: 48.59%,            Test_Total_Loss: 1.5224586725234985, Test_Accuracy_Rate: 50.50%\n",
      "Epoch 141, Total Loss: 1.5621412992477417, Accuracy Rate: 49.54%,            Test_Total_Loss: 1.5229476690292358, Test_Accuracy_Rate: 51.41%\n",
      "Epoch 142, Total Loss: 1.5683925151824951, Accuracy Rate: 48.68%,            Test_Total_Loss: 1.5819400548934937, Test_Accuracy_Rate: 49.17%\n",
      "Epoch 143, Total Loss: 1.608743667602539, Accuracy Rate: 48.14%,            Test_Total_Loss: 1.5329972505569458, Test_Accuracy_Rate: 50.33%\n",
      "Epoch 144, Total Loss: 1.5825058221817017, Accuracy Rate: 48.84%,            Test_Total_Loss: 1.5532773733139038, Test_Accuracy_Rate: 49.17%\n",
      "Epoch 145, Total Loss: 1.5550694465637207, Accuracy Rate: 49.34%,            Test_Total_Loss: 1.5491597652435303, Test_Accuracy_Rate: 50.00%\n",
      "Epoch 146, Total Loss: 1.5907747745513916, Accuracy Rate: 48.05%,            Test_Total_Loss: 1.5228948593139648, Test_Accuracy_Rate: 50.95%\n",
      "Epoch 147, Total Loss: 1.5352153778076172, Accuracy Rate: 49.88%,            Test_Total_Loss: 1.5034314393997192, Test_Accuracy_Rate: 51.95%\n",
      "Epoch 148, Total Loss: 1.5544475317001343, Accuracy Rate: 49.59%,            Test_Total_Loss: 1.5375354290008545, Test_Accuracy_Rate: 50.46%\n",
      "Epoch 149, Total Loss: 1.547555923461914, Accuracy Rate: 50.12%,            Test_Total_Loss: 1.5493189096450806, Test_Accuracy_Rate: 50.04%\n",
      "Epoch 150, Total Loss: 1.5325465202331543, Accuracy Rate: 50.33%,            Test_Total_Loss: 1.4844367504119873, Test_Accuracy_Rate: 51.82%\n",
      "Epoch 151, Total Loss: 1.524682641029358, Accuracy Rate: 50.25%,            Test_Total_Loss: 1.4927263259887695, Test_Accuracy_Rate: 50.54%\n",
      "Epoch 152, Total Loss: 1.5112440586090088, Accuracy Rate: 49.83%,            Test_Total_Loss: 1.4739971160888672, Test_Accuracy_Rate: 52.90%\n",
      "Epoch 153, Total Loss: 1.5058284997940063, Accuracy Rate: 51.70%,            Test_Total_Loss: 1.4552932977676392, Test_Accuracy_Rate: 53.19%\n",
      "Epoch 154, Total Loss: 1.5030778646469116, Accuracy Rate: 51.90%,            Test_Total_Loss: 1.5253276824951172, Test_Accuracy_Rate: 50.37%\n",
      "Epoch 155, Total Loss: 1.5416755676269531, Accuracy Rate: 50.62%,            Test_Total_Loss: 1.4556480646133423, Test_Accuracy_Rate: 53.06%\n",
      "Epoch 156, Total Loss: 1.5098778009414673, Accuracy Rate: 50.41%,            Test_Total_Loss: 1.4778242111206055, Test_Accuracy_Rate: 52.48%\n",
      "Epoch 157, Total Loss: 1.4897910356521606, Accuracy Rate: 51.90%,            Test_Total_Loss: 1.5517739057540894, Test_Accuracy_Rate: 50.17%\n",
      "Epoch 158, Total Loss: 1.4791514873504639, Accuracy Rate: 51.86%,            Test_Total_Loss: 1.425764560699463, Test_Accuracy_Rate: 54.26%\n",
      "Epoch 159, Total Loss: 1.461382508277893, Accuracy Rate: 53.27%,            Test_Total_Loss: 1.4358017444610596, Test_Accuracy_Rate: 53.93%\n",
      "Epoch 160, Total Loss: 1.4802031517028809, Accuracy Rate: 51.12%,            Test_Total_Loss: 1.4616857767105103, Test_Accuracy_Rate: 52.40%\n",
      "Epoch 161, Total Loss: 1.5002772808074951, Accuracy Rate: 51.20%,            Test_Total_Loss: 1.4471864700317383, Test_Accuracy_Rate: 53.27%\n",
      "Epoch 162, Total Loss: 1.4641376733779907, Accuracy Rate: 52.07%,            Test_Total_Loss: 1.391858696937561, Test_Accuracy_Rate: 54.88%\n",
      "Epoch 163, Total Loss: 1.445436716079712, Accuracy Rate: 53.10%,            Test_Total_Loss: 1.4171717166900635, Test_Accuracy_Rate: 54.26%\n",
      "Epoch 164, Total Loss: 1.4596019983291626, Accuracy Rate: 53.27%,            Test_Total_Loss: 1.4993722438812256, Test_Accuracy_Rate: 51.28%\n",
      "Epoch 165, Total Loss: 1.4477202892303467, Accuracy Rate: 52.69%,            Test_Total_Loss: 1.4747346639633179, Test_Accuracy_Rate: 51.74%\n",
      "Epoch 166, Total Loss: 1.488633155822754, Accuracy Rate: 50.70%,            Test_Total_Loss: 1.416377305984497, Test_Accuracy_Rate: 53.81%\n",
      "Epoch 167, Total Loss: 1.4662504196166992, Accuracy Rate: 53.10%,            Test_Total_Loss: 1.4036521911621094, Test_Accuracy_Rate: 53.39%\n",
      "Epoch 168, Total Loss: 1.4388259649276733, Accuracy Rate: 53.68%,            Test_Total_Loss: 1.3930717706680298, Test_Accuracy_Rate: 54.55%\n",
      "Epoch 169, Total Loss: 1.4314717054367065, Accuracy Rate: 53.64%,            Test_Total_Loss: 1.3936926126480103, Test_Accuracy_Rate: 54.88%\n",
      "Epoch 170, Total Loss: 1.4287952184677124, Accuracy Rate: 53.35%,            Test_Total_Loss: 1.3857150077819824, Test_Accuracy_Rate: 55.22%\n",
      "Epoch 171, Total Loss: 1.4086041450500488, Accuracy Rate: 53.85%,            Test_Total_Loss: 1.3589640855789185, Test_Accuracy_Rate: 56.37%\n",
      "Epoch 172, Total Loss: 1.4024888277053833, Accuracy Rate: 54.06%,            Test_Total_Loss: 1.3766181468963623, Test_Accuracy_Rate: 55.42%\n",
      "Epoch 173, Total Loss: 1.4105560779571533, Accuracy Rate: 53.68%,            Test_Total_Loss: 1.4246994256973267, Test_Accuracy_Rate: 53.77%\n",
      "Epoch 174, Total Loss: 1.4155136346817017, Accuracy Rate: 53.27%,            Test_Total_Loss: 1.35813307762146, Test_Accuracy_Rate: 56.08%\n",
      "Epoch 175, Total Loss: 1.3861474990844727, Accuracy Rate: 54.39%,            Test_Total_Loss: 1.4274311065673828, Test_Accuracy_Rate: 53.73%\n",
      "Epoch 176, Total Loss: 1.4231857061386108, Accuracy Rate: 53.68%,            Test_Total_Loss: 1.399856448173523, Test_Accuracy_Rate: 54.22%\n",
      "Epoch 177, Total Loss: 1.4061561822891235, Accuracy Rate: 53.23%,            Test_Total_Loss: 1.3327178955078125, Test_Accuracy_Rate: 56.71%\n",
      "Epoch 178, Total Loss: 1.3681787252426147, Accuracy Rate: 55.50%,            Test_Total_Loss: 1.3197120428085327, Test_Accuracy_Rate: 57.57%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 179, Total Loss: 1.3857065439224243, Accuracy Rate: 54.01%,            Test_Total_Loss: 1.3944543600082397, Test_Accuracy_Rate: 53.85%\n",
      "Epoch 180, Total Loss: 1.3886293172836304, Accuracy Rate: 54.10%,            Test_Total_Loss: 1.3517884016036987, Test_Accuracy_Rate: 55.92%\n",
      "Epoch 181, Total Loss: 1.3982362747192383, Accuracy Rate: 54.64%,            Test_Total_Loss: 1.3426121473312378, Test_Accuracy_Rate: 56.04%\n",
      "Epoch 182, Total Loss: 1.3617717027664185, Accuracy Rate: 55.79%,            Test_Total_Loss: 1.3402072191238403, Test_Accuracy_Rate: 56.71%\n",
      "Epoch 183, Total Loss: 1.3769100904464722, Accuracy Rate: 54.84%,            Test_Total_Loss: 1.6021952629089355, Test_Accuracy_Rate: 48.51%\n",
      "Epoch 184, Total Loss: 1.4388985633850098, Accuracy Rate: 52.73%,            Test_Total_Loss: 1.3242002725601196, Test_Accuracy_Rate: 56.37%\n",
      "Epoch 185, Total Loss: 1.358546257019043, Accuracy Rate: 54.59%,            Test_Total_Loss: 1.3128184080123901, Test_Accuracy_Rate: 56.37%\n",
      "Epoch 186, Total Loss: 1.3549582958221436, Accuracy Rate: 55.38%,            Test_Total_Loss: 1.3622969388961792, Test_Accuracy_Rate: 54.76%\n",
      "Epoch 187, Total Loss: 1.3468923568725586, Accuracy Rate: 55.96%,            Test_Total_Loss: 1.276759147644043, Test_Accuracy_Rate: 58.20%\n",
      "Epoch 188, Total Loss: 1.3300050497055054, Accuracy Rate: 55.50%,            Test_Total_Loss: 1.2884334325790405, Test_Accuracy_Rate: 57.41%\n",
      "Epoch 189, Total Loss: 1.3346343040466309, Accuracy Rate: 55.71%,            Test_Total_Loss: 1.3020732402801514, Test_Accuracy_Rate: 57.16%\n",
      "Epoch 190, Total Loss: 1.3823909759521484, Accuracy Rate: 54.55%,            Test_Total_Loss: 1.3088774681091309, Test_Accuracy_Rate: 56.58%\n",
      "Epoch 191, Total Loss: 1.3394595384597778, Accuracy Rate: 54.88%,            Test_Total_Loss: 1.280442714691162, Test_Accuracy_Rate: 57.53%\n",
      "Epoch 192, Total Loss: 1.3102006912231445, Accuracy Rate: 56.66%,            Test_Total_Loss: 1.2588958740234375, Test_Accuracy_Rate: 58.36%\n",
      "Epoch 193, Total Loss: 1.3077741861343384, Accuracy Rate: 55.92%,            Test_Total_Loss: 1.2546558380126953, Test_Accuracy_Rate: 58.65%\n",
      "Epoch 194, Total Loss: 1.329779028892517, Accuracy Rate: 55.59%,            Test_Total_Loss: 1.2883706092834473, Test_Accuracy_Rate: 57.95%\n",
      "Epoch 195, Total Loss: 1.3070127964019775, Accuracy Rate: 56.50%,            Test_Total_Loss: 1.2705037593841553, Test_Accuracy_Rate: 58.57%\n",
      "Epoch 196, Total Loss: 1.295363187789917, Accuracy Rate: 56.33%,            Test_Total_Loss: 1.2756903171539307, Test_Accuracy_Rate: 58.07%\n",
      "Epoch 197, Total Loss: 1.2945467233657837, Accuracy Rate: 57.49%,            Test_Total_Loss: 1.2726374864578247, Test_Accuracy_Rate: 57.62%\n",
      "Epoch 198, Total Loss: 1.3012069463729858, Accuracy Rate: 56.29%,            Test_Total_Loss: 1.2601044178009033, Test_Accuracy_Rate: 58.07%\n",
      "Epoch 199, Total Loss: 1.2823363542556763, Accuracy Rate: 56.87%,            Test_Total_Loss: 1.265683889389038, Test_Accuracy_Rate: 57.62%\n",
      "Epoch 200, Total Loss: 1.2853574752807617, Accuracy Rate: 57.12%,            Test_Total_Loss: 1.2312616109848022, Test_Accuracy_Rate: 59.48%\n",
      "Epoch 201, Total Loss: 1.278060793876648, Accuracy Rate: 57.95%,            Test_Total_Loss: 1.2232187986373901, Test_Accuracy_Rate: 59.23%\n",
      "Epoch 202, Total Loss: 1.2711589336395264, Accuracy Rate: 57.74%,            Test_Total_Loss: 1.2191388607025146, Test_Accuracy_Rate: 59.60%\n",
      "Epoch 203, Total Loss: 1.265723466873169, Accuracy Rate: 58.44%,            Test_Total_Loss: 1.233615517616272, Test_Accuracy_Rate: 58.86%\n",
      "Epoch 204, Total Loss: 1.2748171091079712, Accuracy Rate: 58.03%,            Test_Total_Loss: 1.2161777019500732, Test_Accuracy_Rate: 59.48%\n",
      "Epoch 205, Total Loss: 1.2679325342178345, Accuracy Rate: 57.95%,            Test_Total_Loss: 1.2411680221557617, Test_Accuracy_Rate: 59.31%\n",
      "Epoch 206, Total Loss: 1.2752206325531006, Accuracy Rate: 57.99%,            Test_Total_Loss: 1.2281919717788696, Test_Accuracy_Rate: 59.31%\n",
      "Epoch 207, Total Loss: 1.2595434188842773, Accuracy Rate: 57.95%,            Test_Total_Loss: 1.2387349605560303, Test_Accuracy_Rate: 58.73%\n",
      "Epoch 208, Total Loss: 1.2556288242340088, Accuracy Rate: 58.11%,            Test_Total_Loss: 1.2053974866867065, Test_Accuracy_Rate: 59.56%\n",
      "Epoch 209, Total Loss: 1.2570420503616333, Accuracy Rate: 58.36%,            Test_Total_Loss: 1.2095139026641846, Test_Accuracy_Rate: 60.18%\n",
      "Epoch 210, Total Loss: 1.2612987756729126, Accuracy Rate: 58.20%,            Test_Total_Loss: 1.2005172967910767, Test_Accuracy_Rate: 60.43%\n",
      "Epoch 211, Total Loss: 1.2466129064559937, Accuracy Rate: 59.02%,            Test_Total_Loss: 1.2238212823867798, Test_Accuracy_Rate: 59.02%\n",
      "Epoch 212, Total Loss: 1.2419261932373047, Accuracy Rate: 58.07%,            Test_Total_Loss: 1.1768306493759155, Test_Accuracy_Rate: 60.76%\n",
      "Epoch 213, Total Loss: 1.2302427291870117, Accuracy Rate: 58.69%,            Test_Total_Loss: 1.1838947534561157, Test_Accuracy_Rate: 60.26%\n",
      "Epoch 214, Total Loss: 1.2727106809616089, Accuracy Rate: 57.33%,            Test_Total_Loss: 1.2111178636550903, Test_Accuracy_Rate: 59.52%\n",
      "Epoch 215, Total Loss: 1.2231117486953735, Accuracy Rate: 58.94%,            Test_Total_Loss: 1.1866241693496704, Test_Accuracy_Rate: 60.43%\n",
      "Epoch 216, Total Loss: 1.2204618453979492, Accuracy Rate: 59.35%,            Test_Total_Loss: 1.2396694421768188, Test_Accuracy_Rate: 58.53%\n",
      "Epoch 217, Total Loss: 1.2540104389190674, Accuracy Rate: 56.83%,            Test_Total_Loss: 1.2450698614120483, Test_Accuracy_Rate: 57.82%\n",
      "Epoch 218, Total Loss: 1.2570180892944336, Accuracy Rate: 57.66%,            Test_Total_Loss: 1.1768903732299805, Test_Accuracy_Rate: 60.35%\n",
      "Epoch 219, Total Loss: 1.2111146450042725, Accuracy Rate: 59.23%,            Test_Total_Loss: 1.1856461763381958, Test_Accuracy_Rate: 60.18%\n",
      "Epoch 220, Total Loss: 1.2233338356018066, Accuracy Rate: 58.53%,            Test_Total_Loss: 1.1617604494094849, Test_Accuracy_Rate: 60.93%\n",
      "Epoch 221, Total Loss: 1.2420594692230225, Accuracy Rate: 57.57%,            Test_Total_Loss: 1.1874220371246338, Test_Accuracy_Rate: 60.51%\n",
      "Epoch 222, Total Loss: 1.2178916931152344, Accuracy Rate: 59.11%,            Test_Total_Loss: 1.1562421321868896, Test_Accuracy_Rate: 61.13%\n",
      "Epoch 223, Total Loss: 1.2047119140625, Accuracy Rate: 59.64%,            Test_Total_Loss: 1.1907237768173218, Test_Accuracy_Rate: 60.10%\n",
      "Epoch 224, Total Loss: 1.2102974653244019, Accuracy Rate: 59.15%,            Test_Total_Loss: 1.177605152130127, Test_Accuracy_Rate: 60.55%\n",
      "Epoch 225, Total Loss: 1.215346336364746, Accuracy Rate: 59.11%,            Test_Total_Loss: 1.2291268110275269, Test_Accuracy_Rate: 58.69%\n",
      "Epoch 226, Total Loss: 1.2416192293167114, Accuracy Rate: 58.15%,            Test_Total_Loss: 1.1573768854141235, Test_Accuracy_Rate: 61.13%\n",
      "Epoch 227, Total Loss: 1.2056928873062134, Accuracy Rate: 58.86%,            Test_Total_Loss: 1.1429306268692017, Test_Accuracy_Rate: 61.34%\n",
      "Epoch 228, Total Loss: 1.1947723627090454, Accuracy Rate: 60.10%,            Test_Total_Loss: 1.1331881284713745, Test_Accuracy_Rate: 61.80%\n",
      "Epoch 229, Total Loss: 1.1911283731460571, Accuracy Rate: 59.11%,            Test_Total_Loss: 1.1610901355743408, Test_Accuracy_Rate: 60.80%\n",
      "Epoch 230, Total Loss: 1.188489556312561, Accuracy Rate: 59.06%,            Test_Total_Loss: 1.1444480419158936, Test_Accuracy_Rate: 61.34%\n",
      "Epoch 231, Total Loss: 1.1922173500061035, Accuracy Rate: 59.35%,            Test_Total_Loss: 1.1439708471298218, Test_Accuracy_Rate: 61.26%\n",
      "Epoch 232, Total Loss: 1.1799681186676025, Accuracy Rate: 59.44%,            Test_Total_Loss: 1.1399775743484497, Test_Accuracy_Rate: 61.67%\n",
      "Epoch 233, Total Loss: 1.2017186880111694, Accuracy Rate: 59.56%,            Test_Total_Loss: 1.1284517049789429, Test_Accuracy_Rate: 62.25%\n",
      "Epoch 234, Total Loss: 1.182120680809021, Accuracy Rate: 59.69%,            Test_Total_Loss: 1.14157235622406, Test_Accuracy_Rate: 61.47%\n",
      "Epoch 235, Total Loss: 1.1803733110427856, Accuracy Rate: 59.93%,            Test_Total_Loss: 1.1381100416183472, Test_Accuracy_Rate: 61.30%\n",
      "Epoch 236, Total Loss: 1.349676489830017, Accuracy Rate: 56.37%,            Test_Total_Loss: 1.1948646306991577, Test_Accuracy_Rate: 60.39%\n",
      "Epoch 237, Total Loss: 1.1942249536514282, Accuracy Rate: 59.98%,            Test_Total_Loss: 1.1279147863388062, Test_Accuracy_Rate: 61.88%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 238, Total Loss: 1.1771929264068604, Accuracy Rate: 60.43%,            Test_Total_Loss: 1.1216963529586792, Test_Accuracy_Rate: 61.88%\n",
      "Epoch 239, Total Loss: 1.1879984140396118, Accuracy Rate: 59.35%,            Test_Total_Loss: 1.1325260400772095, Test_Accuracy_Rate: 61.67%\n",
      "Epoch 240, Total Loss: 1.168059229850769, Accuracy Rate: 60.39%,            Test_Total_Loss: 1.1001497507095337, Test_Accuracy_Rate: 62.62%\n",
      "Epoch 241, Total Loss: 1.1540008783340454, Accuracy Rate: 60.64%,            Test_Total_Loss: 1.1116737127304077, Test_Accuracy_Rate: 62.17%\n",
      "Epoch 242, Total Loss: 1.1602498292922974, Accuracy Rate: 60.10%,            Test_Total_Loss: 1.1033653020858765, Test_Accuracy_Rate: 61.92%\n",
      "Epoch 243, Total Loss: 1.1621646881103516, Accuracy Rate: 60.51%,            Test_Total_Loss: 1.1333531141281128, Test_Accuracy_Rate: 61.55%\n",
      "Epoch 244, Total Loss: 1.2504947185516357, Accuracy Rate: 57.82%,            Test_Total_Loss: 1.1314278841018677, Test_Accuracy_Rate: 62.09%\n",
      "Epoch 245, Total Loss: 1.1821357011795044, Accuracy Rate: 59.69%,            Test_Total_Loss: 1.1187748908996582, Test_Accuracy_Rate: 61.71%\n",
      "Epoch 246, Total Loss: 1.1658823490142822, Accuracy Rate: 60.10%,            Test_Total_Loss: 1.1387476921081543, Test_Accuracy_Rate: 61.13%\n",
      "Epoch 247, Total Loss: 1.159926414489746, Accuracy Rate: 60.02%,            Test_Total_Loss: 1.0966516733169556, Test_Accuracy_Rate: 62.38%\n",
      "Epoch 248, Total Loss: 1.1435811519622803, Accuracy Rate: 60.43%,            Test_Total_Loss: 1.1194438934326172, Test_Accuracy_Rate: 62.04%\n",
      "Epoch 249, Total Loss: 1.1467338800430298, Accuracy Rate: 61.26%,            Test_Total_Loss: 1.1066664457321167, Test_Accuracy_Rate: 62.54%\n",
      "Epoch 250, Total Loss: 1.140913963317871, Accuracy Rate: 61.22%,            Test_Total_Loss: 1.1026707887649536, Test_Accuracy_Rate: 62.17%\n",
      "Epoch 251, Total Loss: 1.140579104423523, Accuracy Rate: 60.76%,            Test_Total_Loss: 1.1043510437011719, Test_Accuracy_Rate: 62.42%\n",
      "Epoch 252, Total Loss: 1.139020562171936, Accuracy Rate: 61.47%,            Test_Total_Loss: 1.09229576587677, Test_Accuracy_Rate: 62.54%\n",
      "Epoch 253, Total Loss: 1.1332907676696777, Accuracy Rate: 61.18%,            Test_Total_Loss: 1.0792405605316162, Test_Accuracy_Rate: 63.25%\n",
      "Epoch 254, Total Loss: 1.134264349937439, Accuracy Rate: 61.13%,            Test_Total_Loss: 1.0831577777862549, Test_Accuracy_Rate: 62.75%\n",
      "Epoch 255, Total Loss: 1.125956416130066, Accuracy Rate: 60.35%,            Test_Total_Loss: 1.1110453605651855, Test_Accuracy_Rate: 62.04%\n",
      "Epoch 256, Total Loss: 1.1283105611801147, Accuracy Rate: 60.55%,            Test_Total_Loss: 1.0910454988479614, Test_Accuracy_Rate: 63.04%\n",
      "Epoch 257, Total Loss: 1.1257230043411255, Accuracy Rate: 60.97%,            Test_Total_Loss: 1.0760012865066528, Test_Accuracy_Rate: 63.16%\n",
      "Epoch 258, Total Loss: 1.1343343257904053, Accuracy Rate: 60.43%,            Test_Total_Loss: 1.0870637893676758, Test_Accuracy_Rate: 62.67%\n",
      "Epoch 259, Total Loss: 1.1554124355316162, Accuracy Rate: 60.64%,            Test_Total_Loss: 1.5603662729263306, Test_Accuracy_Rate: 50.99%\n",
      "Epoch 260, Total Loss: 1.3167037963867188, Accuracy Rate: 56.58%,            Test_Total_Loss: 1.1375876665115356, Test_Accuracy_Rate: 61.34%\n",
      "Epoch 261, Total Loss: 1.1748721599578857, Accuracy Rate: 59.19%,            Test_Total_Loss: 1.2009326219558716, Test_Accuracy_Rate: 59.27%\n",
      "Epoch 262, Total Loss: 1.15827214717865, Accuracy Rate: 59.98%,            Test_Total_Loss: 1.1039565801620483, Test_Accuracy_Rate: 62.13%\n",
      "Epoch 263, Total Loss: 1.1242525577545166, Accuracy Rate: 61.22%,            Test_Total_Loss: 1.0860389471054077, Test_Accuracy_Rate: 62.87%\n",
      "Epoch 264, Total Loss: 1.1216051578521729, Accuracy Rate: 61.05%,            Test_Total_Loss: 1.09377920627594, Test_Accuracy_Rate: 62.33%\n",
      "Epoch 265, Total Loss: 1.1278430223464966, Accuracy Rate: 60.47%,            Test_Total_Loss: 1.0804444551467896, Test_Accuracy_Rate: 62.71%\n",
      "Epoch 266, Total Loss: 1.1124385595321655, Accuracy Rate: 60.97%,            Test_Total_Loss: 1.0754187107086182, Test_Accuracy_Rate: 62.83%\n",
      "Epoch 267, Total Loss: 1.115614891052246, Accuracy Rate: 61.38%,            Test_Total_Loss: 1.0977946519851685, Test_Accuracy_Rate: 62.13%\n",
      "Epoch 268, Total Loss: 1.1088060140609741, Accuracy Rate: 61.67%,            Test_Total_Loss: 1.0570770502090454, Test_Accuracy_Rate: 63.25%\n",
      "Epoch 269, Total Loss: 1.1118764877319336, Accuracy Rate: 61.18%,            Test_Total_Loss: 1.0613540410995483, Test_Accuracy_Rate: 63.74%\n",
      "Epoch 270, Total Loss: 1.1028871536254883, Accuracy Rate: 61.84%,            Test_Total_Loss: 1.0657709836959839, Test_Accuracy_Rate: 63.16%\n",
      "Epoch 271, Total Loss: 1.1007425785064697, Accuracy Rate: 62.09%,            Test_Total_Loss: 1.0604712963104248, Test_Accuracy_Rate: 63.58%\n",
      "Epoch 272, Total Loss: 1.0998870134353638, Accuracy Rate: 62.17%,            Test_Total_Loss: 1.0850765705108643, Test_Accuracy_Rate: 62.54%\n",
      "Epoch 273, Total Loss: 1.1059268712997437, Accuracy Rate: 60.93%,            Test_Total_Loss: 1.0570663213729858, Test_Accuracy_Rate: 63.82%\n",
      "Epoch 274, Total Loss: 1.104392647743225, Accuracy Rate: 61.51%,            Test_Total_Loss: 1.0638192892074585, Test_Accuracy_Rate: 62.91%\n",
      "Epoch 275, Total Loss: 1.1038978099822998, Accuracy Rate: 61.67%,            Test_Total_Loss: 1.0651435852050781, Test_Accuracy_Rate: 63.00%\n",
      "Epoch 276, Total Loss: 1.098677635192871, Accuracy Rate: 61.47%,            Test_Total_Loss: 1.1013695001602173, Test_Accuracy_Rate: 61.96%\n",
      "Epoch 277, Total Loss: 1.126680850982666, Accuracy Rate: 60.76%,            Test_Total_Loss: 1.0828756093978882, Test_Accuracy_Rate: 62.62%\n",
      "Epoch 278, Total Loss: 1.1229674816131592, Accuracy Rate: 60.93%,            Test_Total_Loss: 1.0747861862182617, Test_Accuracy_Rate: 62.71%\n",
      "Epoch 279, Total Loss: 1.1030993461608887, Accuracy Rate: 60.84%,            Test_Total_Loss: 1.0521892309188843, Test_Accuracy_Rate: 63.45%\n",
      "Epoch 280, Total Loss: 1.0901910066604614, Accuracy Rate: 61.01%,            Test_Total_Loss: 1.04811429977417, Test_Accuracy_Rate: 63.87%\n",
      "Epoch 281, Total Loss: 1.0986371040344238, Accuracy Rate: 61.13%,            Test_Total_Loss: 1.0772850513458252, Test_Accuracy_Rate: 63.08%\n",
      "Epoch 282, Total Loss: 1.1043474674224854, Accuracy Rate: 60.84%,            Test_Total_Loss: 1.0829144716262817, Test_Accuracy_Rate: 62.62%\n",
      "Epoch 283, Total Loss: 1.130061149597168, Accuracy Rate: 60.76%,            Test_Total_Loss: 1.0479854345321655, Test_Accuracy_Rate: 63.95%\n",
      "Epoch 284, Total Loss: 1.0952723026275635, Accuracy Rate: 61.67%,            Test_Total_Loss: 1.0418592691421509, Test_Accuracy_Rate: 63.49%\n",
      "Epoch 285, Total Loss: 1.1015628576278687, Accuracy Rate: 61.88%,            Test_Total_Loss: 1.0729000568389893, Test_Accuracy_Rate: 62.46%\n",
      "Epoch 286, Total Loss: 1.0880086421966553, Accuracy Rate: 62.17%,            Test_Total_Loss: 1.0457035303115845, Test_Accuracy_Rate: 63.58%\n",
      "Epoch 287, Total Loss: 1.087295651435852, Accuracy Rate: 61.59%,            Test_Total_Loss: 1.0382213592529297, Test_Accuracy_Rate: 63.95%\n",
      "Epoch 288, Total Loss: 1.0857493877410889, Accuracy Rate: 61.42%,            Test_Total_Loss: 1.049978256225586, Test_Accuracy_Rate: 63.49%\n",
      "Epoch 289, Total Loss: 1.1092990636825562, Accuracy Rate: 60.97%,            Test_Total_Loss: 1.0367975234985352, Test_Accuracy_Rate: 63.16%\n",
      "Epoch 290, Total Loss: 1.176082968711853, Accuracy Rate: 58.98%,            Test_Total_Loss: 1.1036983728408813, Test_Accuracy_Rate: 62.29%\n",
      "Epoch 291, Total Loss: 1.1318190097808838, Accuracy Rate: 60.47%,            Test_Total_Loss: 1.0344678163528442, Test_Accuracy_Rate: 64.07%\n",
      "Epoch 292, Total Loss: 1.0864161252975464, Accuracy Rate: 62.29%,            Test_Total_Loss: 1.034512996673584, Test_Accuracy_Rate: 63.49%\n",
      "Epoch 293, Total Loss: 1.0787713527679443, Accuracy Rate: 61.63%,            Test_Total_Loss: 1.0505367517471313, Test_Accuracy_Rate: 63.58%\n",
      "Epoch 294, Total Loss: 1.076560616493225, Accuracy Rate: 62.17%,            Test_Total_Loss: 1.0165024995803833, Test_Accuracy_Rate: 64.16%\n",
      "Epoch 295, Total Loss: 1.070237159729004, Accuracy Rate: 61.71%,            Test_Total_Loss: 1.0346883535385132, Test_Accuracy_Rate: 63.99%\n",
      "Epoch 296, Total Loss: 1.0742747783660889, Accuracy Rate: 61.84%,            Test_Total_Loss: 1.018857717514038, Test_Accuracy_Rate: 64.11%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 297, Total Loss: 1.0804129838943481, Accuracy Rate: 61.67%,            Test_Total_Loss: 1.0431745052337646, Test_Accuracy_Rate: 64.03%\n",
      "Epoch 298, Total Loss: 1.0816030502319336, Accuracy Rate: 61.05%,            Test_Total_Loss: 1.0217280387878418, Test_Accuracy_Rate: 63.95%\n",
      "Epoch 299, Total Loss: 1.0696512460708618, Accuracy Rate: 61.47%,            Test_Total_Loss: 1.0349012613296509, Test_Accuracy_Rate: 63.70%\n",
      "Epoch 300, Total Loss: 1.071004033088684, Accuracy Rate: 61.84%,            Test_Total_Loss: 1.0177199840545654, Test_Accuracy_Rate: 63.91%\n",
      "Epoch 301, Total Loss: 1.0629388093948364, Accuracy Rate: 62.00%,            Test_Total_Loss: 1.0285016298294067, Test_Accuracy_Rate: 64.07%\n",
      "Epoch 302, Total Loss: 1.0710796117782593, Accuracy Rate: 61.75%,            Test_Total_Loss: 1.0122277736663818, Test_Accuracy_Rate: 63.82%\n",
      "Epoch 303, Total Loss: 1.3886016607284546, Accuracy Rate: 56.50%,            Test_Total_Loss: 1.5855107307434082, Test_Accuracy_Rate: 51.66%\n",
      "Epoch 304, Total Loss: 1.4567009210586548, Accuracy Rate: 53.52%,            Test_Total_Loss: 1.2223845720291138, Test_Accuracy_Rate: 58.24%\n",
      "Epoch 305, Total Loss: 1.2328742742538452, Accuracy Rate: 58.69%,            Test_Total_Loss: 1.1654285192489624, Test_Accuracy_Rate: 60.31%\n",
      "Epoch 306, Total Loss: 1.156286597251892, Accuracy Rate: 60.02%,            Test_Total_Loss: 1.0526916980743408, Test_Accuracy_Rate: 63.70%\n",
      "Epoch 307, Total Loss: 1.079440951347351, Accuracy Rate: 61.84%,            Test_Total_Loss: 1.023686170578003, Test_Accuracy_Rate: 64.20%\n",
      "Epoch 308, Total Loss: 1.0720793008804321, Accuracy Rate: 61.96%,            Test_Total_Loss: 1.0185140371322632, Test_Accuracy_Rate: 64.11%\n",
      "Epoch 309, Total Loss: 1.0626429319381714, Accuracy Rate: 62.25%,            Test_Total_Loss: 1.0148614645004272, Test_Accuracy_Rate: 64.28%\n",
      "Epoch 310, Total Loss: 1.05362868309021, Accuracy Rate: 62.33%,            Test_Total_Loss: 1.0246996879577637, Test_Accuracy_Rate: 63.91%\n",
      "Epoch 311, Total Loss: 1.0584102869033813, Accuracy Rate: 62.46%,            Test_Total_Loss: 1.0069489479064941, Test_Accuracy_Rate: 64.40%\n",
      "Epoch 312, Total Loss: 1.0473130941390991, Accuracy Rate: 62.17%,            Test_Total_Loss: 1.0321582555770874, Test_Accuracy_Rate: 63.33%\n",
      "Epoch 313, Total Loss: 1.0470672845840454, Accuracy Rate: 62.75%,            Test_Total_Loss: 1.0222113132476807, Test_Accuracy_Rate: 63.99%\n",
      "Epoch 314, Total Loss: 1.0478805303573608, Accuracy Rate: 62.71%,            Test_Total_Loss: 1.0116993188858032, Test_Accuracy_Rate: 63.99%\n",
      "Epoch 315, Total Loss: 1.0444647073745728, Accuracy Rate: 62.13%,            Test_Total_Loss: 1.0200676918029785, Test_Accuracy_Rate: 63.87%\n",
      "Epoch 316, Total Loss: 1.0535497665405273, Accuracy Rate: 62.25%,            Test_Total_Loss: 0.9993267059326172, Test_Accuracy_Rate: 64.74%\n",
      "Epoch 317, Total Loss: 1.0458253622055054, Accuracy Rate: 62.09%,            Test_Total_Loss: 1.0046716928482056, Test_Accuracy_Rate: 64.24%\n",
      "Epoch 318, Total Loss: 1.0434852838516235, Accuracy Rate: 62.38%,            Test_Total_Loss: 0.9943086504936218, Test_Accuracy_Rate: 64.86%\n",
      "Epoch 319, Total Loss: 1.0471701622009277, Accuracy Rate: 62.13%,            Test_Total_Loss: 0.9910903573036194, Test_Accuracy_Rate: 64.65%\n",
      "Epoch 320, Total Loss: 1.039790391921997, Accuracy Rate: 62.75%,            Test_Total_Loss: 1.000835657119751, Test_Accuracy_Rate: 64.53%\n",
      "Epoch 321, Total Loss: 1.0571764707565308, Accuracy Rate: 62.13%,            Test_Total_Loss: 1.0028140544891357, Test_Accuracy_Rate: 64.20%\n",
      "Epoch 322, Total Loss: 1.047760009765625, Accuracy Rate: 62.29%,            Test_Total_Loss: 1.049088954925537, Test_Accuracy_Rate: 63.45%\n",
      "Epoch 323, Total Loss: 1.049587368965149, Accuracy Rate: 62.54%,            Test_Total_Loss: 1.011200189590454, Test_Accuracy_Rate: 64.49%\n",
      "Epoch 324, Total Loss: 1.0527702569961548, Accuracy Rate: 62.38%,            Test_Total_Loss: 1.015882968902588, Test_Accuracy_Rate: 64.07%\n",
      "Epoch 325, Total Loss: 1.0464578866958618, Accuracy Rate: 62.17%,            Test_Total_Loss: 1.0147982835769653, Test_Accuracy_Rate: 64.03%\n",
      "Epoch 326, Total Loss: 1.0401132106781006, Accuracy Rate: 62.38%,            Test_Total_Loss: 1.007724642753601, Test_Accuracy_Rate: 64.36%\n",
      "Epoch 327, Total Loss: 1.039293885231018, Accuracy Rate: 63.08%,            Test_Total_Loss: 0.9906303286552429, Test_Accuracy_Rate: 64.61%\n",
      "Epoch 328, Total Loss: 1.037587285041809, Accuracy Rate: 63.16%,            Test_Total_Loss: 0.9960306882858276, Test_Accuracy_Rate: 64.53%\n",
      "Epoch 329, Total Loss: 1.0367227792739868, Accuracy Rate: 62.00%,            Test_Total_Loss: 0.9783397316932678, Test_Accuracy_Rate: 64.53%\n",
      "Epoch 330, Total Loss: 1.036912441253662, Accuracy Rate: 62.38%,            Test_Total_Loss: 0.9824897646903992, Test_Accuracy_Rate: 64.20%\n",
      "Epoch 331, Total Loss: 1.035518765449524, Accuracy Rate: 61.88%,            Test_Total_Loss: 0.9926225543022156, Test_Accuracy_Rate: 64.36%\n",
      "Epoch 332, Total Loss: 1.036247968673706, Accuracy Rate: 62.25%,            Test_Total_Loss: 0.996910810470581, Test_Accuracy_Rate: 64.11%\n",
      "Epoch 333, Total Loss: 1.0387331247329712, Accuracy Rate: 61.96%,            Test_Total_Loss: 0.9860014915466309, Test_Accuracy_Rate: 64.78%\n",
      "Epoch 334, Total Loss: 1.0278379917144775, Accuracy Rate: 63.08%,            Test_Total_Loss: 0.9909300804138184, Test_Accuracy_Rate: 64.61%\n",
      "Epoch 335, Total Loss: 1.0347293615341187, Accuracy Rate: 62.29%,            Test_Total_Loss: 0.9882009625434875, Test_Accuracy_Rate: 64.57%\n",
      "Epoch 336, Total Loss: 1.0319501161575317, Accuracy Rate: 62.04%,            Test_Total_Loss: 0.9864471554756165, Test_Accuracy_Rate: 64.53%\n",
      "Epoch 337, Total Loss: 1.0288509130477905, Accuracy Rate: 62.04%,            Test_Total_Loss: 0.990617573261261, Test_Accuracy_Rate: 64.57%\n",
      "Epoch 338, Total Loss: 1.0308516025543213, Accuracy Rate: 62.46%,            Test_Total_Loss: 0.9844335317611694, Test_Accuracy_Rate: 64.74%\n",
      "Epoch 339, Total Loss: 1.025963306427002, Accuracy Rate: 61.80%,            Test_Total_Loss: 0.9914504885673523, Test_Accuracy_Rate: 64.24%\n",
      "Epoch 340, Total Loss: 1.0336833000183105, Accuracy Rate: 62.09%,            Test_Total_Loss: 0.9870041012763977, Test_Accuracy_Rate: 64.49%\n",
      "Epoch 341, Total Loss: 1.0312687158584595, Accuracy Rate: 62.46%,            Test_Total_Loss: 0.9956154227256775, Test_Accuracy_Rate: 64.16%\n",
      "Epoch 342, Total Loss: 1.0305591821670532, Accuracy Rate: 61.92%,            Test_Total_Loss: 0.9705562591552734, Test_Accuracy_Rate: 64.82%\n",
      "Epoch 343, Total Loss: 1.023187279701233, Accuracy Rate: 61.59%,            Test_Total_Loss: 0.9790976643562317, Test_Accuracy_Rate: 64.61%\n",
      "Epoch 344, Total Loss: 1.0360523462295532, Accuracy Rate: 61.92%,            Test_Total_Loss: 1.4500319957733154, Test_Accuracy_Rate: 54.39%\n",
      "Epoch 345, Total Loss: 1.3477017879486084, Accuracy Rate: 56.79%,            Test_Total_Loss: 1.1486704349517822, Test_Accuracy_Rate: 59.73%\n",
      "Epoch 346, Total Loss: 1.1829699277877808, Accuracy Rate: 58.90%,            Test_Total_Loss: 1.1922743320465088, Test_Accuracy_Rate: 60.02%\n",
      "Epoch 347, Total Loss: 1.153548240661621, Accuracy Rate: 59.69%,            Test_Total_Loss: 1.061278223991394, Test_Accuracy_Rate: 62.58%\n",
      "Epoch 348, Total Loss: 1.0788613557815552, Accuracy Rate: 61.71%,            Test_Total_Loss: 1.010563850402832, Test_Accuracy_Rate: 63.82%\n",
      "Epoch 349, Total Loss: 1.0370804071426392, Accuracy Rate: 62.42%,            Test_Total_Loss: 0.9900832772254944, Test_Accuracy_Rate: 64.16%\n",
      "Epoch 350, Total Loss: 1.0302164554595947, Accuracy Rate: 63.04%,            Test_Total_Loss: 0.9818307161331177, Test_Accuracy_Rate: 64.40%\n",
      "Epoch 351, Total Loss: 1.042480707168579, Accuracy Rate: 61.55%,            Test_Total_Loss: 1.0201536417007446, Test_Accuracy_Rate: 63.62%\n",
      "Epoch 352, Total Loss: 1.0367873907089233, Accuracy Rate: 61.88%,            Test_Total_Loss: 1.007676124572754, Test_Accuracy_Rate: 64.07%\n",
      "Epoch 353, Total Loss: 1.030551791191101, Accuracy Rate: 62.71%,            Test_Total_Loss: 1.0065281391143799, Test_Accuracy_Rate: 63.45%\n",
      "Epoch 354, Total Loss: 1.020341157913208, Accuracy Rate: 62.71%,            Test_Total_Loss: 0.9755342602729797, Test_Accuracy_Rate: 64.69%\n",
      "Epoch 355, Total Loss: 1.0191818475723267, Accuracy Rate: 62.13%,            Test_Total_Loss: 0.9926870465278625, Test_Accuracy_Rate: 64.40%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 356, Total Loss: 1.0232576131820679, Accuracy Rate: 62.13%,            Test_Total_Loss: 0.9955912232398987, Test_Accuracy_Rate: 64.45%\n",
      "Epoch 357, Total Loss: 1.0347490310668945, Accuracy Rate: 61.55%,            Test_Total_Loss: 0.97931307554245, Test_Accuracy_Rate: 64.53%\n",
      "Epoch 358, Total Loss: 1.0351812839508057, Accuracy Rate: 62.38%,            Test_Total_Loss: 0.9714592099189758, Test_Accuracy_Rate: 64.74%\n",
      "Epoch 359, Total Loss: 1.0175049304962158, Accuracy Rate: 62.58%,            Test_Total_Loss: 0.9822666049003601, Test_Accuracy_Rate: 64.65%\n",
      "Epoch 360, Total Loss: 1.0129635334014893, Accuracy Rate: 62.54%,            Test_Total_Loss: 0.9670915007591248, Test_Accuracy_Rate: 65.23%\n",
      "Epoch 361, Total Loss: 1.0125657320022583, Accuracy Rate: 62.75%,            Test_Total_Loss: 0.9713450074195862, Test_Accuracy_Rate: 64.86%\n",
      "Epoch 362, Total Loss: 1.0120011568069458, Accuracy Rate: 62.54%,            Test_Total_Loss: 0.9603548645973206, Test_Accuracy_Rate: 65.11%\n",
      "Epoch 363, Total Loss: 1.008002519607544, Accuracy Rate: 63.08%,            Test_Total_Loss: 0.9596064686775208, Test_Accuracy_Rate: 65.11%\n",
      "Epoch 364, Total Loss: 1.0090214014053345, Accuracy Rate: 63.04%,            Test_Total_Loss: 0.9545071125030518, Test_Accuracy_Rate: 64.98%\n",
      "Epoch 365, Total Loss: 1.0098469257354736, Accuracy Rate: 61.88%,            Test_Total_Loss: 0.960851788520813, Test_Accuracy_Rate: 65.23%\n",
      "Epoch 366, Total Loss: 1.0154474973678589, Accuracy Rate: 62.25%,            Test_Total_Loss: 0.9568861722946167, Test_Accuracy_Rate: 64.98%\n",
      "Epoch 367, Total Loss: 1.0115995407104492, Accuracy Rate: 62.42%,            Test_Total_Loss: 0.9786975979804993, Test_Accuracy_Rate: 64.69%\n",
      "Epoch 368, Total Loss: 1.0181511640548706, Accuracy Rate: 62.13%,            Test_Total_Loss: 0.9635898470878601, Test_Accuracy_Rate: 65.23%\n",
      "Epoch 369, Total Loss: 1.0102713108062744, Accuracy Rate: 62.91%,            Test_Total_Loss: 0.9617488980293274, Test_Accuracy_Rate: 64.86%\n",
      "Epoch 370, Total Loss: 1.009986400604248, Accuracy Rate: 62.67%,            Test_Total_Loss: 0.9613575339317322, Test_Accuracy_Rate: 65.11%\n",
      "Epoch 371, Total Loss: 1.0087474584579468, Accuracy Rate: 62.58%,            Test_Total_Loss: 0.9558332562446594, Test_Accuracy_Rate: 65.15%\n",
      "Epoch 372, Total Loss: 1.0081281661987305, Accuracy Rate: 62.38%,            Test_Total_Loss: 0.9653839468955994, Test_Accuracy_Rate: 64.86%\n",
      "Epoch 373, Total Loss: 1.0015255212783813, Accuracy Rate: 62.62%,            Test_Total_Loss: 0.9624374508857727, Test_Accuracy_Rate: 65.02%\n",
      "Epoch 374, Total Loss: 1.0019131898880005, Accuracy Rate: 62.91%,            Test_Total_Loss: 0.9533075094223022, Test_Accuracy_Rate: 65.07%\n",
      "Epoch 375, Total Loss: 1.0097684860229492, Accuracy Rate: 62.09%,            Test_Total_Loss: 0.9579083323478699, Test_Accuracy_Rate: 65.44%\n",
      "Epoch 376, Total Loss: 1.0078954696655273, Accuracy Rate: 62.42%,            Test_Total_Loss: 1.0898425579071045, Test_Accuracy_Rate: 61.92%\n",
      "Epoch 377, Total Loss: 1.128002405166626, Accuracy Rate: 59.64%,            Test_Total_Loss: 0.9792574644088745, Test_Accuracy_Rate: 64.65%\n",
      "Epoch 378, Total Loss: 1.0189958810806274, Accuracy Rate: 62.58%,            Test_Total_Loss: 0.9739611744880676, Test_Accuracy_Rate: 64.49%\n",
      "Epoch 379, Total Loss: 1.0135235786437988, Accuracy Rate: 63.04%,            Test_Total_Loss: 1.0019817352294922, Test_Accuracy_Rate: 64.20%\n",
      "Epoch 380, Total Loss: 1.0203657150268555, Accuracy Rate: 62.46%,            Test_Total_Loss: 0.9627090096473694, Test_Accuracy_Rate: 65.19%\n",
      "Epoch 381, Total Loss: 1.0110442638397217, Accuracy Rate: 62.91%,            Test_Total_Loss: 0.9726141691207886, Test_Accuracy_Rate: 64.45%\n",
      "Epoch 382, Total Loss: 1.0055400133132935, Accuracy Rate: 62.75%,            Test_Total_Loss: 0.9532548785209656, Test_Accuracy_Rate: 65.15%\n",
      "Epoch 383, Total Loss: 1.0007309913635254, Accuracy Rate: 62.71%,            Test_Total_Loss: 0.9562373757362366, Test_Accuracy_Rate: 65.31%\n",
      "Epoch 384, Total Loss: 1.0042238235473633, Accuracy Rate: 62.67%,            Test_Total_Loss: 0.9573349356651306, Test_Accuracy_Rate: 64.61%\n",
      "Epoch 385, Total Loss: 0.9991499185562134, Accuracy Rate: 61.92%,            Test_Total_Loss: 0.9802797436714172, Test_Accuracy_Rate: 64.86%\n",
      "Epoch 386, Total Loss: 0.9929969310760498, Accuracy Rate: 62.87%,            Test_Total_Loss: 0.950923502445221, Test_Accuracy_Rate: 65.23%\n",
      "Epoch 387, Total Loss: 0.9998911619186401, Accuracy Rate: 63.41%,            Test_Total_Loss: 0.9558925628662109, Test_Accuracy_Rate: 65.07%\n",
      "Epoch 388, Total Loss: 1.0033496618270874, Accuracy Rate: 62.67%,            Test_Total_Loss: 0.9606074094772339, Test_Accuracy_Rate: 64.69%\n",
      "Epoch 389, Total Loss: 1.0015321969985962, Accuracy Rate: 62.46%,            Test_Total_Loss: 0.9515693783760071, Test_Accuracy_Rate: 65.52%\n",
      "Epoch 390, Total Loss: 1.0025207996368408, Accuracy Rate: 62.17%,            Test_Total_Loss: 0.9577320218086243, Test_Accuracy_Rate: 64.86%\n",
      "Epoch 391, Total Loss: 1.009181022644043, Accuracy Rate: 63.33%,            Test_Total_Loss: 0.9664651155471802, Test_Accuracy_Rate: 64.61%\n",
      "Epoch 392, Total Loss: 1.0187807083129883, Accuracy Rate: 61.59%,            Test_Total_Loss: 0.9737369418144226, Test_Accuracy_Rate: 64.28%\n",
      "Epoch 393, Total Loss: 1.0113416910171509, Accuracy Rate: 61.63%,            Test_Total_Loss: 0.9632275104522705, Test_Accuracy_Rate: 64.86%\n",
      "Epoch 394, Total Loss: 1.0024787187576294, Accuracy Rate: 63.12%,            Test_Total_Loss: 0.9507444500923157, Test_Accuracy_Rate: 64.69%\n",
      "Epoch 395, Total Loss: 1.0092686414718628, Accuracy Rate: 62.00%,            Test_Total_Loss: 0.9542930126190186, Test_Accuracy_Rate: 65.31%\n",
      "Epoch 396, Total Loss: 1.0047717094421387, Accuracy Rate: 62.46%,            Test_Total_Loss: 0.9444112181663513, Test_Accuracy_Rate: 65.44%\n",
      "Epoch 397, Total Loss: 1.005642056465149, Accuracy Rate: 62.46%,            Test_Total_Loss: 0.9768118858337402, Test_Accuracy_Rate: 64.28%\n",
      "Epoch 398, Total Loss: 1.0242524147033691, Accuracy Rate: 62.54%,            Test_Total_Loss: 0.9743479490280151, Test_Accuracy_Rate: 64.65%\n",
      "Epoch 399, Total Loss: 1.0081919431686401, Accuracy Rate: 62.33%,            Test_Total_Loss: 0.9516468048095703, Test_Accuracy_Rate: 64.94%\n",
      "Epoch 400, Total Loss: 0.9961776733398438, Accuracy Rate: 63.04%,            Test_Total_Loss: 0.9494858980178833, Test_Accuracy_Rate: 65.31%\n",
      "Epoch 401, Total Loss: 1.0012067556381226, Accuracy Rate: 62.50%,            Test_Total_Loss: 0.9719048738479614, Test_Accuracy_Rate: 64.36%\n",
      "Epoch 402, Total Loss: 1.0067224502563477, Accuracy Rate: 62.13%,            Test_Total_Loss: 0.9662063717842102, Test_Accuracy_Rate: 65.36%\n",
      "Epoch 403, Total Loss: 1.0846519470214844, Accuracy Rate: 60.64%,            Test_Total_Loss: 1.009883165359497, Test_Accuracy_Rate: 63.16%\n",
      "Epoch 404, Total Loss: 1.0414454936981201, Accuracy Rate: 62.17%,            Test_Total_Loss: 1.0072851181030273, Test_Accuracy_Rate: 63.78%\n",
      "Epoch 405, Total Loss: 1.0231884717941284, Accuracy Rate: 63.20%,            Test_Total_Loss: 0.9921254515647888, Test_Accuracy_Rate: 64.36%\n",
      "Epoch 406, Total Loss: 1.0174834728240967, Accuracy Rate: 62.62%,            Test_Total_Loss: 0.9597578644752502, Test_Accuracy_Rate: 64.90%\n",
      "Epoch 407, Total Loss: 1.0137804746627808, Accuracy Rate: 62.54%,            Test_Total_Loss: 1.0010626316070557, Test_Accuracy_Rate: 64.16%\n",
      "Epoch 408, Total Loss: 1.0136795043945312, Accuracy Rate: 62.46%,            Test_Total_Loss: 0.955150306224823, Test_Accuracy_Rate: 64.98%\n",
      "Epoch 409, Total Loss: 1.0095746517181396, Accuracy Rate: 62.79%,            Test_Total_Loss: 0.9462013244628906, Test_Accuracy_Rate: 65.23%\n",
      "Epoch 410, Total Loss: 0.9929842948913574, Accuracy Rate: 62.38%,            Test_Total_Loss: 0.9503585696220398, Test_Accuracy_Rate: 65.31%\n",
      "Epoch 411, Total Loss: 0.9905205965042114, Accuracy Rate: 62.54%,            Test_Total_Loss: 0.9443434476852417, Test_Accuracy_Rate: 65.65%\n",
      "Epoch 412, Total Loss: 0.9869826436042786, Accuracy Rate: 63.16%,            Test_Total_Loss: 0.9276449084281921, Test_Accuracy_Rate: 65.73%\n",
      "Epoch 413, Total Loss: 0.9822990298271179, Accuracy Rate: 63.58%,            Test_Total_Loss: 0.9438316822052002, Test_Accuracy_Rate: 65.23%\n",
      "Epoch 414, Total Loss: 0.9839490652084351, Accuracy Rate: 63.41%,            Test_Total_Loss: 0.9297996163368225, Test_Accuracy_Rate: 65.44%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 415, Total Loss: 0.9749484658241272, Accuracy Rate: 63.33%,            Test_Total_Loss: 0.9321150779724121, Test_Accuracy_Rate: 65.44%\n",
      "Epoch 416, Total Loss: 0.9869654774665833, Accuracy Rate: 63.33%,            Test_Total_Loss: 0.9606902599334717, Test_Accuracy_Rate: 64.98%\n",
      "Epoch 417, Total Loss: 0.9859361052513123, Accuracy Rate: 62.79%,            Test_Total_Loss: 0.9418520927429199, Test_Accuracy_Rate: 64.45%\n",
      "Epoch 418, Total Loss: 0.9808678030967712, Accuracy Rate: 62.67%,            Test_Total_Loss: 0.9401296377182007, Test_Accuracy_Rate: 65.07%\n",
      "Epoch 419, Total Loss: 0.9849953651428223, Accuracy Rate: 63.00%,            Test_Total_Loss: 0.9502695798873901, Test_Accuracy_Rate: 65.11%\n",
      "Epoch 420, Total Loss: 0.998996913433075, Accuracy Rate: 62.42%,            Test_Total_Loss: 0.951575756072998, Test_Accuracy_Rate: 65.31%\n",
      "Epoch 421, Total Loss: 1.018107295036316, Accuracy Rate: 62.87%,            Test_Total_Loss: 0.9720297455787659, Test_Accuracy_Rate: 64.57%\n",
      "Epoch 422, Total Loss: 1.0221586227416992, Accuracy Rate: 62.33%,            Test_Total_Loss: 0.9666969180107117, Test_Accuracy_Rate: 64.94%\n",
      "Epoch 423, Total Loss: 0.9916348457336426, Accuracy Rate: 62.83%,            Test_Total_Loss: 0.9525397419929504, Test_Accuracy_Rate: 65.40%\n",
      "Epoch 424, Total Loss: 0.9876063466072083, Accuracy Rate: 62.87%,            Test_Total_Loss: 0.9530425071716309, Test_Accuracy_Rate: 64.98%\n",
      "Epoch 425, Total Loss: 0.9873420596122742, Accuracy Rate: 62.91%,            Test_Total_Loss: 0.947124183177948, Test_Accuracy_Rate: 65.23%\n",
      "Epoch 426, Total Loss: 0.9851702451705933, Accuracy Rate: 63.04%,            Test_Total_Loss: 0.9407668113708496, Test_Accuracy_Rate: 65.40%\n",
      "Epoch 427, Total Loss: 0.9865954518318176, Accuracy Rate: 62.62%,            Test_Total_Loss: 0.9407647848129272, Test_Accuracy_Rate: 65.07%\n",
      "Epoch 428, Total Loss: 0.9897388815879822, Accuracy Rate: 62.54%,            Test_Total_Loss: 0.9355496764183044, Test_Accuracy_Rate: 65.11%\n",
      "Epoch 429, Total Loss: 0.9770130515098572, Accuracy Rate: 63.29%,            Test_Total_Loss: 0.9465047717094421, Test_Accuracy_Rate: 65.02%\n",
      "Epoch 430, Total Loss: 0.9783797860145569, Accuracy Rate: 63.29%,            Test_Total_Loss: 0.9369648098945618, Test_Accuracy_Rate: 65.52%\n",
      "Epoch 431, Total Loss: 0.980080246925354, Accuracy Rate: 63.00%,            Test_Total_Loss: 0.9370083212852478, Test_Accuracy_Rate: 65.07%\n",
      "Epoch 432, Total Loss: 0.9777992367744446, Accuracy Rate: 62.50%,            Test_Total_Loss: 0.9297966957092285, Test_Accuracy_Rate: 65.27%\n",
      "Epoch 433, Total Loss: 0.9704682230949402, Accuracy Rate: 63.37%,            Test_Total_Loss: 0.927091121673584, Test_Accuracy_Rate: 65.36%\n",
      "Epoch 434, Total Loss: 0.9734033346176147, Accuracy Rate: 62.38%,            Test_Total_Loss: 0.951465904712677, Test_Accuracy_Rate: 65.48%\n",
      "Epoch 435, Total Loss: 0.975825846195221, Accuracy Rate: 62.75%,            Test_Total_Loss: 0.9447598457336426, Test_Accuracy_Rate: 65.23%\n",
      "Epoch 436, Total Loss: 0.9798402190208435, Accuracy Rate: 63.62%,            Test_Total_Loss: 0.9484581351280212, Test_Accuracy_Rate: 65.27%\n",
      "Epoch 437, Total Loss: 0.9808465838432312, Accuracy Rate: 62.79%,            Test_Total_Loss: 0.9296688437461853, Test_Accuracy_Rate: 65.15%\n",
      "Epoch 438, Total Loss: 0.9718809723854065, Accuracy Rate: 63.66%,            Test_Total_Loss: 0.9267324209213257, Test_Accuracy_Rate: 65.77%\n",
      "Epoch 439, Total Loss: 0.9764617085456848, Accuracy Rate: 62.54%,            Test_Total_Loss: 0.922686755657196, Test_Accuracy_Rate: 65.81%\n",
      "Epoch 440, Total Loss: 0.9906110167503357, Accuracy Rate: 63.12%,            Test_Total_Loss: 0.9471469521522522, Test_Accuracy_Rate: 65.07%\n",
      "Epoch 441, Total Loss: 0.9793006777763367, Accuracy Rate: 63.12%,            Test_Total_Loss: 0.9375312328338623, Test_Accuracy_Rate: 65.23%\n",
      "Epoch 442, Total Loss: 0.983696699142456, Accuracy Rate: 63.53%,            Test_Total_Loss: 0.935727059841156, Test_Accuracy_Rate: 65.40%\n",
      "Epoch 443, Total Loss: 0.9871427416801453, Accuracy Rate: 63.12%,            Test_Total_Loss: 0.9786023497581482, Test_Accuracy_Rate: 63.95%\n",
      "Epoch 444, Total Loss: 0.9929456114768982, Accuracy Rate: 62.79%,            Test_Total_Loss: 0.9199309349060059, Test_Accuracy_Rate: 65.81%\n",
      "Epoch 445, Total Loss: 0.9757483601570129, Accuracy Rate: 63.25%,            Test_Total_Loss: 0.9345415234565735, Test_Accuracy_Rate: 64.82%\n",
      "Epoch 446, Total Loss: 1.1037498712539673, Accuracy Rate: 61.42%,            Test_Total_Loss: 0.9983612895011902, Test_Accuracy_Rate: 63.74%\n",
      "Epoch 447, Total Loss: 1.0109167098999023, Accuracy Rate: 62.71%,            Test_Total_Loss: 0.9447668790817261, Test_Accuracy_Rate: 65.56%\n",
      "Epoch 448, Total Loss: 0.9884373545646667, Accuracy Rate: 62.75%,            Test_Total_Loss: 0.94741290807724, Test_Accuracy_Rate: 64.61%\n",
      "Epoch 449, Total Loss: 1.0141388177871704, Accuracy Rate: 61.92%,            Test_Total_Loss: 0.9789859652519226, Test_Accuracy_Rate: 64.45%\n",
      "Epoch 450, Total Loss: 1.0036078691482544, Accuracy Rate: 62.87%,            Test_Total_Loss: 0.959963321685791, Test_Accuracy_Rate: 64.28%\n",
      "Epoch 451, Total Loss: 0.9944277405738831, Accuracy Rate: 63.04%,            Test_Total_Loss: 0.9319069385528564, Test_Accuracy_Rate: 65.15%\n",
      "Epoch 452, Total Loss: 0.9759736061096191, Accuracy Rate: 62.79%,            Test_Total_Loss: 0.931135892868042, Test_Accuracy_Rate: 64.98%\n",
      "Epoch 453, Total Loss: 0.9714841246604919, Accuracy Rate: 62.75%,            Test_Total_Loss: 0.9247148633003235, Test_Accuracy_Rate: 65.81%\n",
      "Epoch 454, Total Loss: 0.9691888093948364, Accuracy Rate: 63.78%,            Test_Total_Loss: 0.9259963631629944, Test_Accuracy_Rate: 65.81%\n",
      "Epoch 455, Total Loss: 0.973855197429657, Accuracy Rate: 63.20%,            Test_Total_Loss: 0.9253129363059998, Test_Accuracy_Rate: 65.81%\n",
      "Epoch 456, Total Loss: 0.965988039970398, Accuracy Rate: 63.91%,            Test_Total_Loss: 0.9200523495674133, Test_Accuracy_Rate: 65.52%\n",
      "Epoch 457, Total Loss: 0.9751003384590149, Accuracy Rate: 63.87%,            Test_Total_Loss: 0.9280674457550049, Test_Accuracy_Rate: 65.56%\n",
      "Epoch 458, Total Loss: 0.977271556854248, Accuracy Rate: 63.16%,            Test_Total_Loss: 0.9266841411590576, Test_Accuracy_Rate: 65.56%\n",
      "Epoch 459, Total Loss: 0.9687283039093018, Accuracy Rate: 62.79%,            Test_Total_Loss: 0.933294415473938, Test_Accuracy_Rate: 65.31%\n",
      "Epoch 460, Total Loss: 0.9662781953811646, Accuracy Rate: 63.66%,            Test_Total_Loss: 0.9342900514602661, Test_Accuracy_Rate: 65.27%\n",
      "Epoch 461, Total Loss: 0.9689341187477112, Accuracy Rate: 62.87%,            Test_Total_Loss: 0.9266869425773621, Test_Accuracy_Rate: 65.19%\n",
      "Epoch 462, Total Loss: 0.9689156413078308, Accuracy Rate: 62.79%,            Test_Total_Loss: 0.9144656658172607, Test_Accuracy_Rate: 65.81%\n",
      "Epoch 463, Total Loss: 0.9936642646789551, Accuracy Rate: 63.25%,            Test_Total_Loss: 0.9787451028823853, Test_Accuracy_Rate: 64.49%\n",
      "Epoch 464, Total Loss: 1.0209640264511108, Accuracy Rate: 62.13%,            Test_Total_Loss: 1.0143041610717773, Test_Accuracy_Rate: 63.37%\n",
      "Epoch 465, Total Loss: 1.005362629890442, Accuracy Rate: 62.75%,            Test_Total_Loss: 0.9338074922561646, Test_Accuracy_Rate: 65.27%\n",
      "Epoch 466, Total Loss: 0.9828130602836609, Accuracy Rate: 62.79%,            Test_Total_Loss: 0.92730313539505, Test_Accuracy_Rate: 65.69%\n",
      "Epoch 467, Total Loss: 0.9726136922836304, Accuracy Rate: 63.29%,            Test_Total_Loss: 0.9440577626228333, Test_Accuracy_Rate: 65.07%\n",
      "Epoch 468, Total Loss: 0.9740779995918274, Accuracy Rate: 63.33%,            Test_Total_Loss: 0.9196873903274536, Test_Accuracy_Rate: 65.36%\n",
      "Epoch 469, Total Loss: 0.9674421548843384, Accuracy Rate: 63.58%,            Test_Total_Loss: 1.0439481735229492, Test_Accuracy_Rate: 63.53%\n",
      "Epoch 470, Total Loss: 1.0068668127059937, Accuracy Rate: 62.96%,            Test_Total_Loss: 0.9312940835952759, Test_Accuracy_Rate: 65.19%\n",
      "Epoch 471, Total Loss: 0.9898810386657715, Accuracy Rate: 63.45%,            Test_Total_Loss: 0.9366547465324402, Test_Accuracy_Rate: 64.86%\n",
      "Epoch 472, Total Loss: 0.9944252371788025, Accuracy Rate: 63.58%,            Test_Total_Loss: 0.9453362822532654, Test_Accuracy_Rate: 65.31%\n",
      "Epoch 473, Total Loss: 0.9846590161323547, Accuracy Rate: 63.04%,            Test_Total_Loss: 0.937635600566864, Test_Accuracy_Rate: 65.69%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 474, Total Loss: 0.982077419757843, Accuracy Rate: 63.04%,            Test_Total_Loss: 0.9597063660621643, Test_Accuracy_Rate: 64.90%\n",
      "Epoch 475, Total Loss: 0.9896824359893799, Accuracy Rate: 63.25%,            Test_Total_Loss: 0.9450851082801819, Test_Accuracy_Rate: 65.36%\n",
      "Epoch 476, Total Loss: 0.981434166431427, Accuracy Rate: 63.12%,            Test_Total_Loss: 0.9380396604537964, Test_Accuracy_Rate: 65.31%\n",
      "Epoch 477, Total Loss: 0.9713755249977112, Accuracy Rate: 63.74%,            Test_Total_Loss: 0.9237226247787476, Test_Accuracy_Rate: 65.56%\n",
      "Epoch 478, Total Loss: 0.956985354423523, Accuracy Rate: 64.11%,            Test_Total_Loss: 0.9221469759941101, Test_Accuracy_Rate: 65.56%\n",
      "Epoch 479, Total Loss: 0.9631897807121277, Accuracy Rate: 63.41%,            Test_Total_Loss: 0.936224102973938, Test_Accuracy_Rate: 65.69%\n",
      "Epoch 480, Total Loss: 0.9685849547386169, Accuracy Rate: 63.70%,            Test_Total_Loss: 0.9078409075737, Test_Accuracy_Rate: 65.94%\n",
      "Epoch 481, Total Loss: 0.9612310528755188, Accuracy Rate: 62.87%,            Test_Total_Loss: 0.9119321703910828, Test_Accuracy_Rate: 65.77%\n",
      "Epoch 482, Total Loss: 0.9573588967323303, Accuracy Rate: 63.00%,            Test_Total_Loss: 0.908233642578125, Test_Accuracy_Rate: 65.94%\n",
      "Epoch 483, Total Loss: 0.9582726359367371, Accuracy Rate: 64.03%,            Test_Total_Loss: 0.9159601330757141, Test_Accuracy_Rate: 65.85%\n",
      "Epoch 484, Total Loss: 0.9539486765861511, Accuracy Rate: 63.58%,            Test_Total_Loss: 0.9153143167495728, Test_Accuracy_Rate: 65.94%\n",
      "Epoch 485, Total Loss: 0.959755539894104, Accuracy Rate: 63.33%,            Test_Total_Loss: 0.9261733889579773, Test_Accuracy_Rate: 65.56%\n",
      "Epoch 486, Total Loss: 0.9675429463386536, Accuracy Rate: 63.08%,            Test_Total_Loss: 0.9554479718208313, Test_Accuracy_Rate: 65.23%\n",
      "Epoch 487, Total Loss: 0.9594715237617493, Accuracy Rate: 63.16%,            Test_Total_Loss: 0.9099650382995605, Test_Accuracy_Rate: 66.06%\n",
      "Epoch 488, Total Loss: 0.9534686207771301, Accuracy Rate: 63.70%,            Test_Total_Loss: 0.9167966246604919, Test_Accuracy_Rate: 65.44%\n",
      "Epoch 489, Total Loss: 0.9626731872558594, Accuracy Rate: 63.29%,            Test_Total_Loss: 0.9139068126678467, Test_Accuracy_Rate: 65.89%\n",
      "Epoch 490, Total Loss: 0.9577953219413757, Accuracy Rate: 63.49%,            Test_Total_Loss: 0.9263558387756348, Test_Accuracy_Rate: 65.69%\n",
      "Epoch 491, Total Loss: 0.9629018902778625, Accuracy Rate: 62.91%,            Test_Total_Loss: 0.9244381785392761, Test_Accuracy_Rate: 65.52%\n",
      "Epoch 492, Total Loss: 0.9703128337860107, Accuracy Rate: 62.91%,            Test_Total_Loss: 0.9117969870567322, Test_Accuracy_Rate: 65.56%\n",
      "Epoch 493, Total Loss: 0.9630072712898254, Accuracy Rate: 63.29%,            Test_Total_Loss: 0.9092238545417786, Test_Accuracy_Rate: 66.02%\n",
      "Epoch 494, Total Loss: 0.9546928405761719, Accuracy Rate: 63.53%,            Test_Total_Loss: 0.9232541918754578, Test_Accuracy_Rate: 65.60%\n",
      "Epoch 495, Total Loss: 0.9639979004859924, Accuracy Rate: 63.16%,            Test_Total_Loss: 0.9089969992637634, Test_Accuracy_Rate: 65.85%\n",
      "Epoch 496, Total Loss: 0.9648977518081665, Accuracy Rate: 63.49%,            Test_Total_Loss: 0.911567211151123, Test_Accuracy_Rate: 65.31%\n",
      "Epoch 497, Total Loss: 0.9552842974662781, Accuracy Rate: 63.25%,            Test_Total_Loss: 0.9108836650848389, Test_Accuracy_Rate: 65.85%\n",
      "Epoch 498, Total Loss: 0.9622989296913147, Accuracy Rate: 63.20%,            Test_Total_Loss: 0.9234462380409241, Test_Accuracy_Rate: 65.65%\n",
      "Epoch 499, Total Loss: 0.9692354798316956, Accuracy Rate: 62.83%,            Test_Total_Loss: 0.9206754565238953, Test_Accuracy_Rate: 65.77%\n",
      "Epoch 500, Total Loss: 0.96054607629776, Accuracy Rate: 63.70%,            Test_Total_Loss: 0.9105483889579773, Test_Accuracy_Rate: 65.77%\n",
      "Epoch 501, Total Loss: 0.9670544862747192, Accuracy Rate: 63.25%,            Test_Total_Loss: 0.9460880160331726, Test_Accuracy_Rate: 65.48%\n",
      "Epoch 502, Total Loss: 0.976814866065979, Accuracy Rate: 63.37%,            Test_Total_Loss: 0.9182459115982056, Test_Accuracy_Rate: 65.77%\n",
      "Epoch 503, Total Loss: 0.9628596901893616, Accuracy Rate: 63.16%,            Test_Total_Loss: 0.9147670269012451, Test_Accuracy_Rate: 66.18%\n",
      "Epoch 504, Total Loss: 0.9562674164772034, Accuracy Rate: 64.45%,            Test_Total_Loss: 0.9291783571243286, Test_Accuracy_Rate: 65.52%\n",
      "Epoch 505, Total Loss: 0.9684643149375916, Accuracy Rate: 63.33%,            Test_Total_Loss: 0.9071427583694458, Test_Accuracy_Rate: 66.18%\n",
      "Epoch 506, Total Loss: 0.9535161852836609, Accuracy Rate: 63.04%,            Test_Total_Loss: 0.9124240875244141, Test_Accuracy_Rate: 66.06%\n",
      "Epoch 507, Total Loss: 0.9550777077674866, Accuracy Rate: 64.20%,            Test_Total_Loss: 0.9024980664253235, Test_Accuracy_Rate: 65.89%\n",
      "Epoch 508, Total Loss: 0.9575839638710022, Accuracy Rate: 63.08%,            Test_Total_Loss: 0.9508687853813171, Test_Accuracy_Rate: 64.90%\n",
      "Epoch 509, Total Loss: 0.9652768969535828, Accuracy Rate: 63.04%,            Test_Total_Loss: 0.9331312775611877, Test_Accuracy_Rate: 64.94%\n",
      "Epoch 510, Total Loss: 0.9608827829360962, Accuracy Rate: 63.33%,            Test_Total_Loss: 0.9438610076904297, Test_Accuracy_Rate: 65.15%\n",
      "Epoch 511, Total Loss: 0.9806532859802246, Accuracy Rate: 63.99%,            Test_Total_Loss: 0.925625741481781, Test_Accuracy_Rate: 65.56%\n",
      "Epoch 512, Total Loss: 0.96359783411026, Accuracy Rate: 63.33%,            Test_Total_Loss: 0.9156451225280762, Test_Accuracy_Rate: 65.36%\n",
      "Epoch 513, Total Loss: 0.9590599536895752, Accuracy Rate: 63.49%,            Test_Total_Loss: 0.9083552956581116, Test_Accuracy_Rate: 66.06%\n",
      "Epoch 514, Total Loss: 0.9608833193778992, Accuracy Rate: 63.95%,            Test_Total_Loss: 1.0006641149520874, Test_Accuracy_Rate: 63.74%\n",
      "Epoch 515, Total Loss: 1.084905982017517, Accuracy Rate: 60.26%,            Test_Total_Loss: 1.0904085636138916, Test_Accuracy_Rate: 61.26%\n",
      "Epoch 516, Total Loss: 1.0798826217651367, Accuracy Rate: 61.01%,            Test_Total_Loss: 1.0203349590301514, Test_Accuracy_Rate: 63.58%\n",
      "Epoch 517, Total Loss: 1.110822319984436, Accuracy Rate: 61.67%,            Test_Total_Loss: 0.998176097869873, Test_Accuracy_Rate: 63.74%\n",
      "Epoch 518, Total Loss: 1.0211471319198608, Accuracy Rate: 62.38%,            Test_Total_Loss: 0.9853692054748535, Test_Accuracy_Rate: 64.11%\n",
      "Epoch 519, Total Loss: 1.0128141641616821, Accuracy Rate: 62.33%,            Test_Total_Loss: 0.9256696105003357, Test_Accuracy_Rate: 65.11%\n",
      "Epoch 520, Total Loss: 0.963344156742096, Accuracy Rate: 63.70%,            Test_Total_Loss: 0.9225916862487793, Test_Accuracy_Rate: 65.89%\n",
      "Epoch 521, Total Loss: 0.9556504487991333, Accuracy Rate: 64.07%,            Test_Total_Loss: 0.9181533455848694, Test_Accuracy_Rate: 65.56%\n",
      "Epoch 522, Total Loss: 0.987703800201416, Accuracy Rate: 63.20%,            Test_Total_Loss: 0.9573664665222168, Test_Accuracy_Rate: 64.82%\n",
      "Epoch 523, Total Loss: 0.983644962310791, Accuracy Rate: 63.04%,            Test_Total_Loss: 0.9231731295585632, Test_Accuracy_Rate: 65.65%\n",
      "Epoch 524, Total Loss: 0.9591054320335388, Accuracy Rate: 63.66%,            Test_Total_Loss: 0.9152922630310059, Test_Accuracy_Rate: 66.10%\n",
      "Epoch 525, Total Loss: 0.9610956311225891, Accuracy Rate: 63.45%,            Test_Total_Loss: 0.9379867911338806, Test_Accuracy_Rate: 65.40%\n",
      "Epoch 526, Total Loss: 0.9593449831008911, Accuracy Rate: 63.33%,            Test_Total_Loss: 0.908822238445282, Test_Accuracy_Rate: 66.10%\n",
      "Epoch 527, Total Loss: 0.9415898323059082, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.9164892435073853, Test_Accuracy_Rate: 65.73%\n",
      "Epoch 528, Total Loss: 0.9532057046890259, Accuracy Rate: 63.41%,            Test_Total_Loss: 0.9039472341537476, Test_Accuracy_Rate: 65.98%\n",
      "Epoch 529, Total Loss: 0.9453001618385315, Accuracy Rate: 63.70%,            Test_Total_Loss: 0.8980448842048645, Test_Accuracy_Rate: 66.10%\n",
      "Epoch 530, Total Loss: 0.9414467215538025, Accuracy Rate: 63.25%,            Test_Total_Loss: 0.9055994153022766, Test_Accuracy_Rate: 65.85%\n",
      "Epoch 531, Total Loss: 0.9456047415733337, Accuracy Rate: 63.41%,            Test_Total_Loss: 0.9050573110580444, Test_Accuracy_Rate: 65.89%\n",
      "Epoch 532, Total Loss: 0.9422773122787476, Accuracy Rate: 63.95%,            Test_Total_Loss: 0.9000346660614014, Test_Accuracy_Rate: 65.85%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 533, Total Loss: 0.9430439472198486, Accuracy Rate: 63.99%,            Test_Total_Loss: 0.899885892868042, Test_Accuracy_Rate: 66.47%\n",
      "Epoch 534, Total Loss: 0.9386928081512451, Accuracy Rate: 63.74%,            Test_Total_Loss: 0.89593905210495, Test_Accuracy_Rate: 66.27%\n",
      "Epoch 535, Total Loss: 0.9400408864021301, Accuracy Rate: 63.91%,            Test_Total_Loss: 0.9025027751922607, Test_Accuracy_Rate: 66.27%\n",
      "Epoch 536, Total Loss: 0.9395578503608704, Accuracy Rate: 64.03%,            Test_Total_Loss: 0.9014377593994141, Test_Accuracy_Rate: 66.06%\n",
      "Epoch 537, Total Loss: 0.9372209310531616, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.9037496447563171, Test_Accuracy_Rate: 65.94%\n",
      "Epoch 538, Total Loss: 0.9439678192138672, Accuracy Rate: 64.45%,            Test_Total_Loss: 0.8994299173355103, Test_Accuracy_Rate: 66.23%\n",
      "Epoch 539, Total Loss: 0.9480323791503906, Accuracy Rate: 63.95%,            Test_Total_Loss: 0.9208266735076904, Test_Accuracy_Rate: 65.98%\n",
      "Epoch 540, Total Loss: 0.9495088458061218, Accuracy Rate: 63.41%,            Test_Total_Loss: 0.9055498838424683, Test_Accuracy_Rate: 66.02%\n",
      "Epoch 541, Total Loss: 0.9458516836166382, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.9274155497550964, Test_Accuracy_Rate: 65.44%\n",
      "Epoch 542, Total Loss: 0.9489189982414246, Accuracy Rate: 63.25%,            Test_Total_Loss: 0.9279544353485107, Test_Accuracy_Rate: 65.73%\n",
      "Epoch 543, Total Loss: 0.9492639303207397, Accuracy Rate: 64.57%,            Test_Total_Loss: 0.9318091869354248, Test_Accuracy_Rate: 65.07%\n",
      "Epoch 544, Total Loss: 0.9482672810554504, Accuracy Rate: 63.78%,            Test_Total_Loss: 0.896113932132721, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 545, Total Loss: 0.9408978819847107, Accuracy Rate: 63.16%,            Test_Total_Loss: 0.8980678915977478, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 546, Total Loss: 0.9395061135292053, Accuracy Rate: 64.28%,            Test_Total_Loss: 0.9048630595207214, Test_Accuracy_Rate: 65.94%\n",
      "Epoch 547, Total Loss: 0.9430575370788574, Accuracy Rate: 63.99%,            Test_Total_Loss: 0.9003303647041321, Test_Accuracy_Rate: 65.98%\n",
      "Epoch 548, Total Loss: 0.9439162015914917, Accuracy Rate: 63.82%,            Test_Total_Loss: 0.8913010954856873, Test_Accuracy_Rate: 66.35%\n",
      "Epoch 549, Total Loss: 0.9364015460014343, Accuracy Rate: 63.95%,            Test_Total_Loss: 0.899350106716156, Test_Accuracy_Rate: 66.39%\n",
      "Epoch 550, Total Loss: 0.9353052377700806, Accuracy Rate: 63.99%,            Test_Total_Loss: 0.8896865844726562, Test_Accuracy_Rate: 66.23%\n",
      "Epoch 551, Total Loss: 0.9347026944160461, Accuracy Rate: 63.70%,            Test_Total_Loss: 0.893716037273407, Test_Accuracy_Rate: 66.39%\n",
      "Epoch 552, Total Loss: 0.9336830973625183, Accuracy Rate: 63.87%,            Test_Total_Loss: 0.9017493724822998, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 553, Total Loss: 0.9397910237312317, Accuracy Rate: 63.41%,            Test_Total_Loss: 0.9017935395240784, Test_Accuracy_Rate: 65.73%\n",
      "Epoch 554, Total Loss: 0.9425030946731567, Accuracy Rate: 63.58%,            Test_Total_Loss: 0.9005181193351746, Test_Accuracy_Rate: 66.18%\n",
      "Epoch 555, Total Loss: 0.936953604221344, Accuracy Rate: 63.87%,            Test_Total_Loss: 0.9020283222198486, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 556, Total Loss: 0.9387786388397217, Accuracy Rate: 63.99%,            Test_Total_Loss: 0.9011613726615906, Test_Accuracy_Rate: 66.27%\n",
      "Epoch 557, Total Loss: 0.9461420774459839, Accuracy Rate: 64.03%,            Test_Total_Loss: 0.907037615776062, Test_Accuracy_Rate: 65.98%\n",
      "Epoch 558, Total Loss: 0.9448493123054504, Accuracy Rate: 63.04%,            Test_Total_Loss: 0.8996987342834473, Test_Accuracy_Rate: 65.81%\n",
      "Epoch 559, Total Loss: 0.9470769762992859, Accuracy Rate: 63.74%,            Test_Total_Loss: 0.9092885255813599, Test_Accuracy_Rate: 65.77%\n",
      "Epoch 560, Total Loss: 0.9440234303474426, Accuracy Rate: 63.66%,            Test_Total_Loss: 0.9030189514160156, Test_Accuracy_Rate: 65.98%\n",
      "Epoch 561, Total Loss: 0.9377400279045105, Accuracy Rate: 63.49%,            Test_Total_Loss: 0.8976392149925232, Test_Accuracy_Rate: 66.06%\n",
      "Epoch 562, Total Loss: 0.9349063038825989, Accuracy Rate: 63.70%,            Test_Total_Loss: 0.8965815305709839, Test_Accuracy_Rate: 66.10%\n",
      "Epoch 563, Total Loss: 0.9371971487998962, Accuracy Rate: 63.66%,            Test_Total_Loss: 0.8952738046646118, Test_Accuracy_Rate: 66.02%\n",
      "Epoch 564, Total Loss: 0.9381306171417236, Accuracy Rate: 63.45%,            Test_Total_Loss: 0.9045027494430542, Test_Accuracy_Rate: 66.14%\n",
      "Epoch 565, Total Loss: 0.9567782878875732, Accuracy Rate: 63.62%,            Test_Total_Loss: 0.9094549417495728, Test_Accuracy_Rate: 66.35%\n",
      "Epoch 566, Total Loss: 0.9500154256820679, Accuracy Rate: 64.20%,            Test_Total_Loss: 0.9080721139907837, Test_Accuracy_Rate: 65.81%\n",
      "Epoch 567, Total Loss: 0.9434067606925964, Accuracy Rate: 64.24%,            Test_Total_Loss: 0.8923460245132446, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 568, Total Loss: 0.9317848682403564, Accuracy Rate: 63.91%,            Test_Total_Loss: 0.9005544781684875, Test_Accuracy_Rate: 66.27%\n",
      "Epoch 569, Total Loss: 0.9464115500450134, Accuracy Rate: 64.20%,            Test_Total_Loss: 0.9049390554428101, Test_Accuracy_Rate: 66.02%\n",
      "Epoch 570, Total Loss: 0.9438750743865967, Accuracy Rate: 63.70%,            Test_Total_Loss: 0.8882968425750732, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 571, Total Loss: 0.9375689625740051, Accuracy Rate: 63.82%,            Test_Total_Loss: 0.8885049223899841, Test_Accuracy_Rate: 66.68%\n",
      "Epoch 572, Total Loss: 0.9347007870674133, Accuracy Rate: 63.08%,            Test_Total_Loss: 0.8868058919906616, Test_Accuracy_Rate: 66.56%\n",
      "Epoch 573, Total Loss: 0.9344263672828674, Accuracy Rate: 65.11%,            Test_Total_Loss: 0.8893886208534241, Test_Accuracy_Rate: 66.64%\n",
      "Epoch 574, Total Loss: 0.9377105236053467, Accuracy Rate: 64.11%,            Test_Total_Loss: 0.8933444023132324, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 575, Total Loss: 0.9407880902290344, Accuracy Rate: 64.11%,            Test_Total_Loss: 0.8961818814277649, Test_Accuracy_Rate: 66.10%\n",
      "Epoch 576, Total Loss: 0.9419134259223938, Accuracy Rate: 63.49%,            Test_Total_Loss: 0.8925251364707947, Test_Accuracy_Rate: 66.02%\n",
      "Epoch 577, Total Loss: 0.936638355255127, Accuracy Rate: 63.91%,            Test_Total_Loss: 0.8959887623786926, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 578, Total Loss: 0.9350088834762573, Accuracy Rate: 64.24%,            Test_Total_Loss: 0.8925526142120361, Test_Accuracy_Rate: 66.10%\n",
      "Epoch 579, Total Loss: 0.9333622455596924, Accuracy Rate: 64.28%,            Test_Total_Loss: 0.883653461933136, Test_Accuracy_Rate: 66.27%\n",
      "Epoch 580, Total Loss: 0.9336609840393066, Accuracy Rate: 63.70%,            Test_Total_Loss: 0.8891079425811768, Test_Accuracy_Rate: 65.98%\n",
      "Epoch 581, Total Loss: 0.9328213930130005, Accuracy Rate: 63.74%,            Test_Total_Loss: 0.8875700235366821, Test_Accuracy_Rate: 66.14%\n",
      "Epoch 582, Total Loss: 0.9341716766357422, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8889991641044617, Test_Accuracy_Rate: 66.60%\n",
      "Epoch 583, Total Loss: 0.9396300315856934, Accuracy Rate: 64.49%,            Test_Total_Loss: 0.8972796201705933, Test_Accuracy_Rate: 66.18%\n",
      "Epoch 584, Total Loss: 0.9517426490783691, Accuracy Rate: 63.53%,            Test_Total_Loss: 0.9044007658958435, Test_Accuracy_Rate: 65.60%\n",
      "Epoch 585, Total Loss: 0.959145724773407, Accuracy Rate: 63.62%,            Test_Total_Loss: 0.9121944904327393, Test_Accuracy_Rate: 65.77%\n",
      "Epoch 586, Total Loss: 1.0299019813537598, Accuracy Rate: 62.42%,            Test_Total_Loss: 1.013983130455017, Test_Accuracy_Rate: 63.49%\n",
      "Epoch 587, Total Loss: 1.0519874095916748, Accuracy Rate: 61.63%,            Test_Total_Loss: 1.1818727254867554, Test_Accuracy_Rate: 57.49%\n",
      "Epoch 588, Total Loss: 1.1354728937149048, Accuracy Rate: 59.69%,            Test_Total_Loss: 1.0152699947357178, Test_Accuracy_Rate: 63.12%\n",
      "Epoch 589, Total Loss: 1.0666364431381226, Accuracy Rate: 61.05%,            Test_Total_Loss: 1.1630048751831055, Test_Accuracy_Rate: 60.18%\n",
      "Epoch 590, Total Loss: 1.0562041997909546, Accuracy Rate: 61.96%,            Test_Total_Loss: 0.9978315234184265, Test_Accuracy_Rate: 63.16%\n",
      "Epoch 591, Total Loss: 1.0132468938827515, Accuracy Rate: 62.46%,            Test_Total_Loss: 0.918838620185852, Test_Accuracy_Rate: 65.19%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 592, Total Loss: 0.9720994830131531, Accuracy Rate: 62.96%,            Test_Total_Loss: 0.9112275838851929, Test_Accuracy_Rate: 65.98%\n",
      "Epoch 593, Total Loss: 0.9514589905738831, Accuracy Rate: 64.11%,            Test_Total_Loss: 0.9222579002380371, Test_Accuracy_Rate: 65.98%\n",
      "Epoch 594, Total Loss: 0.9689064025878906, Accuracy Rate: 63.25%,            Test_Total_Loss: 0.9154261350631714, Test_Accuracy_Rate: 65.60%\n",
      "Epoch 595, Total Loss: 0.9652387499809265, Accuracy Rate: 63.74%,            Test_Total_Loss: 0.903826892375946, Test_Accuracy_Rate: 66.14%\n",
      "Epoch 596, Total Loss: 0.9475948810577393, Accuracy Rate: 63.74%,            Test_Total_Loss: 0.9146427512168884, Test_Accuracy_Rate: 65.65%\n",
      "Epoch 597, Total Loss: 0.9359660148620605, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.8914947509765625, Test_Accuracy_Rate: 66.39%\n",
      "Epoch 598, Total Loss: 0.9348447918891907, Accuracy Rate: 63.66%,            Test_Total_Loss: 0.8843904733657837, Test_Accuracy_Rate: 66.76%\n",
      "Epoch 599, Total Loss: 0.9370421171188354, Accuracy Rate: 64.16%,            Test_Total_Loss: 0.8994972109794617, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 600, Total Loss: 0.9399991035461426, Accuracy Rate: 64.20%,            Test_Total_Loss: 0.8902621865272522, Test_Accuracy_Rate: 66.68%\n",
      "Epoch 601, Total Loss: 0.9302904009819031, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8853765726089478, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 602, Total Loss: 0.928785502910614, Accuracy Rate: 63.99%,            Test_Total_Loss: 0.8857998847961426, Test_Accuracy_Rate: 66.56%\n",
      "Epoch 603, Total Loss: 0.9499261975288391, Accuracy Rate: 63.33%,            Test_Total_Loss: 0.9153688549995422, Test_Accuracy_Rate: 65.44%\n",
      "Epoch 604, Total Loss: 0.9424600005149841, Accuracy Rate: 63.95%,            Test_Total_Loss: 0.890752375125885, Test_Accuracy_Rate: 66.27%\n",
      "Epoch 605, Total Loss: 0.9268481731414795, Accuracy Rate: 64.20%,            Test_Total_Loss: 0.8822797536849976, Test_Accuracy_Rate: 66.47%\n",
      "Epoch 606, Total Loss: 0.9263740181922913, Accuracy Rate: 64.65%,            Test_Total_Loss: 0.8807023763656616, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 607, Total Loss: 0.9213842153549194, Accuracy Rate: 64.98%,            Test_Total_Loss: 0.8817405700683594, Test_Accuracy_Rate: 66.47%\n",
      "Epoch 608, Total Loss: 0.9230119585990906, Accuracy Rate: 64.20%,            Test_Total_Loss: 0.8836070895195007, Test_Accuracy_Rate: 66.51%\n",
      "Epoch 609, Total Loss: 0.9270650744438171, Accuracy Rate: 63.78%,            Test_Total_Loss: 0.8868462443351746, Test_Accuracy_Rate: 66.43%\n",
      "Epoch 610, Total Loss: 0.9305776357650757, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.8854960203170776, Test_Accuracy_Rate: 66.47%\n",
      "Epoch 611, Total Loss: 0.9273103475570679, Accuracy Rate: 64.07%,            Test_Total_Loss: 0.8833490610122681, Test_Accuracy_Rate: 66.10%\n",
      "Epoch 612, Total Loss: 0.9274042248725891, Accuracy Rate: 63.82%,            Test_Total_Loss: 0.8867513537406921, Test_Accuracy_Rate: 66.18%\n",
      "Epoch 613, Total Loss: 0.9296571016311646, Accuracy Rate: 63.74%,            Test_Total_Loss: 0.8935126066207886, Test_Accuracy_Rate: 65.40%\n",
      "Epoch 614, Total Loss: 0.9281215667724609, Accuracy Rate: 63.91%,            Test_Total_Loss: 0.8827371001243591, Test_Accuracy_Rate: 66.47%\n",
      "Epoch 615, Total Loss: 0.9182055592536926, Accuracy Rate: 64.28%,            Test_Total_Loss: 0.8803264498710632, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 616, Total Loss: 0.9182707071304321, Accuracy Rate: 64.28%,            Test_Total_Loss: 0.8925794363021851, Test_Accuracy_Rate: 66.43%\n",
      "Epoch 617, Total Loss: 0.9253261685371399, Accuracy Rate: 64.11%,            Test_Total_Loss: 0.9007150530815125, Test_Accuracy_Rate: 65.81%\n",
      "Epoch 618, Total Loss: 0.9292521476745605, Accuracy Rate: 63.87%,            Test_Total_Loss: 0.8942772746086121, Test_Accuracy_Rate: 66.23%\n",
      "Epoch 619, Total Loss: 0.9231101870536804, Accuracy Rate: 64.28%,            Test_Total_Loss: 0.8805968761444092, Test_Accuracy_Rate: 66.18%\n",
      "Epoch 620, Total Loss: 0.924216091632843, Accuracy Rate: 63.70%,            Test_Total_Loss: 0.8766862154006958, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 621, Total Loss: 0.9197303652763367, Accuracy Rate: 64.11%,            Test_Total_Loss: 0.8838801383972168, Test_Accuracy_Rate: 66.64%\n",
      "Epoch 622, Total Loss: 0.9688602089881897, Accuracy Rate: 64.16%,            Test_Total_Loss: 0.9564537405967712, Test_Accuracy_Rate: 64.78%\n",
      "Epoch 623, Total Loss: 0.9594766497612, Accuracy Rate: 64.03%,            Test_Total_Loss: 1.2026182413101196, Test_Accuracy_Rate: 58.49%\n",
      "Epoch 624, Total Loss: 1.1260507106781006, Accuracy Rate: 60.64%,            Test_Total_Loss: 3.101191520690918, Test_Accuracy_Rate: 41.35%\n",
      "Epoch 625, Total Loss: 2.3745148181915283, Accuracy Rate: 42.59%,            Test_Total_Loss: 1.7273744344711304, Test_Accuracy_Rate: 48.39%\n",
      "Epoch 626, Total Loss: 1.589218020439148, Accuracy Rate: 50.37%,            Test_Total_Loss: 1.3689450025558472, Test_Accuracy_Rate: 55.50%\n",
      "Epoch 627, Total Loss: 1.239112377166748, Accuracy Rate: 56.83%,            Test_Total_Loss: 1.223801612854004, Test_Accuracy_Rate: 58.86%\n",
      "Epoch 628, Total Loss: 1.1353685855865479, Accuracy Rate: 60.31%,            Test_Total_Loss: 1.0065441131591797, Test_Accuracy_Rate: 63.58%\n",
      "Epoch 629, Total Loss: 1.0430505275726318, Accuracy Rate: 61.26%,            Test_Total_Loss: 0.9565365314483643, Test_Accuracy_Rate: 64.65%\n",
      "Epoch 630, Total Loss: 1.0158357620239258, Accuracy Rate: 62.25%,            Test_Total_Loss: 0.952383816242218, Test_Accuracy_Rate: 64.82%\n",
      "Epoch 631, Total Loss: 1.0540516376495361, Accuracy Rate: 62.00%,            Test_Total_Loss: 1.0011053085327148, Test_Accuracy_Rate: 64.07%\n",
      "Epoch 632, Total Loss: 1.008349061012268, Accuracy Rate: 63.41%,            Test_Total_Loss: 0.9375520944595337, Test_Accuracy_Rate: 65.07%\n",
      "Epoch 633, Total Loss: 0.9597857594490051, Accuracy Rate: 63.87%,            Test_Total_Loss: 0.9100183248519897, Test_Accuracy_Rate: 66.06%\n",
      "Epoch 634, Total Loss: 0.9364110827445984, Accuracy Rate: 64.65%,            Test_Total_Loss: 0.8990871906280518, Test_Accuracy_Rate: 66.43%\n",
      "Epoch 635, Total Loss: 0.9330188632011414, Accuracy Rate: 64.45%,            Test_Total_Loss: 0.8962783813476562, Test_Accuracy_Rate: 66.51%\n",
      "Epoch 636, Total Loss: 0.9272372722625732, Accuracy Rate: 64.49%,            Test_Total_Loss: 0.8864349722862244, Test_Accuracy_Rate: 66.64%\n",
      "Epoch 637, Total Loss: 0.9275835752487183, Accuracy Rate: 64.36%,            Test_Total_Loss: 0.8915790915489197, Test_Accuracy_Rate: 66.60%\n",
      "Epoch 638, Total Loss: 0.9265195727348328, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.8935484886169434, Test_Accuracy_Rate: 66.27%\n",
      "Epoch 639, Total Loss: 0.9270069599151611, Accuracy Rate: 64.16%,            Test_Total_Loss: 0.8866063356399536, Test_Accuracy_Rate: 66.47%\n",
      "Epoch 640, Total Loss: 0.922374427318573, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8837223052978516, Test_Accuracy_Rate: 66.76%\n",
      "Epoch 641, Total Loss: 0.9189448356628418, Accuracy Rate: 64.16%,            Test_Total_Loss: 0.8870628476142883, Test_Accuracy_Rate: 66.60%\n",
      "Epoch 642, Total Loss: 0.920426607131958, Accuracy Rate: 64.11%,            Test_Total_Loss: 0.8815311789512634, Test_Accuracy_Rate: 66.56%\n",
      "Epoch 643, Total Loss: 0.9174772500991821, Accuracy Rate: 64.20%,            Test_Total_Loss: 0.8781657218933105, Test_Accuracy_Rate: 66.68%\n",
      "Epoch 644, Total Loss: 0.9107065796852112, Accuracy Rate: 64.57%,            Test_Total_Loss: 0.879533052444458, Test_Accuracy_Rate: 66.56%\n",
      "Epoch 645, Total Loss: 0.9119147062301636, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.878232479095459, Test_Accuracy_Rate: 66.64%\n",
      "Epoch 646, Total Loss: 0.9142877459526062, Accuracy Rate: 64.32%,            Test_Total_Loss: 0.8770867586135864, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 647, Total Loss: 0.9143808484077454, Accuracy Rate: 64.57%,            Test_Total_Loss: 0.8839990496635437, Test_Accuracy_Rate: 66.51%\n",
      "Epoch 648, Total Loss: 0.9168605804443359, Accuracy Rate: 64.28%,            Test_Total_Loss: 0.8817546367645264, Test_Accuracy_Rate: 66.76%\n",
      "Epoch 649, Total Loss: 0.9151062369346619, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8864238262176514, Test_Accuracy_Rate: 66.60%\n",
      "Epoch 650, Total Loss: 0.9171892404556274, Accuracy Rate: 64.61%,            Test_Total_Loss: 0.8770092129707336, Test_Accuracy_Rate: 66.93%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 651, Total Loss: 0.9168444275856018, Accuracy Rate: 64.28%,            Test_Total_Loss: 0.8837749361991882, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 652, Total Loss: 0.9112868905067444, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.8775737285614014, Test_Accuracy_Rate: 66.60%\n",
      "Epoch 653, Total Loss: 0.9169129729270935, Accuracy Rate: 64.32%,            Test_Total_Loss: 0.8754386901855469, Test_Accuracy_Rate: 66.80%\n",
      "Epoch 654, Total Loss: 0.9110535979270935, Accuracy Rate: 64.69%,            Test_Total_Loss: 0.8750954866409302, Test_Accuracy_Rate: 66.76%\n",
      "Epoch 655, Total Loss: 0.9128136038780212, Accuracy Rate: 64.20%,            Test_Total_Loss: 0.8787015676498413, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 656, Total Loss: 0.914405107498169, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8799636363983154, Test_Accuracy_Rate: 66.56%\n",
      "Epoch 657, Total Loss: 0.9127860069274902, Accuracy Rate: 64.69%,            Test_Total_Loss: 0.8774879574775696, Test_Accuracy_Rate: 66.60%\n",
      "Epoch 658, Total Loss: 0.9104533195495605, Accuracy Rate: 63.99%,            Test_Total_Loss: 0.8790233135223389, Test_Accuracy_Rate: 66.47%\n",
      "Epoch 659, Total Loss: 0.9096338152885437, Accuracy Rate: 64.28%,            Test_Total_Loss: 0.8711923360824585, Test_Accuracy_Rate: 66.97%\n",
      "Epoch 660, Total Loss: 0.9124371409416199, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.875068187713623, Test_Accuracy_Rate: 66.76%\n",
      "Epoch 661, Total Loss: 0.9126184582710266, Accuracy Rate: 64.65%,            Test_Total_Loss: 0.8716414570808411, Test_Accuracy_Rate: 66.93%\n",
      "Epoch 662, Total Loss: 0.9109271168708801, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.8752660155296326, Test_Accuracy_Rate: 66.35%\n",
      "Epoch 663, Total Loss: 0.9188522696495056, Accuracy Rate: 64.16%,            Test_Total_Loss: 0.8728318214416504, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 664, Total Loss: 0.9123887419700623, Accuracy Rate: 64.16%,            Test_Total_Loss: 0.8718263506889343, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 665, Total Loss: 0.9085193276405334, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8696845173835754, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 666, Total Loss: 0.9077674746513367, Accuracy Rate: 64.45%,            Test_Total_Loss: 0.8687348365783691, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 667, Total Loss: 0.9060552716255188, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.8673495054244995, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 668, Total Loss: 0.9084855914115906, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8756905794143677, Test_Accuracy_Rate: 66.97%\n",
      "Epoch 669, Total Loss: 0.9160459637641907, Accuracy Rate: 63.78%,            Test_Total_Loss: 0.8767839670181274, Test_Accuracy_Rate: 66.80%\n",
      "Epoch 670, Total Loss: 0.9160935282707214, Accuracy Rate: 63.74%,            Test_Total_Loss: 0.8927422761917114, Test_Accuracy_Rate: 66.56%\n",
      "Epoch 671, Total Loss: 0.9123889803886414, Accuracy Rate: 64.61%,            Test_Total_Loss: 0.874077558517456, Test_Accuracy_Rate: 66.60%\n",
      "Epoch 672, Total Loss: 0.9179099202156067, Accuracy Rate: 64.32%,            Test_Total_Loss: 0.8714332580566406, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 673, Total Loss: 0.9219164848327637, Accuracy Rate: 64.65%,            Test_Total_Loss: 0.8737468719482422, Test_Accuracy_Rate: 66.80%\n",
      "Epoch 674, Total Loss: 0.9128215312957764, Accuracy Rate: 64.20%,            Test_Total_Loss: 0.8687170743942261, Test_Accuracy_Rate: 66.80%\n",
      "Epoch 675, Total Loss: 0.9122851490974426, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.8849218487739563, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 676, Total Loss: 0.9156140685081482, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.9065670371055603, Test_Accuracy_Rate: 65.94%\n",
      "Epoch 677, Total Loss: 0.9271680116653442, Accuracy Rate: 63.62%,            Test_Total_Loss: 0.8885204195976257, Test_Accuracy_Rate: 66.43%\n",
      "Epoch 678, Total Loss: 0.9314547181129456, Accuracy Rate: 64.20%,            Test_Total_Loss: 0.9089595675468445, Test_Accuracy_Rate: 65.52%\n",
      "Epoch 679, Total Loss: 0.9282411336898804, Accuracy Rate: 64.20%,            Test_Total_Loss: 0.989006519317627, Test_Accuracy_Rate: 63.87%\n",
      "Epoch 680, Total Loss: 0.935708224773407, Accuracy Rate: 64.20%,            Test_Total_Loss: 0.8834210634231567, Test_Accuracy_Rate: 66.56%\n",
      "Epoch 681, Total Loss: 0.9195713400840759, Accuracy Rate: 64.20%,            Test_Total_Loss: 0.874580979347229, Test_Accuracy_Rate: 66.39%\n",
      "Epoch 682, Total Loss: 0.9206839203834534, Accuracy Rate: 64.03%,            Test_Total_Loss: 0.8744392395019531, Test_Accuracy_Rate: 66.80%\n",
      "Epoch 683, Total Loss: 0.9187658429145813, Accuracy Rate: 64.32%,            Test_Total_Loss: 0.8796072602272034, Test_Accuracy_Rate: 66.60%\n",
      "Epoch 684, Total Loss: 0.9199571013450623, Accuracy Rate: 64.11%,            Test_Total_Loss: 0.8896817564964294, Test_Accuracy_Rate: 66.60%\n",
      "Epoch 685, Total Loss: 0.9111406207084656, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.8712977170944214, Test_Accuracy_Rate: 66.80%\n",
      "Epoch 686, Total Loss: 0.9083157777786255, Accuracy Rate: 64.69%,            Test_Total_Loss: 0.869817316532135, Test_Accuracy_Rate: 66.80%\n",
      "Epoch 687, Total Loss: 0.9167735576629639, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.8707177042961121, Test_Accuracy_Rate: 66.43%\n",
      "Epoch 688, Total Loss: 0.9106304049491882, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.8672128915786743, Test_Accuracy_Rate: 67.14%\n",
      "Epoch 689, Total Loss: 0.9077677726745605, Accuracy Rate: 64.65%,            Test_Total_Loss: 0.8678873777389526, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 690, Total Loss: 0.9038468599319458, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8706015348434448, Test_Accuracy_Rate: 66.60%\n",
      "Epoch 691, Total Loss: 0.9062711000442505, Accuracy Rate: 64.16%,            Test_Total_Loss: 0.8772587180137634, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 692, Total Loss: 0.9147520661354065, Accuracy Rate: 64.57%,            Test_Total_Loss: 0.8767028450965881, Test_Accuracy_Rate: 66.47%\n",
      "Epoch 693, Total Loss: 0.9105512499809265, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8745567202568054, Test_Accuracy_Rate: 66.64%\n",
      "Epoch 694, Total Loss: 0.9122503399848938, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.870985746383667, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 695, Total Loss: 0.9115746021270752, Accuracy Rate: 64.16%,            Test_Total_Loss: 0.8650020956993103, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 696, Total Loss: 0.9159561991691589, Accuracy Rate: 63.58%,            Test_Total_Loss: 0.8760204315185547, Test_Accuracy_Rate: 66.80%\n",
      "Epoch 697, Total Loss: 0.905266523361206, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8734732270240784, Test_Accuracy_Rate: 66.68%\n",
      "Epoch 698, Total Loss: 0.9135780334472656, Accuracy Rate: 63.91%,            Test_Total_Loss: 0.8652052879333496, Test_Accuracy_Rate: 66.93%\n",
      "Epoch 699, Total Loss: 0.9141748547554016, Accuracy Rate: 64.36%,            Test_Total_Loss: 0.8771926760673523, Test_Accuracy_Rate: 66.64%\n",
      "Epoch 700, Total Loss: 0.9064800143241882, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8819347023963928, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 701, Total Loss: 0.9105755686759949, Accuracy Rate: 64.03%,            Test_Total_Loss: 0.8752966523170471, Test_Accuracy_Rate: 66.51%\n",
      "Epoch 702, Total Loss: 0.9185617566108704, Accuracy Rate: 64.49%,            Test_Total_Loss: 0.8758602142333984, Test_Accuracy_Rate: 66.89%\n",
      "Epoch 703, Total Loss: 0.9134190082550049, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8682918548583984, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 704, Total Loss: 0.910727858543396, Accuracy Rate: 64.28%,            Test_Total_Loss: 0.8770748972892761, Test_Accuracy_Rate: 66.68%\n",
      "Epoch 705, Total Loss: 0.9187474250793457, Accuracy Rate: 63.87%,            Test_Total_Loss: 0.8739826083183289, Test_Accuracy_Rate: 66.43%\n",
      "Epoch 706, Total Loss: 0.9139068722724915, Accuracy Rate: 63.53%,            Test_Total_Loss: 0.8750708699226379, Test_Accuracy_Rate: 66.89%\n",
      "Epoch 707, Total Loss: 0.9088330268859863, Accuracy Rate: 64.36%,            Test_Total_Loss: 0.8752398490905762, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 708, Total Loss: 0.9194160103797913, Accuracy Rate: 63.78%,            Test_Total_Loss: 0.8895494341850281, Test_Accuracy_Rate: 66.27%\n",
      "Epoch 709, Total Loss: 0.9270763397216797, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.8960478901863098, Test_Accuracy_Rate: 66.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 710, Total Loss: 0.9343151450157166, Accuracy Rate: 64.07%,            Test_Total_Loss: 0.9216104745864868, Test_Accuracy_Rate: 65.31%\n",
      "Epoch 711, Total Loss: 0.9250052571296692, Accuracy Rate: 63.70%,            Test_Total_Loss: 0.8806009888648987, Test_Accuracy_Rate: 66.68%\n",
      "Epoch 712, Total Loss: 0.9228602051734924, Accuracy Rate: 64.49%,            Test_Total_Loss: 0.8781151175498962, Test_Accuracy_Rate: 66.10%\n",
      "Epoch 713, Total Loss: 0.9203665852546692, Accuracy Rate: 63.53%,            Test_Total_Loss: 0.8834910988807678, Test_Accuracy_Rate: 66.47%\n",
      "Epoch 714, Total Loss: 0.9175354838371277, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8630539774894714, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 715, Total Loss: 0.9110351204872131, Accuracy Rate: 64.65%,            Test_Total_Loss: 0.876434326171875, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 716, Total Loss: 0.9158821105957031, Accuracy Rate: 63.53%,            Test_Total_Loss: 0.8631452322006226, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 717, Total Loss: 0.9111868143081665, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8969982266426086, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 718, Total Loss: 0.9321856498718262, Accuracy Rate: 63.82%,            Test_Total_Loss: 0.8814355731010437, Test_Accuracy_Rate: 66.56%\n",
      "Epoch 719, Total Loss: 0.918003261089325, Accuracy Rate: 64.61%,            Test_Total_Loss: 0.8848361968994141, Test_Accuracy_Rate: 66.43%\n",
      "Epoch 720, Total Loss: 0.9160235524177551, Accuracy Rate: 64.45%,            Test_Total_Loss: 0.8839290738105774, Test_Accuracy_Rate: 66.10%\n",
      "Epoch 721, Total Loss: 0.9196788668632507, Accuracy Rate: 64.32%,            Test_Total_Loss: 0.8826925158500671, Test_Accuracy_Rate: 66.39%\n",
      "Epoch 722, Total Loss: 0.9329543709754944, Accuracy Rate: 63.99%,            Test_Total_Loss: 0.8754512667655945, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 723, Total Loss: 0.9629914164543152, Accuracy Rate: 63.70%,            Test_Total_Loss: 1.0156991481781006, Test_Accuracy_Rate: 62.13%\n",
      "Epoch 724, Total Loss: 0.9735475778579712, Accuracy Rate: 63.12%,            Test_Total_Loss: 0.8894794583320618, Test_Accuracy_Rate: 66.35%\n",
      "Epoch 725, Total Loss: 0.9409101605415344, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.9042119383811951, Test_Accuracy_Rate: 65.77%\n",
      "Epoch 726, Total Loss: 0.939153254032135, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8968790173530579, Test_Accuracy_Rate: 66.18%\n",
      "Epoch 727, Total Loss: 0.9235037565231323, Accuracy Rate: 64.20%,            Test_Total_Loss: 0.8700132966041565, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 728, Total Loss: 0.9105511903762817, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8693423867225647, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 729, Total Loss: 0.9108578562736511, Accuracy Rate: 64.65%,            Test_Total_Loss: 0.863949716091156, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 730, Total Loss: 0.9074491262435913, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.8695388436317444, Test_Accuracy_Rate: 66.76%\n",
      "Epoch 731, Total Loss: 0.9099775552749634, Accuracy Rate: 64.57%,            Test_Total_Loss: 0.8751327395439148, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 732, Total Loss: 0.9129418730735779, Accuracy Rate: 63.99%,            Test_Total_Loss: 0.8659778237342834, Test_Accuracy_Rate: 66.97%\n",
      "Epoch 733, Total Loss: 0.9123623371124268, Accuracy Rate: 63.99%,            Test_Total_Loss: 0.8650015592575073, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 734, Total Loss: 0.9105364084243774, Accuracy Rate: 64.36%,            Test_Total_Loss: 0.8616876006126404, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 735, Total Loss: 0.9132524728775024, Accuracy Rate: 64.16%,            Test_Total_Loss: 0.8882280588150024, Test_Accuracy_Rate: 66.10%\n",
      "Epoch 736, Total Loss: 0.9259029030799866, Accuracy Rate: 64.45%,            Test_Total_Loss: 0.8817505240440369, Test_Accuracy_Rate: 66.76%\n",
      "Epoch 737, Total Loss: 0.9225884675979614, Accuracy Rate: 64.24%,            Test_Total_Loss: 0.8791266083717346, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 738, Total Loss: 0.923270046710968, Accuracy Rate: 64.03%,            Test_Total_Loss: 0.8761313557624817, Test_Accuracy_Rate: 66.68%\n",
      "Epoch 739, Total Loss: 0.927756667137146, Accuracy Rate: 64.69%,            Test_Total_Loss: 0.8702850341796875, Test_Accuracy_Rate: 66.56%\n",
      "Epoch 740, Total Loss: 0.9154098033905029, Accuracy Rate: 64.49%,            Test_Total_Loss: 0.8724180459976196, Test_Accuracy_Rate: 66.97%\n",
      "Epoch 741, Total Loss: 0.9165146350860596, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8664449453353882, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 742, Total Loss: 1.1155513525009155, Accuracy Rate: 61.30%,            Test_Total_Loss: 1.1271048784255981, Test_Accuracy_Rate: 61.01%\n",
      "Epoch 743, Total Loss: 1.112672209739685, Accuracy Rate: 60.10%,            Test_Total_Loss: 0.996596097946167, Test_Accuracy_Rate: 63.70%\n",
      "Epoch 744, Total Loss: 1.0096160173416138, Accuracy Rate: 62.67%,            Test_Total_Loss: 0.9126249551773071, Test_Accuracy_Rate: 66.14%\n",
      "Epoch 745, Total Loss: 0.9469199180603027, Accuracy Rate: 64.45%,            Test_Total_Loss: 0.9165400266647339, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 746, Total Loss: 0.9531310200691223, Accuracy Rate: 64.49%,            Test_Total_Loss: 0.8995975852012634, Test_Accuracy_Rate: 66.27%\n",
      "Epoch 747, Total Loss: 0.956007719039917, Accuracy Rate: 63.20%,            Test_Total_Loss: 0.9030613899230957, Test_Accuracy_Rate: 65.89%\n",
      "Epoch 748, Total Loss: 0.9308708310127258, Accuracy Rate: 63.91%,            Test_Total_Loss: 0.8880458474159241, Test_Accuracy_Rate: 66.51%\n",
      "Epoch 749, Total Loss: 0.934066891670227, Accuracy Rate: 63.74%,            Test_Total_Loss: 0.8811551928520203, Test_Accuracy_Rate: 66.60%\n",
      "Epoch 750, Total Loss: 0.9251337647438049, Accuracy Rate: 64.16%,            Test_Total_Loss: 0.8782219886779785, Test_Accuracy_Rate: 66.60%\n",
      "Epoch 751, Total Loss: 0.9109761118888855, Accuracy Rate: 64.24%,            Test_Total_Loss: 0.8664496541023254, Test_Accuracy_Rate: 66.89%\n",
      "Epoch 752, Total Loss: 0.9050042033195496, Accuracy Rate: 64.98%,            Test_Total_Loss: 0.8723904490470886, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 753, Total Loss: 0.9100471138954163, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8705991506576538, Test_Accuracy_Rate: 66.80%\n",
      "Epoch 754, Total Loss: 0.9080828428268433, Accuracy Rate: 64.36%,            Test_Total_Loss: 0.8839746713638306, Test_Accuracy_Rate: 66.18%\n",
      "Epoch 755, Total Loss: 0.9181243777275085, Accuracy Rate: 64.32%,            Test_Total_Loss: 0.8797903656959534, Test_Accuracy_Rate: 66.27%\n",
      "Epoch 756, Total Loss: 0.9121477007865906, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.8770826458930969, Test_Accuracy_Rate: 66.76%\n",
      "Epoch 757, Total Loss: 0.9115814566612244, Accuracy Rate: 64.69%,            Test_Total_Loss: 0.8669599294662476, Test_Accuracy_Rate: 66.89%\n",
      "Epoch 758, Total Loss: 0.9154649972915649, Accuracy Rate: 64.65%,            Test_Total_Loss: 0.8698835968971252, Test_Accuracy_Rate: 66.93%\n",
      "Epoch 759, Total Loss: 0.9075747132301331, Accuracy Rate: 64.45%,            Test_Total_Loss: 0.8631637692451477, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 760, Total Loss: 0.9004455208778381, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8601606488227844, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 761, Total Loss: 0.8999149203300476, Accuracy Rate: 64.36%,            Test_Total_Loss: 0.862464189529419, Test_Accuracy_Rate: 66.93%\n",
      "Epoch 762, Total Loss: 0.9033927917480469, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.8635650277137756, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 763, Total Loss: 0.9016892313957214, Accuracy Rate: 64.49%,            Test_Total_Loss: 0.8615091443061829, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 764, Total Loss: 0.9004520773887634, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.8631781339645386, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 765, Total Loss: 0.9069650769233704, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8617323637008667, Test_Accuracy_Rate: 66.76%\n",
      "Epoch 766, Total Loss: 0.9046899080276489, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8640367388725281, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 767, Total Loss: 0.9056143760681152, Accuracy Rate: 64.32%,            Test_Total_Loss: 0.8736620545387268, Test_Accuracy_Rate: 66.89%\n",
      "Epoch 768, Total Loss: 0.9174087047576904, Accuracy Rate: 64.69%,            Test_Total_Loss: 0.8716175556182861, Test_Accuracy_Rate: 66.51%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 769, Total Loss: 0.9053276181221008, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8673415184020996, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 770, Total Loss: 0.9043672680854797, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.8638663291931152, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 771, Total Loss: 0.9076234102249146, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.8601858019828796, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 772, Total Loss: 0.9352761507034302, Accuracy Rate: 63.91%,            Test_Total_Loss: 0.9167333841323853, Test_Accuracy_Rate: 65.81%\n",
      "Epoch 773, Total Loss: 0.9651858806610107, Accuracy Rate: 63.00%,            Test_Total_Loss: 0.917905330657959, Test_Accuracy_Rate: 65.44%\n",
      "Epoch 774, Total Loss: 0.9485245943069458, Accuracy Rate: 63.82%,            Test_Total_Loss: 0.8995538949966431, Test_Accuracy_Rate: 66.02%\n",
      "Epoch 775, Total Loss: 0.928494930267334, Accuracy Rate: 64.45%,            Test_Total_Loss: 0.8821896910667419, Test_Accuracy_Rate: 66.18%\n",
      "Epoch 776, Total Loss: 0.9156416654586792, Accuracy Rate: 64.03%,            Test_Total_Loss: 0.8616973757743835, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 777, Total Loss: 0.9056839346885681, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8627622723579407, Test_Accuracy_Rate: 67.14%\n",
      "Epoch 778, Total Loss: 0.9031643271446228, Accuracy Rate: 64.07%,            Test_Total_Loss: 0.8611839413642883, Test_Accuracy_Rate: 66.76%\n",
      "Epoch 779, Total Loss: 0.9011949300765991, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8643158078193665, Test_Accuracy_Rate: 66.97%\n",
      "Epoch 780, Total Loss: 0.9045135974884033, Accuracy Rate: 64.57%,            Test_Total_Loss: 0.8562623262405396, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 781, Total Loss: 0.8995866179466248, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.8592498302459717, Test_Accuracy_Rate: 66.97%\n",
      "Epoch 782, Total Loss: 0.9001743793487549, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8722800016403198, Test_Accuracy_Rate: 66.93%\n",
      "Epoch 783, Total Loss: 0.9069455862045288, Accuracy Rate: 64.57%,            Test_Total_Loss: 0.8580923676490784, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 784, Total Loss: 0.8984452486038208, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.8629258871078491, Test_Accuracy_Rate: 66.68%\n",
      "Epoch 785, Total Loss: 0.9018359780311584, Accuracy Rate: 64.69%,            Test_Total_Loss: 0.8597018718719482, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 786, Total Loss: 0.9075438976287842, Accuracy Rate: 64.07%,            Test_Total_Loss: 0.8673021197319031, Test_Accuracy_Rate: 66.89%\n",
      "Epoch 787, Total Loss: 0.9055736064910889, Accuracy Rate: 64.36%,            Test_Total_Loss: 0.862698495388031, Test_Accuracy_Rate: 67.14%\n",
      "Epoch 788, Total Loss: 0.9043253660202026, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8660960793495178, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 789, Total Loss: 0.9034959673881531, Accuracy Rate: 64.32%,            Test_Total_Loss: 0.8596490621566772, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 790, Total Loss: 0.8950191140174866, Accuracy Rate: 64.61%,            Test_Total_Loss: 0.8601508140563965, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 791, Total Loss: 0.8993290066719055, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8576319813728333, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 792, Total Loss: 0.8991038799285889, Accuracy Rate: 64.11%,            Test_Total_Loss: 0.8614086508750916, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 793, Total Loss: 0.9043325185775757, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.853887677192688, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 794, Total Loss: 0.8976410031318665, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.8614692687988281, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 795, Total Loss: 0.8976871967315674, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.873520016670227, Test_Accuracy_Rate: 66.60%\n",
      "Epoch 796, Total Loss: 0.910982608795166, Accuracy Rate: 64.61%,            Test_Total_Loss: 0.8886950612068176, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 797, Total Loss: 0.9794508218765259, Accuracy Rate: 64.40%,            Test_Total_Loss: 1.072462797164917, Test_Accuracy_Rate: 62.21%\n",
      "Epoch 798, Total Loss: 1.1925145387649536, Accuracy Rate: 59.48%,            Test_Total_Loss: 1.0862438678741455, Test_Accuracy_Rate: 62.42%\n",
      "Epoch 799, Total Loss: 1.1070178747177124, Accuracy Rate: 61.18%,            Test_Total_Loss: 1.0389938354492188, Test_Accuracy_Rate: 62.00%\n",
      "Epoch 800, Total Loss: 1.0055832862854004, Accuracy Rate: 62.96%,            Test_Total_Loss: 0.9225330352783203, Test_Accuracy_Rate: 65.69%\n",
      "Epoch 801, Total Loss: 0.9709495902061462, Accuracy Rate: 63.08%,            Test_Total_Loss: 0.9371920228004456, Test_Accuracy_Rate: 65.31%\n",
      "Epoch 802, Total Loss: 0.9690985679626465, Accuracy Rate: 63.45%,            Test_Total_Loss: 0.909976065158844, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 803, Total Loss: 0.9335529208183289, Accuracy Rate: 64.45%,            Test_Total_Loss: 0.8741286396980286, Test_Accuracy_Rate: 66.80%\n",
      "Epoch 804, Total Loss: 0.910137951374054, Accuracy Rate: 64.61%,            Test_Total_Loss: 0.8646038174629211, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 805, Total Loss: 0.9021958708763123, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.857704758644104, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 806, Total Loss: 0.8941145539283752, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8586791157722473, Test_Accuracy_Rate: 66.89%\n",
      "Epoch 807, Total Loss: 0.8964673280715942, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8610186576843262, Test_Accuracy_Rate: 66.76%\n",
      "Epoch 808, Total Loss: 0.8940942883491516, Accuracy Rate: 64.24%,            Test_Total_Loss: 0.8566107749938965, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 809, Total Loss: 0.8986791372299194, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8584794402122498, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 810, Total Loss: 0.8914504647254944, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.856532871723175, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 811, Total Loss: 0.9014455676078796, Accuracy Rate: 64.45%,            Test_Total_Loss: 0.8637545704841614, Test_Accuracy_Rate: 66.56%\n",
      "Epoch 812, Total Loss: 0.8992594480514526, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.8572818636894226, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 813, Total Loss: 0.908469557762146, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8616792559623718, Test_Accuracy_Rate: 66.97%\n",
      "Epoch 814, Total Loss: 0.9012991189956665, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8577682971954346, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 815, Total Loss: 0.8952094912528992, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8561945557594299, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 816, Total Loss: 0.8962115049362183, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8586477041244507, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 817, Total Loss: 0.8939703106880188, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.850896954536438, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 818, Total Loss: 0.8908548951148987, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.8533523678779602, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 819, Total Loss: 0.8930781483650208, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8551155924797058, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 820, Total Loss: 0.8942463397979736, Accuracy Rate: 64.03%,            Test_Total_Loss: 0.8520388603210449, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 821, Total Loss: 0.8868110179901123, Accuracy Rate: 64.65%,            Test_Total_Loss: 0.8574724793434143, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 822, Total Loss: 0.8915629386901855, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8570107817649841, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 823, Total Loss: 0.8882207274436951, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8595952391624451, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 824, Total Loss: 0.8954992890357971, Accuracy Rate: 64.16%,            Test_Total_Loss: 0.8575132489204407, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 825, Total Loss: 0.89825439453125, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.8628954887390137, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 826, Total Loss: 0.9013257026672363, Accuracy Rate: 64.57%,            Test_Total_Loss: 0.8573189973831177, Test_Accuracy_Rate: 67.14%\n",
      "Epoch 827, Total Loss: 0.8946517109870911, Accuracy Rate: 64.69%,            Test_Total_Loss: 0.8489797711372375, Test_Accuracy_Rate: 67.63%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 828, Total Loss: 0.8962151408195496, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.8563979268074036, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 829, Total Loss: 0.8913987874984741, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8534422516822815, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 830, Total Loss: 0.8988974690437317, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.8541502952575684, Test_Accuracy_Rate: 66.80%\n",
      "Epoch 831, Total Loss: 0.8911464214324951, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8563939929008484, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 832, Total Loss: 0.893913745880127, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.9145869016647339, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 833, Total Loss: 0.9197573661804199, Accuracy Rate: 64.11%,            Test_Total_Loss: 0.9992826581001282, Test_Accuracy_Rate: 63.49%\n",
      "Epoch 834, Total Loss: 0.9855678677558899, Accuracy Rate: 64.11%,            Test_Total_Loss: 0.9548370838165283, Test_Accuracy_Rate: 63.58%\n",
      "Epoch 835, Total Loss: 1.077208161354065, Accuracy Rate: 59.81%,            Test_Total_Loss: 1.0184855461120605, Test_Accuracy_Rate: 63.58%\n",
      "Epoch 836, Total Loss: 1.0722665786743164, Accuracy Rate: 60.76%,            Test_Total_Loss: 0.9502108693122864, Test_Accuracy_Rate: 64.74%\n",
      "Epoch 837, Total Loss: 0.9994432330131531, Accuracy Rate: 62.83%,            Test_Total_Loss: 0.9198875427246094, Test_Accuracy_Rate: 65.77%\n",
      "Epoch 838, Total Loss: 0.9618461728096008, Accuracy Rate: 63.87%,            Test_Total_Loss: 0.912628173828125, Test_Accuracy_Rate: 65.73%\n",
      "Epoch 839, Total Loss: 0.9468575716018677, Accuracy Rate: 63.87%,            Test_Total_Loss: 0.8795274496078491, Test_Accuracy_Rate: 66.80%\n",
      "Epoch 840, Total Loss: 0.9163661003112793, Accuracy Rate: 64.32%,            Test_Total_Loss: 0.8638638854026794, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 841, Total Loss: 0.9003885984420776, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.8576441407203674, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 842, Total Loss: 0.8950836658477783, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8717579245567322, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 843, Total Loss: 0.8950935006141663, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8554700613021851, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 844, Total Loss: 0.8893315196037292, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8609653115272522, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 845, Total Loss: 0.8977769613265991, Accuracy Rate: 64.45%,            Test_Total_Loss: 0.8602036833763123, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 846, Total Loss: 0.8976095914840698, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.856431245803833, Test_Accuracy_Rate: 67.38%\n",
      "Epoch 847, Total Loss: 0.8902907967567444, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8600345253944397, Test_Accuracy_Rate: 66.97%\n",
      "Epoch 848, Total Loss: 0.8983428478240967, Accuracy Rate: 64.49%,            Test_Total_Loss: 0.8672332763671875, Test_Accuracy_Rate: 67.14%\n",
      "Epoch 849, Total Loss: 0.8953372836112976, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.8559574484825134, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 850, Total Loss: 0.8986425995826721, Accuracy Rate: 64.69%,            Test_Total_Loss: 0.8581227660179138, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 851, Total Loss: 0.8921601176261902, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8569811582565308, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 852, Total Loss: 0.8932629823684692, Accuracy Rate: 64.32%,            Test_Total_Loss: 0.8546446561813354, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 853, Total Loss: 0.8870970010757446, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8531319499015808, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 854, Total Loss: 0.8940900564193726, Accuracy Rate: 64.49%,            Test_Total_Loss: 0.8588331341743469, Test_Accuracy_Rate: 67.38%\n",
      "Epoch 855, Total Loss: 0.8904788494110107, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8633809089660645, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 856, Total Loss: 0.8941501975059509, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.8551264405250549, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 857, Total Loss: 0.889358639717102, Accuracy Rate: 65.11%,            Test_Total_Loss: 0.8599115610122681, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 858, Total Loss: 0.8933234214782715, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8521313071250916, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 859, Total Loss: 0.8902862071990967, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8485214114189148, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 860, Total Loss: 0.8903406262397766, Accuracy Rate: 65.11%,            Test_Total_Loss: 0.8561491966247559, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 861, Total Loss: 0.8896727561950684, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8540584444999695, Test_Accuracy_Rate: 66.97%\n",
      "Epoch 862, Total Loss: 0.8923153877258301, Accuracy Rate: 64.36%,            Test_Total_Loss: 0.8528939485549927, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 863, Total Loss: 0.8893088102340698, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.8590965270996094, Test_Accuracy_Rate: 66.97%\n",
      "Epoch 864, Total Loss: 0.8915716409683228, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.8498875498771667, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 865, Total Loss: 0.8871338367462158, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8474583029747009, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 866, Total Loss: 0.8830803036689758, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8505947589874268, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 867, Total Loss: 0.8856824636459351, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.8467040061950684, Test_Accuracy_Rate: 67.38%\n",
      "Epoch 868, Total Loss: 0.8853651285171509, Accuracy Rate: 64.98%,            Test_Total_Loss: 0.8475081920623779, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 869, Total Loss: 0.8892796039581299, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.8550977110862732, Test_Accuracy_Rate: 66.97%\n",
      "Epoch 870, Total Loss: 0.8929287791252136, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.847912609577179, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 871, Total Loss: 0.8896436095237732, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8515130281448364, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 872, Total Loss: 0.8874030113220215, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8636873960494995, Test_Accuracy_Rate: 66.89%\n",
      "Epoch 873, Total Loss: 0.8913679718971252, Accuracy Rate: 64.49%,            Test_Total_Loss: 0.8547665476799011, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 874, Total Loss: 0.8871999979019165, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8552985787391663, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 875, Total Loss: 0.8892289400100708, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8466971516609192, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 876, Total Loss: 0.8906591534614563, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8452674746513367, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 877, Total Loss: 0.8941200375556946, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8551358580589294, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 878, Total Loss: 0.8952375650405884, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8679532408714294, Test_Accuracy_Rate: 66.97%\n",
      "Epoch 879, Total Loss: 0.9005301594734192, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8593668937683105, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 880, Total Loss: 0.9049522876739502, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.8610156178474426, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 881, Total Loss: 0.8994835019111633, Accuracy Rate: 64.57%,            Test_Total_Loss: 0.862686276435852, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 882, Total Loss: 0.8995147347450256, Accuracy Rate: 65.11%,            Test_Total_Loss: 0.8595044016838074, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 883, Total Loss: 0.8965696096420288, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8526376485824585, Test_Accuracy_Rate: 66.80%\n",
      "Epoch 884, Total Loss: 0.8974645733833313, Accuracy Rate: 64.36%,            Test_Total_Loss: 0.8521590828895569, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 885, Total Loss: 0.8956860899925232, Accuracy Rate: 64.49%,            Test_Total_Loss: 0.8516826629638672, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 886, Total Loss: 0.892346203327179, Accuracy Rate: 64.61%,            Test_Total_Loss: 0.8576614260673523, Test_Accuracy_Rate: 67.43%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 887, Total Loss: 0.8935745358467102, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8622168302536011, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 888, Total Loss: 0.8954501748085022, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8511684536933899, Test_Accuracy_Rate: 67.14%\n",
      "Epoch 889, Total Loss: 0.8912378549575806, Accuracy Rate: 64.36%,            Test_Total_Loss: 0.8541683554649353, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 890, Total Loss: 0.8917514681816101, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8558728098869324, Test_Accuracy_Rate: 66.76%\n",
      "Epoch 891, Total Loss: 0.8910080790519714, Accuracy Rate: 64.57%,            Test_Total_Loss: 0.8563736081123352, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 892, Total Loss: 0.8934491872787476, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.8571841716766357, Test_Accuracy_Rate: 67.38%\n",
      "Epoch 893, Total Loss: 0.8929768800735474, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.849509596824646, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 894, Total Loss: 0.8879519104957581, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8658215403556824, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 895, Total Loss: 0.893484890460968, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8530489206314087, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 896, Total Loss: 0.8898888826370239, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.8501327037811279, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 897, Total Loss: 0.8924996256828308, Accuracy Rate: 64.03%,            Test_Total_Loss: 0.8530938029289246, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 898, Total Loss: 0.8956233859062195, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8518070578575134, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 899, Total Loss: 0.8985424041748047, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.8691231608390808, Test_Accuracy_Rate: 66.76%\n",
      "Epoch 900, Total Loss: 0.9074473977088928, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8658185005187988, Test_Accuracy_Rate: 66.97%\n",
      "Epoch 901, Total Loss: 0.9074662923812866, Accuracy Rate: 64.20%,            Test_Total_Loss: 0.8551427125930786, Test_Accuracy_Rate: 67.14%\n",
      "Epoch 902, Total Loss: 0.908427894115448, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.8784273266792297, Test_Accuracy_Rate: 66.56%\n",
      "Epoch 903, Total Loss: 0.9587047100067139, Accuracy Rate: 64.40%,            Test_Total_Loss: 1.04779851436615, Test_Accuracy_Rate: 62.96%\n",
      "Epoch 904, Total Loss: 1.3239965438842773, Accuracy Rate: 57.41%,            Test_Total_Loss: 1.2800277471542358, Test_Accuracy_Rate: 58.24%\n",
      "Epoch 905, Total Loss: 1.249547004699707, Accuracy Rate: 57.95%,            Test_Total_Loss: 1.2250522375106812, Test_Accuracy_Rate: 58.20%\n",
      "Epoch 906, Total Loss: 1.2157331705093384, Accuracy Rate: 58.36%,            Test_Total_Loss: 1.0597294569015503, Test_Accuracy_Rate: 62.67%\n",
      "Epoch 907, Total Loss: 1.066798210144043, Accuracy Rate: 61.47%,            Test_Total_Loss: 0.9490280151367188, Test_Accuracy_Rate: 64.49%\n",
      "Epoch 908, Total Loss: 1.0048213005065918, Accuracy Rate: 63.04%,            Test_Total_Loss: 0.9279821515083313, Test_Accuracy_Rate: 65.52%\n",
      "Epoch 909, Total Loss: 1.0241155624389648, Accuracy Rate: 62.50%,            Test_Total_Loss: 1.0224113464355469, Test_Accuracy_Rate: 63.33%\n",
      "Epoch 910, Total Loss: 0.9970547556877136, Accuracy Rate: 63.04%,            Test_Total_Loss: 0.9061927795410156, Test_Accuracy_Rate: 65.81%\n",
      "Epoch 911, Total Loss: 0.9288691282272339, Accuracy Rate: 64.61%,            Test_Total_Loss: 0.8937882781028748, Test_Accuracy_Rate: 66.47%\n",
      "Epoch 912, Total Loss: 0.9165714383125305, Accuracy Rate: 64.61%,            Test_Total_Loss: 0.8727767467498779, Test_Accuracy_Rate: 66.43%\n",
      "Epoch 913, Total Loss: 0.9086623191833496, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.8599241971969604, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 914, Total Loss: 0.8953042030334473, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.8531608581542969, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 915, Total Loss: 0.8904477953910828, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.8540206551551819, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 916, Total Loss: 0.8861562609672546, Accuracy Rate: 64.98%,            Test_Total_Loss: 0.8568280935287476, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 917, Total Loss: 0.8924237489700317, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.8566165566444397, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 918, Total Loss: 0.896350085735321, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8626952767372131, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 919, Total Loss: 0.8906480669975281, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8477253913879395, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 920, Total Loss: 0.8865141868591309, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8527142405509949, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 921, Total Loss: 0.8871322274208069, Accuracy Rate: 64.61%,            Test_Total_Loss: 0.850885808467865, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 922, Total Loss: 0.8859431743621826, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8564372062683105, Test_Accuracy_Rate: 67.14%\n",
      "Epoch 923, Total Loss: 0.880582332611084, Accuracy Rate: 65.11%,            Test_Total_Loss: 0.8452497720718384, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 924, Total Loss: 0.8810369372367859, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8448657989501953, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 925, Total Loss: 0.8844014406204224, Accuracy Rate: 65.15%,            Test_Total_Loss: 0.842053234577179, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 926, Total Loss: 0.8822398781776428, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8503692150115967, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 927, Total Loss: 0.8784881234169006, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8429232239723206, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 928, Total Loss: 0.8790046572685242, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8454101085662842, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 929, Total Loss: 0.8820512890815735, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8440324664115906, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 930, Total Loss: 0.8798990845680237, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8489651083946228, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 931, Total Loss: 0.8827469348907471, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.846675455570221, Test_Accuracy_Rate: 67.38%\n",
      "Epoch 932, Total Loss: 0.8834520578384399, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8443660736083984, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 933, Total Loss: 0.877638041973114, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8484476804733276, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 934, Total Loss: 0.8888492584228516, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.845127522945404, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 935, Total Loss: 0.879666805267334, Accuracy Rate: 64.98%,            Test_Total_Loss: 0.8412056565284729, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 936, Total Loss: 0.8801926970481873, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8458854556083679, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 937, Total Loss: 0.8845695853233337, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.8403233885765076, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 938, Total Loss: 0.8783314228057861, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8448401093482971, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 939, Total Loss: 0.880021870136261, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.8451189994812012, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 940, Total Loss: 0.8818556070327759, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8525669574737549, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 941, Total Loss: 0.8819020986557007, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.8444353342056274, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 942, Total Loss: 0.887620747089386, Accuracy Rate: 64.98%,            Test_Total_Loss: 0.8476412892341614, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 943, Total Loss: 0.8815649151802063, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8451358675956726, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 944, Total Loss: 0.8822191953659058, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.851132333278656, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 945, Total Loss: 0.8796065449714661, Accuracy Rate: 64.65%,            Test_Total_Loss: 0.8432313799858093, Test_Accuracy_Rate: 67.43%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 946, Total Loss: 0.8799437284469604, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.840233564376831, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 947, Total Loss: 0.8817882537841797, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.8456566333770752, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 948, Total Loss: 0.8818863034248352, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8478547930717468, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 949, Total Loss: 0.887015163898468, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8455058932304382, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 950, Total Loss: 0.8862670063972473, Accuracy Rate: 65.15%,            Test_Total_Loss: 0.8513343334197998, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 951, Total Loss: 0.8847151398658752, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.854099690914154, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 952, Total Loss: 0.90603107213974, Accuracy Rate: 64.36%,            Test_Total_Loss: 0.8992418050765991, Test_Accuracy_Rate: 66.64%\n",
      "Epoch 953, Total Loss: 0.9206055998802185, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.8576456308364868, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 954, Total Loss: 0.8979362845420837, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8559581637382507, Test_Accuracy_Rate: 67.14%\n",
      "Epoch 955, Total Loss: 0.8933753967285156, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.858108639717102, Test_Accuracy_Rate: 66.97%\n",
      "Epoch 956, Total Loss: 0.8965515494346619, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8603760600090027, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 957, Total Loss: 0.8883355855941772, Accuracy Rate: 64.61%,            Test_Total_Loss: 0.8598814606666565, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 958, Total Loss: 0.900911808013916, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8781046867370605, Test_Accuracy_Rate: 66.68%\n",
      "Epoch 959, Total Loss: 0.9168274402618408, Accuracy Rate: 64.65%,            Test_Total_Loss: 0.8698073029518127, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 960, Total Loss: 0.9021706581115723, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8646526336669922, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 961, Total Loss: 0.9062395691871643, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.8789303302764893, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 962, Total Loss: 0.9019836783409119, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8579647541046143, Test_Accuracy_Rate: 67.14%\n",
      "Epoch 963, Total Loss: 0.909676730632782, Accuracy Rate: 65.15%,            Test_Total_Loss: 0.8619130849838257, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 964, Total Loss: 0.8936614990234375, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.876379132270813, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 965, Total Loss: 0.9337177276611328, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8729246854782104, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 966, Total Loss: 0.9100944995880127, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8781612515449524, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 967, Total Loss: 0.9111382365226746, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.8582732677459717, Test_Accuracy_Rate: 66.89%\n",
      "Epoch 968, Total Loss: 0.9133484363555908, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8727805614471436, Test_Accuracy_Rate: 66.18%\n",
      "Epoch 969, Total Loss: 0.9089798927307129, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.8530074954032898, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 970, Total Loss: 0.8957728147506714, Accuracy Rate: 65.15%,            Test_Total_Loss: 0.8604767322540283, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 971, Total Loss: 0.8964152932167053, Accuracy Rate: 64.57%,            Test_Total_Loss: 0.8570103049278259, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 972, Total Loss: 0.8968276977539062, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8587568402290344, Test_Accuracy_Rate: 66.97%\n",
      "Epoch 973, Total Loss: 0.9037968516349792, Accuracy Rate: 65.11%,            Test_Total_Loss: 0.8602657318115234, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 974, Total Loss: 0.9049890637397766, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.8747604489326477, Test_Accuracy_Rate: 66.47%\n",
      "Epoch 975, Total Loss: 0.9224303364753723, Accuracy Rate: 63.91%,            Test_Total_Loss: 0.8603490591049194, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 976, Total Loss: 0.900581955909729, Accuracy Rate: 64.49%,            Test_Total_Loss: 0.859086275100708, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 977, Total Loss: 0.8885017037391663, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.8517316579818726, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 978, Total Loss: 0.8834075927734375, Accuracy Rate: 64.98%,            Test_Total_Loss: 0.8477960824966431, Test_Accuracy_Rate: 66.89%\n",
      "Epoch 979, Total Loss: 0.8847575187683105, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8453850746154785, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 980, Total Loss: 0.8822954893112183, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8404898047447205, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 981, Total Loss: 0.8806269764900208, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8483396768569946, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 982, Total Loss: 0.8891091346740723, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8576593995094299, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 983, Total Loss: 0.8899726867675781, Accuracy Rate: 64.49%,            Test_Total_Loss: 0.8489683270454407, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 984, Total Loss: 0.8851999044418335, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8504571318626404, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 985, Total Loss: 0.8841357231140137, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8450091481208801, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 986, Total Loss: 0.8795639872550964, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8394668698310852, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 987, Total Loss: 0.8764085173606873, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8473306894302368, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 988, Total Loss: 0.8842918276786804, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8449074625968933, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 989, Total Loss: 0.8826383352279663, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.8474165201187134, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 990, Total Loss: 0.8837177157402039, Accuracy Rate: 64.45%,            Test_Total_Loss: 0.8402624726295471, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 991, Total Loss: 0.8754488229751587, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8422476053237915, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 992, Total Loss: 0.8756360411643982, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.8395992517471313, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 993, Total Loss: 0.8756716847419739, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.8444640636444092, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 994, Total Loss: 0.8763706088066101, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8433936238288879, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 995, Total Loss: 0.8821390271186829, Accuracy Rate: 64.98%,            Test_Total_Loss: 0.8411362767219543, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 996, Total Loss: 0.886459469795227, Accuracy Rate: 64.28%,            Test_Total_Loss: 0.8680262565612793, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 997, Total Loss: 0.9042158722877502, Accuracy Rate: 65.11%,            Test_Total_Loss: 0.8557682037353516, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 998, Total Loss: 0.9012845754623413, Accuracy Rate: 64.11%,            Test_Total_Loss: 0.8669679760932922, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 999, Total Loss: 0.9164421558380127, Accuracy Rate: 64.61%,            Test_Total_Loss: 0.8842812776565552, Test_Accuracy_Rate: 66.43%\n",
      "Epoch 1000, Total Loss: 0.9056599140167236, Accuracy Rate: 64.57%,            Test_Total_Loss: 0.864199697971344, Test_Accuracy_Rate: 66.93%\n",
      "Epoch 1001, Total Loss: 0.900902509689331, Accuracy Rate: 64.36%,            Test_Total_Loss: 0.8768394589424133, Test_Accuracy_Rate: 66.47%\n",
      "Epoch 1002, Total Loss: 0.9158222079277039, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.8600519299507141, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 1003, Total Loss: 0.8907491564750671, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8467995524406433, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1004, Total Loss: 0.8812375068664551, Accuracy Rate: 64.65%,            Test_Total_Loss: 0.8439116477966309, Test_Accuracy_Rate: 67.59%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1005, Total Loss: 0.881719708442688, Accuracy Rate: 65.15%,            Test_Total_Loss: 0.841183066368103, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1006, Total Loss: 0.8828883767127991, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.842697024345398, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 1007, Total Loss: 0.8752652406692505, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8430222868919373, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 1008, Total Loss: 0.8812204599380493, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8429821133613586, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 1009, Total Loss: 0.8833513855934143, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8547502756118774, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 1010, Total Loss: 0.8980107307434082, Accuracy Rate: 64.69%,            Test_Total_Loss: 0.8554967641830444, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 1011, Total Loss: 0.9258824586868286, Accuracy Rate: 64.61%,            Test_Total_Loss: 0.9072563648223877, Test_Accuracy_Rate: 65.81%\n",
      "Epoch 1012, Total Loss: 0.9829109311103821, Accuracy Rate: 63.12%,            Test_Total_Loss: 0.9200009703636169, Test_Accuracy_Rate: 65.48%\n",
      "Epoch 1013, Total Loss: 0.9599621295928955, Accuracy Rate: 63.49%,            Test_Total_Loss: 0.8865561485290527, Test_Accuracy_Rate: 66.56%\n",
      "Epoch 1014, Total Loss: 0.9492232799530029, Accuracy Rate: 63.66%,            Test_Total_Loss: 0.901411235332489, Test_Accuracy_Rate: 65.73%\n",
      "Epoch 1015, Total Loss: 0.9225372672080994, Accuracy Rate: 64.69%,            Test_Total_Loss: 0.8889002799987793, Test_Accuracy_Rate: 66.18%\n",
      "Epoch 1016, Total Loss: 0.9205561280250549, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.8532528281211853, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1017, Total Loss: 0.888979971408844, Accuracy Rate: 64.98%,            Test_Total_Loss: 0.8497332334518433, Test_Accuracy_Rate: 67.38%\n",
      "Epoch 1018, Total Loss: 0.8827816843986511, Accuracy Rate: 65.15%,            Test_Total_Loss: 0.8506259918212891, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 1019, Total Loss: 0.8855658173561096, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.8455719351768494, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1020, Total Loss: 0.9090585112571716, Accuracy Rate: 64.36%,            Test_Total_Loss: 0.8791658282279968, Test_Accuracy_Rate: 66.89%\n",
      "Epoch 1021, Total Loss: 0.9023762941360474, Accuracy Rate: 65.11%,            Test_Total_Loss: 0.8580727577209473, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 1022, Total Loss: 0.8922597765922546, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8655559420585632, Test_Accuracy_Rate: 66.89%\n",
      "Epoch 1023, Total Loss: 0.9069382548332214, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8526367545127869, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 1024, Total Loss: 0.8923507332801819, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.8490453958511353, Test_Accuracy_Rate: 67.38%\n",
      "Epoch 1025, Total Loss: 0.8898465037345886, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8516901135444641, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 1026, Total Loss: 0.8828901052474976, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8441526293754578, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1027, Total Loss: 0.8770904541015625, Accuracy Rate: 65.11%,            Test_Total_Loss: 0.8433308005332947, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 1028, Total Loss: 0.8794649839401245, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8425442576408386, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 1029, Total Loss: 0.8765559196472168, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.8364453911781311, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1030, Total Loss: 0.8717870116233826, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8447594046592712, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 1031, Total Loss: 0.8817904591560364, Accuracy Rate: 64.69%,            Test_Total_Loss: 0.8445321321487427, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1032, Total Loss: 0.8814296722412109, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8435529470443726, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 1033, Total Loss: 0.8828588724136353, Accuracy Rate: 64.69%,            Test_Total_Loss: 0.8428491353988647, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1034, Total Loss: 0.8759868144989014, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8467336893081665, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 1035, Total Loss: 0.8867529630661011, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8466690182685852, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 1036, Total Loss: 0.8870134949684143, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.847616970539093, Test_Accuracy_Rate: 67.14%\n",
      "Epoch 1037, Total Loss: 0.8830240368843079, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8486739993095398, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 1038, Total Loss: 0.8919017314910889, Accuracy Rate: 64.69%,            Test_Total_Loss: 0.8751641511917114, Test_Accuracy_Rate: 66.35%\n",
      "Epoch 1039, Total Loss: 0.9227057099342346, Accuracy Rate: 64.36%,            Test_Total_Loss: 0.8956490755081177, Test_Accuracy_Rate: 66.27%\n",
      "Epoch 1040, Total Loss: 0.9238649606704712, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.8714881539344788, Test_Accuracy_Rate: 66.68%\n",
      "Epoch 1041, Total Loss: 0.9082172513008118, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.896968424320221, Test_Accuracy_Rate: 66.39%\n",
      "Epoch 1042, Total Loss: 0.9323052763938904, Accuracy Rate: 64.24%,            Test_Total_Loss: 0.8991649746894836, Test_Accuracy_Rate: 66.14%\n",
      "Epoch 1043, Total Loss: 0.9203758239746094, Accuracy Rate: 64.32%,            Test_Total_Loss: 0.8755227327346802, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 1044, Total Loss: 0.9300560355186462, Accuracy Rate: 64.07%,            Test_Total_Loss: 0.9152114987373352, Test_Accuracy_Rate: 65.77%\n",
      "Epoch 1045, Total Loss: 1.0228074789047241, Accuracy Rate: 63.04%,            Test_Total_Loss: 0.9828130006790161, Test_Accuracy_Rate: 64.03%\n",
      "Epoch 1046, Total Loss: 1.0004745721817017, Accuracy Rate: 62.58%,            Test_Total_Loss: 0.9431315064430237, Test_Accuracy_Rate: 65.69%\n",
      "Epoch 1047, Total Loss: 0.9594569802284241, Accuracy Rate: 64.65%,            Test_Total_Loss: 0.9172698855400085, Test_Accuracy_Rate: 65.73%\n",
      "Epoch 1048, Total Loss: 0.9292213320732117, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8876169919967651, Test_Accuracy_Rate: 66.39%\n",
      "Epoch 1049, Total Loss: 0.93259197473526, Accuracy Rate: 64.11%,            Test_Total_Loss: 0.8772643208503723, Test_Accuracy_Rate: 66.60%\n",
      "Epoch 1050, Total Loss: 0.9021795988082886, Accuracy Rate: 64.32%,            Test_Total_Loss: 0.8491511940956116, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 1051, Total Loss: 0.886715829372406, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8527840971946716, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 1052, Total Loss: 0.8822159767150879, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8443253040313721, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1053, Total Loss: 0.8768543601036072, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8410663604736328, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 1054, Total Loss: 0.8733044862747192, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.838599681854248, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1055, Total Loss: 0.8743684887886047, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8425596952438354, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1056, Total Loss: 0.8766297698020935, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8364793658256531, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1057, Total Loss: 0.8701698184013367, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.8354191780090332, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1058, Total Loss: 0.8716059327125549, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8345041871070862, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 1059, Total Loss: 0.8770718574523926, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8386286497116089, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1060, Total Loss: 0.8723354339599609, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8405572175979614, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1061, Total Loss: 0.8745884895324707, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8363474011421204, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 1062, Total Loss: 0.8705254793167114, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8370855450630188, Test_Accuracy_Rate: 67.72%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1063, Total Loss: 0.875128984451294, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8432404398918152, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 1064, Total Loss: 0.8713368773460388, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8341717720031738, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1065, Total Loss: 0.8740787506103516, Accuracy Rate: 64.57%,            Test_Total_Loss: 0.8349035382270813, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1066, Total Loss: 0.8693702816963196, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8366522192955017, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1067, Total Loss: 0.8725672364234924, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8401553630828857, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 1068, Total Loss: 0.876357913017273, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8372291922569275, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1069, Total Loss: 0.877312421798706, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8410231471061707, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1070, Total Loss: 0.8772430419921875, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8465865254402161, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 1071, Total Loss: 0.8773237466812134, Accuracy Rate: 65.11%,            Test_Total_Loss: 0.8400168418884277, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 1072, Total Loss: 0.8731881976127625, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8424349427223206, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 1073, Total Loss: 0.8761356472969055, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.851516842842102, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 1074, Total Loss: 0.8755655884742737, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8368921279907227, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1075, Total Loss: 0.8742715120315552, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8393709063529968, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1076, Total Loss: 0.8724120259284973, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.836027204990387, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1077, Total Loss: 0.8755066394805908, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8418307304382324, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1078, Total Loss: 0.8718296885490417, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8452174663543701, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 1079, Total Loss: 0.8757316470146179, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8462223410606384, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 1080, Total Loss: 0.8893940448760986, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.8424010276794434, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 1081, Total Loss: 0.8782853484153748, Accuracy Rate: 64.98%,            Test_Total_Loss: 0.8468579649925232, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 1082, Total Loss: 0.8836727142333984, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8354364037513733, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1083, Total Loss: 0.8719176054000854, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.8385480642318726, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1084, Total Loss: 0.8740105032920837, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8427316546440125, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 1085, Total Loss: 0.8794375061988831, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8509911894798279, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 1086, Total Loss: 0.8809748291969299, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.8363072276115417, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1087, Total Loss: 0.8727614879608154, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.8420248031616211, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 1088, Total Loss: 0.8807572722434998, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8389208316802979, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1089, Total Loss: 0.8739769458770752, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8448572754859924, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 1090, Total Loss: 0.8829146027565002, Accuracy Rate: 64.24%,            Test_Total_Loss: 0.8368108868598938, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 1091, Total Loss: 0.8766445517539978, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8389261960983276, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1092, Total Loss: 0.8737341165542603, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8440013527870178, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 1093, Total Loss: 0.874640941619873, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8450863361358643, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1094, Total Loss: 0.8751446604728699, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8318708539009094, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1095, Total Loss: 0.8678715229034424, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8374829292297363, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1096, Total Loss: 0.869145929813385, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.833203136920929, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1097, Total Loss: 0.8677864074707031, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8312320113182068, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1098, Total Loss: 0.8696798086166382, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8398857712745667, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 1099, Total Loss: 0.8692333698272705, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8320198655128479, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1100, Total Loss: 0.8685160279273987, Accuracy Rate: 64.98%,            Test_Total_Loss: 0.8301937580108643, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1101, Total Loss: 0.8696553111076355, Accuracy Rate: 66.27%,            Test_Total_Loss: 0.8284662365913391, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1102, Total Loss: 0.8710890412330627, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8331541419029236, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1103, Total Loss: 0.8689188957214355, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8409628868103027, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 1104, Total Loss: 0.874997615814209, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.832648754119873, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1105, Total Loss: 0.8736159801483154, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8394363522529602, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1106, Total Loss: 0.875969409942627, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8340843319892883, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1107, Total Loss: 0.8742533326148987, Accuracy Rate: 64.57%,            Test_Total_Loss: 0.8439404368400574, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 1108, Total Loss: 0.881334125995636, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.842218816280365, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1109, Total Loss: 0.9013370871543884, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.867382824420929, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 1110, Total Loss: 0.9210367798805237, Accuracy Rate: 64.69%,            Test_Total_Loss: 1.024502158164978, Test_Accuracy_Rate: 62.58%\n",
      "Epoch 1111, Total Loss: 1.0975706577301025, Accuracy Rate: 60.18%,            Test_Total_Loss: 1.1765607595443726, Test_Accuracy_Rate: 59.19%\n",
      "Epoch 1112, Total Loss: 1.5144052505493164, Accuracy Rate: 54.64%,            Test_Total_Loss: 1.5488672256469727, Test_Accuracy_Rate: 53.48%\n",
      "Epoch 1113, Total Loss: 1.5829975605010986, Accuracy Rate: 52.69%,            Test_Total_Loss: 1.2454322576522827, Test_Accuracy_Rate: 58.98%\n",
      "Epoch 1114, Total Loss: 1.3202658891677856, Accuracy Rate: 57.62%,            Test_Total_Loss: 1.1041926145553589, Test_Accuracy_Rate: 61.22%\n",
      "Epoch 1115, Total Loss: 1.2864476442337036, Accuracy Rate: 57.49%,            Test_Total_Loss: 1.1063252687454224, Test_Accuracy_Rate: 61.51%\n",
      "Epoch 1116, Total Loss: 1.1057429313659668, Accuracy Rate: 60.68%,            Test_Total_Loss: 0.9944305419921875, Test_Accuracy_Rate: 63.45%\n",
      "Epoch 1117, Total Loss: 1.005065679550171, Accuracy Rate: 62.83%,            Test_Total_Loss: 0.9199952483177185, Test_Accuracy_Rate: 65.60%\n",
      "Epoch 1118, Total Loss: 0.9503857493400574, Accuracy Rate: 64.40%,            Test_Total_Loss: 0.8861212730407715, Test_Accuracy_Rate: 66.68%\n",
      "Epoch 1119, Total Loss: 0.9105945229530334, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8655153512954712, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 1120, Total Loss: 0.8938152194023132, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8543280363082886, Test_Accuracy_Rate: 67.47%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1121, Total Loss: 0.888817548751831, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8509766459465027, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 1122, Total Loss: 0.8790347576141357, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8462939858436584, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 1123, Total Loss: 0.8783242106437683, Accuracy Rate: 64.98%,            Test_Total_Loss: 0.8471330404281616, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1124, Total Loss: 0.8734700083732605, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8402255177497864, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1125, Total Loss: 0.8699303865432739, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8407679200172424, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1126, Total Loss: 0.8663824200630188, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8376930356025696, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1127, Total Loss: 0.8676511645317078, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8398336172103882, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1128, Total Loss: 0.8684096336364746, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8403087854385376, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1129, Total Loss: 0.8683909177780151, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8374060392379761, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 1130, Total Loss: 0.868532657623291, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8377456665039062, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 1131, Total Loss: 0.8667371869087219, Accuracy Rate: 65.11%,            Test_Total_Loss: 0.836113691329956, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1132, Total Loss: 0.8664038777351379, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8339037895202637, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1133, Total Loss: 0.8634925484657288, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.831582248210907, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1134, Total Loss: 0.864854097366333, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8362741470336914, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1135, Total Loss: 0.8670672178268433, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.835723340511322, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1136, Total Loss: 0.8634718656539917, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.832551121711731, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1137, Total Loss: 0.8603212237358093, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.835815966129303, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1138, Total Loss: 0.8627749085426331, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8345534801483154, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1139, Total Loss: 0.8621077537536621, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8308641910552979, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1140, Total Loss: 0.8623825907707214, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8342682123184204, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1141, Total Loss: 0.867431640625, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8326607942581177, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1142, Total Loss: 0.8672198057174683, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8365923762321472, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1143, Total Loss: 0.8708633780479431, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8457081317901611, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1144, Total Loss: 0.874862790107727, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8372519016265869, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1145, Total Loss: 0.8662055134773254, Accuracy Rate: 66.27%,            Test_Total_Loss: 0.8332656025886536, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1146, Total Loss: 0.8620337247848511, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8367071747779846, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1147, Total Loss: 0.865272045135498, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8312938809394836, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1148, Total Loss: 0.8590483665466309, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.833061695098877, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1149, Total Loss: 0.8705922961235046, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8387271165847778, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1150, Total Loss: 0.8663913607597351, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8581843972206116, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 1151, Total Loss: 0.9033529162406921, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8598698973655701, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 1152, Total Loss: 0.893319308757782, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8797422647476196, Test_Accuracy_Rate: 66.56%\n",
      "Epoch 1153, Total Loss: 0.9066558480262756, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.8717367053031921, Test_Accuracy_Rate: 67.14%\n",
      "Epoch 1154, Total Loss: 0.8932210206985474, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8778865337371826, Test_Accuracy_Rate: 67.14%\n",
      "Epoch 1155, Total Loss: 0.8991152048110962, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8505263328552246, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 1156, Total Loss: 0.8776074647903442, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8405009508132935, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1157, Total Loss: 0.871711254119873, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8325677514076233, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1158, Total Loss: 0.8686618804931641, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.829980731010437, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1159, Total Loss: 0.8642026782035828, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8363183736801147, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1160, Total Loss: 0.8665806651115417, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.830156147480011, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1161, Total Loss: 0.8654888272285461, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8364635705947876, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1162, Total Loss: 0.864722490310669, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8308507800102234, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1163, Total Loss: 0.8615078926086426, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8298984169960022, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1164, Total Loss: 0.8657159209251404, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8337966799736023, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1165, Total Loss: 0.8656131625175476, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8305270075798035, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1166, Total Loss: 0.8623635172843933, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8342070579528809, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1167, Total Loss: 0.8653066754341125, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8353298306465149, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1168, Total Loss: 0.8670946359634399, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8362419605255127, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 1169, Total Loss: 0.866130530834198, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8305071592330933, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1170, Total Loss: 0.8601397275924683, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.8375664949417114, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1171, Total Loss: 0.8873242735862732, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8601248860359192, Test_Accuracy_Rate: 67.38%\n",
      "Epoch 1172, Total Loss: 0.8762781023979187, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8463956117630005, Test_Accuracy_Rate: 67.38%\n",
      "Epoch 1173, Total Loss: 0.8851937651634216, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8690210580825806, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 1174, Total Loss: 0.8865106701850891, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8494154214859009, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 1175, Total Loss: 0.8759070038795471, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8385879397392273, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1176, Total Loss: 0.8765047192573547, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8383176922798157, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1177, Total Loss: 0.8703357577323914, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8333454132080078, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 1178, Total Loss: 0.8670159578323364, Accuracy Rate: 64.98%,            Test_Total_Loss: 0.8303748965263367, Test_Accuracy_Rate: 68.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1179, Total Loss: 0.8646911978721619, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8329154849052429, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1180, Total Loss: 0.8690053224563599, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8454466462135315, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 1181, Total Loss: 0.8867404460906982, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8600329160690308, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 1182, Total Loss: 0.8790428638458252, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8338937759399414, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1183, Total Loss: 0.8726562857627869, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.8295801877975464, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1184, Total Loss: 0.8628653287887573, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8344337940216064, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 1185, Total Loss: 0.8649557828903198, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8283215761184692, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1186, Total Loss: 0.8634598255157471, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.827806293964386, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1187, Total Loss: 0.8611960411071777, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8382847309112549, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1188, Total Loss: 0.8683655261993408, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8363039493560791, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1189, Total Loss: 0.8698710203170776, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8378528952598572, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 1190, Total Loss: 0.8728631734848022, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8350444436073303, Test_Accuracy_Rate: 67.38%\n",
      "Epoch 1191, Total Loss: 0.8652960062026978, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8378445506095886, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1192, Total Loss: 0.8698865175247192, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8275144696235657, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1193, Total Loss: 0.8750601410865784, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8380201458930969, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 1194, Total Loss: 0.8738226890563965, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.838606595993042, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1195, Total Loss: 0.8690759539604187, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8390793800354004, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 1196, Total Loss: 0.8740420341491699, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8455548286437988, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 1197, Total Loss: 0.8824018239974976, Accuracy Rate: 64.69%,            Test_Total_Loss: 0.8511548042297363, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 1198, Total Loss: 0.8980826139450073, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8593406677246094, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 1199, Total Loss: 0.8944611549377441, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.852342426776886, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 1200, Total Loss: 0.8867903351783752, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8436012268066406, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 1201, Total Loss: 0.8764262795448303, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8345924019813538, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1202, Total Loss: 0.8692512512207031, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8322011232376099, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1203, Total Loss: 0.8682893514633179, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8292545676231384, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1204, Total Loss: 0.8600354194641113, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8271854519844055, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1205, Total Loss: 0.860694169998169, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8257251977920532, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1206, Total Loss: 0.8593503832817078, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8321096301078796, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1207, Total Loss: 0.8692789673805237, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8263314366340637, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1208, Total Loss: 0.8661673665046692, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8323808312416077, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1209, Total Loss: 0.8613250255584717, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8277063369750977, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1210, Total Loss: 0.8602033853530884, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8312801122665405, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1211, Total Loss: 0.8644385933876038, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8323783278465271, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1212, Total Loss: 0.8681836128234863, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.827182948589325, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1213, Total Loss: 0.8654667735099792, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8289577960968018, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1214, Total Loss: 0.8633388876914978, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8280736804008484, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1215, Total Loss: 0.8590407967567444, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8294208645820618, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1216, Total Loss: 0.8672546744346619, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8285840749740601, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1217, Total Loss: 0.9240405559539795, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.8584685325622559, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 1218, Total Loss: 0.9071948528289795, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8566159605979919, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 1219, Total Loss: 0.8922396302223206, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8570975065231323, Test_Accuracy_Rate: 66.76%\n",
      "Epoch 1220, Total Loss: 0.8958938717842102, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8684278726577759, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 1221, Total Loss: 0.8970163464546204, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8407180905342102, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1222, Total Loss: 0.8772688508033752, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.8321151733398438, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1223, Total Loss: 0.8691049814224243, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8323343396186829, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1224, Total Loss: 0.8732732534408569, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.831800639629364, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1225, Total Loss: 0.8687279224395752, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8305569291114807, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1226, Total Loss: 0.8652768135070801, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8320869207382202, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1227, Total Loss: 0.8704107403755188, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8286929130554199, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1228, Total Loss: 0.8695398569107056, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8342792391777039, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1229, Total Loss: 0.8710067868232727, Accuracy Rate: 65.15%,            Test_Total_Loss: 0.8301550149917603, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1230, Total Loss: 0.8664650917053223, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8345954418182373, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1231, Total Loss: 0.8728857636451721, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8351719975471497, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1232, Total Loss: 0.8750036954879761, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.8289576768875122, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1233, Total Loss: 0.8662689328193665, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.828406810760498, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1234, Total Loss: 0.863248348236084, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8336148262023926, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1235, Total Loss: 0.8675309419631958, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8307953476905823, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1236, Total Loss: 0.8655935525894165, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8297924995422363, Test_Accuracy_Rate: 67.92%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1237, Total Loss: 0.8622956275939941, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8263840675354004, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1238, Total Loss: 0.8613909482955933, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.825576901435852, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 1239, Total Loss: 0.8613619804382324, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8295894265174866, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1240, Total Loss: 0.8662433624267578, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8321784138679504, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1241, Total Loss: 0.8711643218994141, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8364612460136414, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1242, Total Loss: 0.8714699149131775, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8309062123298645, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1243, Total Loss: 0.8658332824707031, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8294769525527954, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1244, Total Loss: 0.8659390807151794, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8262391090393066, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1245, Total Loss: 0.8613398671150208, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8316183686256409, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1246, Total Loss: 0.8700552582740784, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.8428472280502319, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 1247, Total Loss: 0.8770298361778259, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.8426864743232727, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1248, Total Loss: 2.5557870864868164, Accuracy Rate: 42.34%,            Test_Total_Loss: 3.3387796878814697, Test_Accuracy_Rate: 35.02%\n",
      "Epoch 1249, Total Loss: 2.8396706581115723, Accuracy Rate: 37.58%,            Test_Total_Loss: 2.160947799682617, Test_Accuracy_Rate: 43.58%\n",
      "Epoch 1250, Total Loss: 2.062739610671997, Accuracy Rate: 43.63%,            Test_Total_Loss: 1.6864206790924072, Test_Accuracy_Rate: 49.67%\n",
      "Epoch 1251, Total Loss: 1.7093497514724731, Accuracy Rate: 48.97%,            Test_Total_Loss: 1.4759901762008667, Test_Accuracy_Rate: 53.31%\n",
      "Epoch 1252, Total Loss: 1.4602957963943481, Accuracy Rate: 53.56%,            Test_Total_Loss: 1.2571555376052856, Test_Accuracy_Rate: 57.82%\n",
      "Epoch 1253, Total Loss: 1.2719411849975586, Accuracy Rate: 56.95%,            Test_Total_Loss: 1.166140079498291, Test_Accuracy_Rate: 60.22%\n",
      "Epoch 1254, Total Loss: 1.175103783607483, Accuracy Rate: 59.44%,            Test_Total_Loss: 1.0817006826400757, Test_Accuracy_Rate: 61.26%\n",
      "Epoch 1255, Total Loss: 1.0966393947601318, Accuracy Rate: 61.09%,            Test_Total_Loss: 1.0259184837341309, Test_Accuracy_Rate: 63.87%\n",
      "Epoch 1256, Total Loss: 1.056746006011963, Accuracy Rate: 62.25%,            Test_Total_Loss: 1.0032553672790527, Test_Accuracy_Rate: 63.99%\n",
      "Epoch 1257, Total Loss: 1.0340522527694702, Accuracy Rate: 62.17%,            Test_Total_Loss: 0.9791526794433594, Test_Accuracy_Rate: 64.69%\n",
      "Epoch 1258, Total Loss: 1.0165578126907349, Accuracy Rate: 62.38%,            Test_Total_Loss: 0.9627107381820679, Test_Accuracy_Rate: 65.07%\n",
      "Epoch 1259, Total Loss: 1.0059795379638672, Accuracy Rate: 63.74%,            Test_Total_Loss: 0.984762966632843, Test_Accuracy_Rate: 64.28%\n",
      "Epoch 1260, Total Loss: 0.9949672818183899, Accuracy Rate: 62.91%,            Test_Total_Loss: 0.9467670917510986, Test_Accuracy_Rate: 65.40%\n",
      "Epoch 1261, Total Loss: 0.9638980031013489, Accuracy Rate: 63.95%,            Test_Total_Loss: 0.9299882650375366, Test_Accuracy_Rate: 65.77%\n",
      "Epoch 1262, Total Loss: 0.9505426287651062, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.9170925617218018, Test_Accuracy_Rate: 66.18%\n",
      "Epoch 1263, Total Loss: 0.9379948973655701, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.9007824063301086, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 1264, Total Loss: 0.9279416799545288, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.8930162191390991, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 1265, Total Loss: 0.9214650392532349, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8890001177787781, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 1266, Total Loss: 0.9254556894302368, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.885919451713562, Test_Accuracy_Rate: 66.68%\n",
      "Epoch 1267, Total Loss: 0.9063102602958679, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8785484433174133, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 1268, Total Loss: 0.9123615622520447, Accuracy Rate: 65.15%,            Test_Total_Loss: 0.8838160634040833, Test_Accuracy_Rate: 66.93%\n",
      "Epoch 1269, Total Loss: 0.89914470911026, Accuracy Rate: 65.15%,            Test_Total_Loss: 0.8654285073280334, Test_Accuracy_Rate: 67.38%\n",
      "Epoch 1270, Total Loss: 0.8955554962158203, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8602911233901978, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 1271, Total Loss: 0.8880754709243774, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8572568297386169, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 1272, Total Loss: 0.8863064646720886, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8541536927223206, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1273, Total Loss: 0.8871498107910156, Accuracy Rate: 64.98%,            Test_Total_Loss: 0.8534945249557495, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1274, Total Loss: 0.885709285736084, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8523337841033936, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 1275, Total Loss: 0.8776691555976868, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8481941223144531, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1276, Total Loss: 0.877892017364502, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8502251505851746, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1277, Total Loss: 0.8781015276908875, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8490056395530701, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1278, Total Loss: 0.8770025968551636, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8442858457565308, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1279, Total Loss: 0.8701984882354736, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8434126973152161, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1280, Total Loss: 0.873539388179779, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8424361348152161, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1281, Total Loss: 0.8696595430374146, Accuracy Rate: 65.15%,            Test_Total_Loss: 0.8406481742858887, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1282, Total Loss: 0.8675121665000916, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8399899005889893, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1283, Total Loss: 0.8652770519256592, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8392974734306335, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1284, Total Loss: 0.8661900758743286, Accuracy Rate: 66.72%,            Test_Total_Loss: 0.8428521156311035, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1285, Total Loss: 0.8739811778068542, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8422627449035645, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1286, Total Loss: 0.8697188496589661, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8398205637931824, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1287, Total Loss: 0.8674210906028748, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8351172804832458, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1288, Total Loss: 0.8645396828651428, Accuracy Rate: 66.27%,            Test_Total_Loss: 0.8370186686515808, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1289, Total Loss: 0.8663305044174194, Accuracy Rate: 66.27%,            Test_Total_Loss: 0.8350340127944946, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1290, Total Loss: 0.865261971950531, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.835289478302002, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1291, Total Loss: 0.8677792549133301, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8344728350639343, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1292, Total Loss: 0.8653476238250732, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8413496613502502, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1293, Total Loss: 0.9268559813499451, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.9005000591278076, Test_Accuracy_Rate: 66.10%\n",
      "Epoch 1294, Total Loss: 0.9113903045654297, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8585394024848938, Test_Accuracy_Rate: 67.47%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1295, Total Loss: 0.8838796019554138, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8399732708930969, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1296, Total Loss: 0.8692421317100525, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8380054235458374, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1297, Total Loss: 0.8647198677062988, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8383551836013794, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1298, Total Loss: 0.8754305839538574, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.846031904220581, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 1299, Total Loss: 0.9063397645950317, Accuracy Rate: 64.65%,            Test_Total_Loss: 0.8966438174247742, Test_Accuracy_Rate: 66.39%\n",
      "Epoch 1300, Total Loss: 0.9095800518989563, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.875379741191864, Test_Accuracy_Rate: 67.05%\n",
      "Epoch 1301, Total Loss: 0.8803187012672424, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8436765074729919, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 1302, Total Loss: 0.8721358776092529, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8363001346588135, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1303, Total Loss: 0.8703289031982422, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.834487795829773, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1304, Total Loss: 0.8645927309989929, Accuracy Rate: 66.51%,            Test_Total_Loss: 0.8348470330238342, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1305, Total Loss: 0.8693904876708984, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8367825150489807, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1306, Total Loss: 0.8673508167266846, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8330414295196533, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1307, Total Loss: 0.859992504119873, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8295425176620483, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1308, Total Loss: 0.8590466380119324, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8279170989990234, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1309, Total Loss: 0.8592808246612549, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8295701742172241, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1310, Total Loss: 0.8592184782028198, Accuracy Rate: 66.76%,            Test_Total_Loss: 0.8302817940711975, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1311, Total Loss: 0.8569077849388123, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8290387988090515, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1312, Total Loss: 0.8607934713363647, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8312670588493347, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1313, Total Loss: 0.8608053922653198, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8269776701927185, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1314, Total Loss: 0.8588955402374268, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8353138566017151, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1315, Total Loss: 0.8618500232696533, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8294690251350403, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1316, Total Loss: 0.8590726256370544, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8344376087188721, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 1317, Total Loss: 0.8690246939659119, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8298150897026062, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1318, Total Loss: 0.856579601764679, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8293970823287964, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1319, Total Loss: 0.85708087682724, Accuracy Rate: 66.43%,            Test_Total_Loss: 0.8327199816703796, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1320, Total Loss: 0.9367426633834839, Accuracy Rate: 64.28%,            Test_Total_Loss: 1.8618179559707642, Test_Accuracy_Rate: 47.93%\n",
      "Epoch 1321, Total Loss: 1.2620223760604858, Accuracy Rate: 58.36%,            Test_Total_Loss: 1.1809824705123901, Test_Accuracy_Rate: 60.14%\n",
      "Epoch 1322, Total Loss: 1.1169182062149048, Accuracy Rate: 61.01%,            Test_Total_Loss: 1.0519129037857056, Test_Accuracy_Rate: 62.46%\n",
      "Epoch 1323, Total Loss: 1.0223113298416138, Accuracy Rate: 62.91%,            Test_Total_Loss: 0.9234859347343445, Test_Accuracy_Rate: 65.36%\n",
      "Epoch 1324, Total Loss: 0.9713245034217834, Accuracy Rate: 63.49%,            Test_Total_Loss: 0.9274811744689941, Test_Accuracy_Rate: 65.89%\n",
      "Epoch 1325, Total Loss: 0.9369709491729736, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.883001446723938, Test_Accuracy_Rate: 66.72%\n",
      "Epoch 1326, Total Loss: 0.9139513969421387, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.8597233295440674, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 1327, Total Loss: 0.8917984366416931, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8514873385429382, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 1328, Total Loss: 0.8825783133506775, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8428744077682495, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1329, Total Loss: 0.869888424873352, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8398458361625671, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1330, Total Loss: 0.8687283396720886, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8381116986274719, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1331, Total Loss: 0.8723554015159607, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.840567946434021, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1332, Total Loss: 0.8683302998542786, Accuracy Rate: 66.60%,            Test_Total_Loss: 0.8348184823989868, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1333, Total Loss: 0.8690603971481323, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8504335284233093, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 1334, Total Loss: 0.8796533346176147, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8384156227111816, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1335, Total Loss: 0.8683000802993774, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8364880681037903, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1336, Total Loss: 0.863369882106781, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8301148414611816, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1337, Total Loss: 0.8569950461387634, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8273833990097046, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1338, Total Loss: 0.8582822680473328, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8282873630523682, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1339, Total Loss: 0.8581588864326477, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8272849917411804, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1340, Total Loss: 0.8578117489814758, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8252703547477722, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1341, Total Loss: 0.8535153269767761, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8247769474983215, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1342, Total Loss: 0.8574255108833313, Accuracy Rate: 66.64%,            Test_Total_Loss: 0.83319491147995, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1343, Total Loss: 0.8615617156028748, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8422811627388, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1344, Total Loss: 0.8690217733383179, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8326358795166016, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1345, Total Loss: 0.8602854013442993, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8328086137771606, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1346, Total Loss: 0.8739004731178284, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8315932750701904, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1347, Total Loss: 0.8683646321296692, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8353926539421082, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1348, Total Loss: 0.8605292439460754, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8267444968223572, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1349, Total Loss: 0.8559661507606506, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8255677819252014, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1350, Total Loss: 0.8568423390388489, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8244792819023132, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1351, Total Loss: 0.8548882007598877, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8242430686950684, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1352, Total Loss: 0.8534767031669617, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.826863706111908, Test_Accuracy_Rate: 68.17%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1353, Total Loss: 0.8583500385284424, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8242107629776001, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1354, Total Loss: 0.858934223651886, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8234774470329285, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1355, Total Loss: 0.8556961417198181, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8253248333930969, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1356, Total Loss: 0.8529254794120789, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8236200213432312, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1357, Total Loss: 0.8630993366241455, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8297434449195862, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1358, Total Loss: 0.8597801327705383, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8308231830596924, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1359, Total Loss: 0.8608281016349792, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8292579054832458, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1360, Total Loss: 0.8646212220191956, Accuracy Rate: 66.43%,            Test_Total_Loss: 0.8359811902046204, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1361, Total Loss: 0.867340087890625, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8414228558540344, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1362, Total Loss: 0.8684777617454529, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8443615436553955, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 1363, Total Loss: 0.8756841421127319, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8574292063713074, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 1364, Total Loss: 0.8832156658172607, Accuracy Rate: 64.61%,            Test_Total_Loss: 0.8465303182601929, Test_Accuracy_Rate: 67.38%\n",
      "Epoch 1365, Total Loss: 0.8733654022216797, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8427808880805969, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 1366, Total Loss: 0.8719820976257324, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8272242546081543, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1367, Total Loss: 0.8636941313743591, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8274534344673157, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1368, Total Loss: 0.8622810244560242, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8237473964691162, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1369, Total Loss: 0.8562706112861633, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.824543833732605, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1370, Total Loss: 0.8589789867401123, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8339794874191284, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1371, Total Loss: 0.8667148947715759, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8305855989456177, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1372, Total Loss: 0.8579176068305969, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8262377977371216, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1373, Total Loss: 0.8556805849075317, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.829329788684845, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1374, Total Loss: 0.8597560524940491, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8236559629440308, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1375, Total Loss: 0.8529844880104065, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8228388428688049, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1376, Total Loss: 0.8557714223861694, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8223767280578613, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1377, Total Loss: 0.8540529012680054, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8224193453788757, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1378, Total Loss: 0.8551555275917053, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8254656791687012, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1379, Total Loss: 0.8531350493431091, Accuracy Rate: 66.56%,            Test_Total_Loss: 0.8260213732719421, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1380, Total Loss: 0.8571903705596924, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8249390125274658, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1381, Total Loss: 0.8537204265594482, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8217775225639343, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1382, Total Loss: 0.8566876649856567, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.8253639340400696, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1383, Total Loss: 0.8541285991668701, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8241786956787109, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1384, Total Loss: 0.8566407561302185, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8216058015823364, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1385, Total Loss: 0.8515843749046326, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8238040208816528, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1386, Total Loss: 0.8549637794494629, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8235645294189453, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1387, Total Loss: 0.8553153872489929, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8206029534339905, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1388, Total Loss: 0.8557422161102295, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8220285773277283, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1389, Total Loss: 0.8568580150604248, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8208646178245544, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1390, Total Loss: 0.8525454998016357, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8291212916374207, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1391, Total Loss: 0.8601762056350708, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8269426822662354, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1392, Total Loss: 0.8588768839836121, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.827354907989502, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1393, Total Loss: 0.8626112937927246, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.836895227432251, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1394, Total Loss: 0.8652198910713196, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8669244647026062, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1395, Total Loss: 0.9084512591362, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8751377463340759, Test_Accuracy_Rate: 66.93%\n",
      "Epoch 1396, Total Loss: 0.9708574414253235, Accuracy Rate: 63.20%,            Test_Total_Loss: 0.9616937041282654, Test_Accuracy_Rate: 64.74%\n",
      "Epoch 1397, Total Loss: 0.9688631296157837, Accuracy Rate: 63.16%,            Test_Total_Loss: 0.8957456350326538, Test_Accuracy_Rate: 66.23%\n",
      "Epoch 1398, Total Loss: 0.924177885055542, Accuracy Rate: 64.24%,            Test_Total_Loss: 0.8761361837387085, Test_Accuracy_Rate: 67.09%\n",
      "Epoch 1399, Total Loss: 0.9010812640190125, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8573123812675476, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 1400, Total Loss: 0.8903024792671204, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8471176028251648, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 1401, Total Loss: 0.8830252289772034, Accuracy Rate: 65.15%,            Test_Total_Loss: 0.8338577747344971, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1402, Total Loss: 0.865206241607666, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8300559520721436, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1403, Total Loss: 0.8637428283691406, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8267099857330322, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1404, Total Loss: 0.8607183694839478, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8265857696533203, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1405, Total Loss: 0.8628847002983093, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8257355690002441, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1406, Total Loss: 0.8707520365715027, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8465190529823303, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1407, Total Loss: 0.8624037504196167, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.82381272315979, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1408, Total Loss: 0.8527024984359741, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8295790553092957, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1409, Total Loss: 0.8571106791496277, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8223159313201904, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1410, Total Loss: 0.8558526039123535, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8213778138160706, Test_Accuracy_Rate: 68.21%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1411, Total Loss: 0.8557730317115784, Accuracy Rate: 66.27%,            Test_Total_Loss: 0.826553463935852, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1412, Total Loss: 0.8529238700866699, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8229889273643494, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1413, Total Loss: 0.8559185862541199, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8214156627655029, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1414, Total Loss: 0.8543164134025574, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8227267265319824, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1415, Total Loss: 0.8553072214126587, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8195869326591492, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1416, Total Loss: 0.8545082211494446, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8211156725883484, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1417, Total Loss: 0.8529990315437317, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8226130604743958, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1418, Total Loss: 0.8532119393348694, Accuracy Rate: 66.72%,            Test_Total_Loss: 0.8216480612754822, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1419, Total Loss: 0.8519718050956726, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8215327262878418, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1420, Total Loss: 0.8534706234931946, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8206769227981567, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1421, Total Loss: 0.8553446531295776, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8238622546195984, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1422, Total Loss: 0.8574348092079163, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8281924724578857, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1423, Total Loss: 0.8621777892112732, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8285829424858093, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1424, Total Loss: 0.8650175333023071, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8388850688934326, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 1425, Total Loss: 0.8649564981460571, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.8279957175254822, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1426, Total Loss: 0.8691534996032715, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8283290863037109, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1427, Total Loss: 0.8801238536834717, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8343266844749451, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1428, Total Loss: 0.8718253970146179, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8535951375961304, Test_Accuracy_Rate: 67.22%\n",
      "Epoch 1429, Total Loss: 0.8765142560005188, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8474533557891846, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 1430, Total Loss: 0.8794195055961609, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8400614261627197, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1431, Total Loss: 0.8684549927711487, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8289629817008972, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1432, Total Loss: 0.8612570762634277, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8248090147972107, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1433, Total Loss: 0.869072675704956, Accuracy Rate: 64.61%,            Test_Total_Loss: 0.8277473449707031, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1434, Total Loss: 0.8687437176704407, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8362188339233398, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1435, Total Loss: 0.8700733184814453, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8879799246788025, Test_Accuracy_Rate: 66.85%\n",
      "Epoch 1436, Total Loss: 0.9231730103492737, Accuracy Rate: 64.86%,            Test_Total_Loss: 0.8854260444641113, Test_Accuracy_Rate: 65.94%\n",
      "Epoch 1437, Total Loss: 0.9391365051269531, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.8978233337402344, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 1438, Total Loss: 0.9378339052200317, Accuracy Rate: 64.03%,            Test_Total_Loss: 0.9503734111785889, Test_Accuracy_Rate: 63.87%\n",
      "Epoch 1439, Total Loss: 0.9772484302520752, Accuracy Rate: 63.49%,            Test_Total_Loss: 0.9578037261962891, Test_Accuracy_Rate: 64.86%\n",
      "Epoch 1440, Total Loss: 0.9768581390380859, Accuracy Rate: 62.71%,            Test_Total_Loss: 0.9198552966117859, Test_Accuracy_Rate: 65.31%\n",
      "Epoch 1441, Total Loss: 0.9918023347854614, Accuracy Rate: 63.74%,            Test_Total_Loss: 0.9793742299079895, Test_Accuracy_Rate: 64.98%\n",
      "Epoch 1442, Total Loss: 0.9685750007629395, Accuracy Rate: 63.78%,            Test_Total_Loss: 0.8983927369117737, Test_Accuracy_Rate: 66.64%\n",
      "Epoch 1443, Total Loss: 0.9309490323066711, Accuracy Rate: 64.61%,            Test_Total_Loss: 0.8658221364021301, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 1444, Total Loss: 0.8909677267074585, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8443692922592163, Test_Accuracy_Rate: 67.38%\n",
      "Epoch 1445, Total Loss: 0.8750990629196167, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8311585187911987, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1446, Total Loss: 0.8629841804504395, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8287860155105591, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1447, Total Loss: 0.8650960326194763, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8295643925666809, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1448, Total Loss: 0.8609530925750732, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8289496898651123, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1449, Total Loss: 0.860143780708313, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8277798295021057, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1450, Total Loss: 0.8574975728988647, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.823418378829956, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1451, Total Loss: 0.8564857840538025, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8251904249191284, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1452, Total Loss: 0.8529756665229797, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8292310833930969, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1453, Total Loss: 0.855726420879364, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8254987597465515, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1454, Total Loss: 0.8543808460235596, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.820417582988739, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1455, Total Loss: 0.8535301685333252, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8221373558044434, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1456, Total Loss: 0.8521460294723511, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8222498893737793, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1457, Total Loss: 0.8515155911445618, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8199397921562195, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1458, Total Loss: 0.8497710824012756, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8184667825698853, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1459, Total Loss: 0.8512555360794067, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8202621340751648, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1460, Total Loss: 0.8531022071838379, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8248351216316223, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1461, Total Loss: 0.8601999878883362, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8259429931640625, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1462, Total Loss: 0.8562321066856384, Accuracy Rate: 65.15%,            Test_Total_Loss: 0.8218593001365662, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1463, Total Loss: 0.8509832620620728, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8193891644477844, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1464, Total Loss: 0.8517295718193054, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8190093040466309, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1465, Total Loss: 0.8536899089813232, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8222290873527527, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1466, Total Loss: 0.8563656210899353, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8210021257400513, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1467, Total Loss: 0.8512765765190125, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8174412846565247, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1468, Total Loss: 0.8495531678199768, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8193362355232239, Test_Accuracy_Rate: 68.13%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1469, Total Loss: 0.8494908213615417, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8200909495353699, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1470, Total Loss: 0.8566844463348389, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.828439474105835, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1471, Total Loss: 0.8713206052780151, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8333342671394348, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1472, Total Loss: 0.8688571453094482, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.825970470905304, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1473, Total Loss: 0.8628100156784058, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8282168507575989, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1474, Total Loss: 0.8600173592567444, Accuracy Rate: 66.27%,            Test_Total_Loss: 0.8306516408920288, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1475, Total Loss: 0.8654638528823853, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8412569165229797, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1476, Total Loss: 0.8724861145019531, Accuracy Rate: 65.02%,            Test_Total_Loss: 0.8261638283729553, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1477, Total Loss: 0.8679677844047546, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8345468640327454, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 1478, Total Loss: 0.8682900667190552, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8262417912483215, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1479, Total Loss: 0.863185703754425, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8279725313186646, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1480, Total Loss: 0.8604222536087036, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8266472816467285, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1481, Total Loss: 0.8555281758308411, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8184434175491333, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1482, Total Loss: 0.8516358733177185, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.820460855960846, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1483, Total Loss: 0.8593274354934692, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8266379833221436, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1484, Total Loss: 0.8612008094787598, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8240434527397156, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1485, Total Loss: 0.8542091846466064, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.821022629737854, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1486, Total Loss: 0.8520025014877319, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8242886662483215, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1487, Total Loss: 0.8537955284118652, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8195422291755676, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1488, Total Loss: 0.8576489686965942, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8188256025314331, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1489, Total Loss: 0.8518026471138, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8198027014732361, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1490, Total Loss: 0.8569285869598389, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8186951875686646, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1491, Total Loss: 0.8560211658477783, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8265705704689026, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1492, Total Loss: 0.8589355945587158, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8307468295097351, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1493, Total Loss: 0.8719692230224609, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8473532795906067, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 1494, Total Loss: 0.8692905306816101, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.832726001739502, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1495, Total Loss: 0.8671357035636902, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8373162150382996, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 1496, Total Loss: 0.9014075398445129, Accuracy Rate: 64.78%,            Test_Total_Loss: 0.9033555388450623, Test_Accuracy_Rate: 65.73%\n",
      "Epoch 1497, Total Loss: 1.0569745302200317, Accuracy Rate: 61.67%,            Test_Total_Loss: 0.9313117861747742, Test_Accuracy_Rate: 65.40%\n",
      "Epoch 1498, Total Loss: 0.9984368681907654, Accuracy Rate: 63.33%,            Test_Total_Loss: 0.8991203308105469, Test_Accuracy_Rate: 66.39%\n",
      "Epoch 1499, Total Loss: 0.9398143291473389, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.9090604186058044, Test_Accuracy_Rate: 66.14%\n",
      "Epoch 1500, Total Loss: 0.8992490172386169, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8547770977020264, Test_Accuracy_Rate: 67.38%\n",
      "Epoch 1501, Total Loss: 0.8817657828330994, Accuracy Rate: 64.74%,            Test_Total_Loss: 0.8374984860420227, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1502, Total Loss: 0.8671844601631165, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.827263593673706, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1503, Total Loss: 0.8624306917190552, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8248063325881958, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1504, Total Loss: 0.8545005917549133, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8240052461624146, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1505, Total Loss: 0.853407084941864, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8214892148971558, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1506, Total Loss: 0.8495110869407654, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8197276592254639, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1507, Total Loss: 0.8501368165016174, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8217228055000305, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1508, Total Loss: 0.8507103323936462, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.819903552532196, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1509, Total Loss: 0.8597567677497864, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8251755237579346, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1510, Total Loss: 0.8560454845428467, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8222187161445618, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1511, Total Loss: 0.8549464344978333, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8286716938018799, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1512, Total Loss: 0.8588907122612, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8188096880912781, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1513, Total Loss: 0.8498207926750183, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8204617500305176, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1514, Total Loss: 0.8523268699645996, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8196577429771423, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1515, Total Loss: 0.8488969206809998, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8340471386909485, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1516, Total Loss: 0.8675795197486877, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8253947496414185, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1517, Total Loss: 0.8510834574699402, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8159232139587402, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1518, Total Loss: 0.8509048819541931, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8149471879005432, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1519, Total Loss: 0.8504441976547241, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.8197712898254395, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1520, Total Loss: 0.8471295237541199, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8203451633453369, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1521, Total Loss: 0.846571683883667, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8158168196678162, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1522, Total Loss: 0.8477607369422913, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8168361783027649, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1523, Total Loss: 0.8482780456542969, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.815466046333313, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1524, Total Loss: 0.8477931022644043, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8183277249336243, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1525, Total Loss: 0.8503049612045288, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8195037841796875, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1526, Total Loss: 0.8505803942680359, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.819098174571991, Test_Accuracy_Rate: 68.29%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1527, Total Loss: 0.8512802124023438, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8183825612068176, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1528, Total Loss: 0.8537010550498962, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8162925839424133, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1529, Total Loss: 0.851468563079834, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8199669122695923, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1530, Total Loss: 0.8484793305397034, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8188567161560059, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1531, Total Loss: 0.8437819480895996, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8142895102500916, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1532, Total Loss: 0.8411797285079956, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8119656443595886, Test_Accuracy_Rate: 68.54%\n",
      "Epoch 1533, Total Loss: 0.8444593548774719, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.811955988407135, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1534, Total Loss: 0.8446643352508545, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8157594799995422, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1535, Total Loss: 0.8464614748954773, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8182763457298279, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1536, Total Loss: 0.8469918966293335, Accuracy Rate: 66.43%,            Test_Total_Loss: 0.8173673748970032, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1537, Total Loss: 0.8493539690971375, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.818114161491394, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1538, Total Loss: 0.8489199280738831, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8237370848655701, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1539, Total Loss: 0.8639459013938904, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8185717463493347, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1540, Total Loss: 0.8552750945091248, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8204585313796997, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1541, Total Loss: 0.8613376617431641, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8328396677970886, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1542, Total Loss: 0.854664146900177, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8235180974006653, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1543, Total Loss: 0.9020577669143677, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.897221028804779, Test_Accuracy_Rate: 66.18%\n",
      "Epoch 1544, Total Loss: 0.9309331178665161, Accuracy Rate: 64.90%,            Test_Total_Loss: 0.8729971647262573, Test_Accuracy_Rate: 66.93%\n",
      "Epoch 1545, Total Loss: 0.8915993571281433, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.844821035861969, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 1546, Total Loss: 0.8887706398963928, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.836086094379425, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1547, Total Loss: 0.8742653727531433, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8313539624214172, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1548, Total Loss: 0.8896433115005493, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8479580283164978, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 1549, Total Loss: 0.8863978981971741, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8387383818626404, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 1550, Total Loss: 0.8742109537124634, Accuracy Rate: 65.11%,            Test_Total_Loss: 0.8242536783218384, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1551, Total Loss: 0.8658026456832886, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8435354828834534, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 1552, Total Loss: 0.8779076933860779, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8307631611824036, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1553, Total Loss: 0.8616465330123901, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8358564376831055, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1554, Total Loss: 0.856857180595398, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8225367069244385, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1555, Total Loss: 0.8596437573432922, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8285176753997803, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1556, Total Loss: 0.8674917817115784, Accuracy Rate: 65.15%,            Test_Total_Loss: 0.8220017552375793, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1557, Total Loss: 0.8577565550804138, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8215609192848206, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1558, Total Loss: 0.8605709075927734, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8246621489524841, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1559, Total Loss: 0.8572091460227966, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8233652114868164, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1560, Total Loss: 0.8555774688720703, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8209152817726135, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1561, Total Loss: 0.850226640701294, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8251706957817078, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1562, Total Loss: 0.8546083569526672, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8160411715507507, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1563, Total Loss: 0.8466799855232239, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8169047832489014, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1564, Total Loss: 0.8519275188446045, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8185250759124756, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1565, Total Loss: 0.8502258658409119, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.821256160736084, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1566, Total Loss: 0.855787456035614, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8203307390213013, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1567, Total Loss: 0.8540666699409485, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8210557699203491, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1568, Total Loss: 0.8496001362800598, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.822978675365448, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1569, Total Loss: 0.8538575172424316, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8253478407859802, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1570, Total Loss: 0.8574103116989136, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8194617629051208, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1571, Total Loss: 0.8546842336654663, Accuracy Rate: 65.15%,            Test_Total_Loss: 0.8177875280380249, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1572, Total Loss: 0.8549212217330933, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8241620063781738, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1573, Total Loss: 0.8514084219932556, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8154293894767761, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1574, Total Loss: 0.8443152904510498, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8147683143615723, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1575, Total Loss: 0.8497195243835449, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8161909580230713, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1576, Total Loss: 0.8508836030960083, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8156805038452148, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1577, Total Loss: 0.849469006061554, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8144441843032837, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1578, Total Loss: 0.843086838722229, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8171023726463318, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1579, Total Loss: 0.8497377634048462, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8158153891563416, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1580, Total Loss: 0.8482152819633484, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.816161572933197, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1581, Total Loss: 0.8507828712463379, Accuracy Rate: 64.98%,            Test_Total_Loss: 0.8142051696777344, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1582, Total Loss: 0.8404642343521118, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8136422038078308, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1583, Total Loss: 0.8503413796424866, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8466663360595703, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 1584, Total Loss: 0.8768248558044434, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8272454738616943, Test_Accuracy_Rate: 68.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1585, Total Loss: 0.8793641328811646, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.8467898964881897, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 1586, Total Loss: 0.8912723064422607, Accuracy Rate: 65.15%,            Test_Total_Loss: 0.8638709187507629, Test_Accuracy_Rate: 66.68%\n",
      "Epoch 1587, Total Loss: 1.0561997890472412, Accuracy Rate: 62.42%,            Test_Total_Loss: 1.0486727952957153, Test_Accuracy_Rate: 63.12%\n",
      "Epoch 1588, Total Loss: 1.1677824258804321, Accuracy Rate: 59.93%,            Test_Total_Loss: 1.0419416427612305, Test_Accuracy_Rate: 63.04%\n",
      "Epoch 1589, Total Loss: 1.044055461883545, Accuracy Rate: 62.04%,            Test_Total_Loss: 0.9676085114479065, Test_Accuracy_Rate: 63.49%\n",
      "Epoch 1590, Total Loss: 1.0192937850952148, Accuracy Rate: 62.21%,            Test_Total_Loss: 0.9399270415306091, Test_Accuracy_Rate: 64.94%\n",
      "Epoch 1591, Total Loss: 0.9631340503692627, Accuracy Rate: 63.78%,            Test_Total_Loss: 0.8906386494636536, Test_Accuracy_Rate: 66.56%\n",
      "Epoch 1592, Total Loss: 0.9102180004119873, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8529242277145386, Test_Accuracy_Rate: 67.55%\n",
      "Epoch 1593, Total Loss: 0.8867033123970032, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8752749562263489, Test_Accuracy_Rate: 66.68%\n",
      "Epoch 1594, Total Loss: 0.890935480594635, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8469213247299194, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 1595, Total Loss: 0.8811356425285339, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8381956815719604, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1596, Total Loss: 0.8650005459785461, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8285618424415588, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 1597, Total Loss: 0.8545389771461487, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8221220374107361, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1598, Total Loss: 0.8532176613807678, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8230354189872742, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1599, Total Loss: 0.8516446948051453, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8188186287879944, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1600, Total Loss: 0.8496147990226746, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8201163411140442, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1601, Total Loss: 0.8472362160682678, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8168966174125671, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1602, Total Loss: 0.8476201295852661, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.814582109451294, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1603, Total Loss: 0.8428654670715332, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.812854528427124, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1604, Total Loss: 0.843367338180542, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8130604028701782, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1605, Total Loss: 0.8430972099304199, Accuracy Rate: 66.27%,            Test_Total_Loss: 0.8117372393608093, Test_Accuracy_Rate: 68.54%\n",
      "Epoch 1606, Total Loss: 0.8413046002388, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8133741617202759, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1607, Total Loss: 0.8399143218994141, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8129010796546936, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1608, Total Loss: 0.8450791239738464, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8121902942657471, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1609, Total Loss: 0.8416290879249573, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8104855418205261, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1610, Total Loss: 0.8403835296630859, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8123762011528015, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1611, Total Loss: 0.8429009318351746, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8129207491874695, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1612, Total Loss: 0.8419239521026611, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8108311295509338, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1613, Total Loss: 0.838966429233551, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8114053606987, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1614, Total Loss: 0.8421494364738464, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8153804540634155, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1615, Total Loss: 0.8429579734802246, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8138566017150879, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1616, Total Loss: 0.8427885174751282, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8117082118988037, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1617, Total Loss: 0.8443942070007324, Accuracy Rate: 66.43%,            Test_Total_Loss: 0.8130239248275757, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1618, Total Loss: 0.841528058052063, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8111453652381897, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1619, Total Loss: 0.8426824808120728, Accuracy Rate: 66.27%,            Test_Total_Loss: 0.8105102181434631, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1620, Total Loss: 0.8430630564689636, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8113735318183899, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1621, Total Loss: 0.8438072204589844, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8167995810508728, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1622, Total Loss: 0.8537872433662415, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8339206576347351, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1623, Total Loss: 0.8528510928153992, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8168549537658691, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1624, Total Loss: 0.8494225740432739, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8163501620292664, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1625, Total Loss: 0.8526178598403931, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8317386507987976, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1626, Total Loss: 0.8547012805938721, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8237008452415466, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1627, Total Loss: 0.8522164225578308, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8132392168045044, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1628, Total Loss: 0.8443939089775085, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8193262815475464, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1629, Total Loss: 0.8448991179466248, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8125489950180054, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1630, Total Loss: 0.8454824686050415, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8127913475036621, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1631, Total Loss: 0.8428600430488586, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8110549449920654, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1632, Total Loss: 0.843802273273468, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8123456239700317, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1633, Total Loss: 0.8414958119392395, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8161327242851257, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1634, Total Loss: 0.8453750610351562, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8117257952690125, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1635, Total Loss: 0.84366375207901, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8181313276290894, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1636, Total Loss: 0.8499215841293335, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8168515563011169, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1637, Total Loss: 0.8477699756622314, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8169940114021301, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1638, Total Loss: 0.8488989472389221, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8135970234870911, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1639, Total Loss: 0.8464104533195496, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8190953135490417, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1640, Total Loss: 0.8460808992385864, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8162796497344971, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1641, Total Loss: 0.8511095643043518, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8162054419517517, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1642, Total Loss: 0.8612186312675476, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8292055726051331, Test_Accuracy_Rate: 68.13%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1643, Total Loss: 0.8570041656494141, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8264985084533691, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1644, Total Loss: 0.8555986285209656, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8175674080848694, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1645, Total Loss: 0.8475727438926697, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.819029688835144, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1646, Total Loss: 0.8498395681381226, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.816644012928009, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1647, Total Loss: 0.8452599048614502, Accuracy Rate: 66.27%,            Test_Total_Loss: 0.8184396624565125, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1648, Total Loss: 0.8498565554618835, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8223370909690857, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1649, Total Loss: 0.9111683368682861, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8780574798583984, Test_Accuracy_Rate: 66.64%\n",
      "Epoch 1650, Total Loss: 0.9829914569854736, Accuracy Rate: 63.70%,            Test_Total_Loss: 0.9664164185523987, Test_Accuracy_Rate: 64.24%\n",
      "Epoch 1651, Total Loss: 0.9812690615653992, Accuracy Rate: 63.25%,            Test_Total_Loss: 0.9651163220405579, Test_Accuracy_Rate: 64.45%\n",
      "Epoch 1652, Total Loss: 0.9786906838417053, Accuracy Rate: 63.45%,            Test_Total_Loss: 0.8796207308769226, Test_Accuracy_Rate: 66.06%\n",
      "Epoch 1653, Total Loss: 0.9289260506629944, Accuracy Rate: 63.49%,            Test_Total_Loss: 0.901953935623169, Test_Accuracy_Rate: 66.06%\n",
      "Epoch 1654, Total Loss: 0.9169372916221619, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.8478148579597473, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 1655, Total Loss: 0.8816646933555603, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8413881659507751, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1656, Total Loss: 0.8670784831047058, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8223322629928589, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1657, Total Loss: 0.8511013984680176, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8208507895469666, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1658, Total Loss: 0.8520594239234924, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.8235107660293579, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1659, Total Loss: 0.8537319302558899, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.818143367767334, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1660, Total Loss: 0.8478929400444031, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.815863847732544, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1661, Total Loss: 0.8452951312065125, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8147218823432922, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1662, Total Loss: 0.8437869548797607, Accuracy Rate: 66.64%,            Test_Total_Loss: 0.8168635964393616, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1663, Total Loss: 0.8431302309036255, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8143725395202637, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1664, Total Loss: 0.8437734842300415, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.810657262802124, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1665, Total Loss: 0.8397126197814941, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8124781847000122, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1666, Total Loss: 0.840853750705719, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.813191294670105, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1667, Total Loss: 0.8417158722877502, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8124621510505676, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1668, Total Loss: 0.8419739007949829, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8098108172416687, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1669, Total Loss: 0.8381774425506592, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8121485114097595, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1670, Total Loss: 0.8480049967765808, Accuracy Rate: 66.68%,            Test_Total_Loss: 0.8161702156066895, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1671, Total Loss: 0.8485604524612427, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.816002607345581, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1672, Total Loss: 0.8445315361022949, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8112509250640869, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1673, Total Loss: 0.8446033000946045, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8133133053779602, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1674, Total Loss: 0.8473907113075256, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8122721910476685, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1675, Total Loss: 0.8437926769256592, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.8100360035896301, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1676, Total Loss: 0.8435818552970886, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8110100626945496, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1677, Total Loss: 0.8418163657188416, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8134029507637024, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1678, Total Loss: 0.8451471924781799, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8127103447914124, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1679, Total Loss: 0.8428133726119995, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8146554827690125, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1680, Total Loss: 0.8457384705543518, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8107705116271973, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1681, Total Loss: 0.8476264476776123, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8104157447814941, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1682, Total Loss: 0.8405777215957642, Accuracy Rate: 65.36%,            Test_Total_Loss: 0.8112556338310242, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1683, Total Loss: 0.843996524810791, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8126514554023743, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1684, Total Loss: 0.8413599729537964, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8132564425468445, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1685, Total Loss: 0.8471349477767944, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8106207847595215, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1686, Total Loss: 0.8472452759742737, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8121626973152161, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1687, Total Loss: 0.8421514630317688, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8115575909614563, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1688, Total Loss: 0.8412525057792664, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8124598264694214, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1689, Total Loss: 0.8427181243896484, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8148024082183838, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1690, Total Loss: 0.8468896150588989, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.815384566783905, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1691, Total Loss: 0.842851459980011, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8155514001846313, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1692, Total Loss: 0.8428492546081543, Accuracy Rate: 64.82%,            Test_Total_Loss: 0.8096650242805481, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1693, Total Loss: 0.838306725025177, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.813179612159729, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1694, Total Loss: 0.8439593315124512, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8109465837478638, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1695, Total Loss: 0.8444520235061646, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8097079396247864, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1696, Total Loss: 0.8484837412834167, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8158758282661438, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1697, Total Loss: 0.8565248250961304, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8357719779014587, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1698, Total Loss: 0.8686158061027527, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8472657203674316, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 1699, Total Loss: 0.8778889179229736, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.838661253452301, Test_Accuracy_Rate: 67.47%\n",
      "Epoch 1700, Total Loss: 0.8806061148643494, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.9002458453178406, Test_Accuracy_Rate: 66.76%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1701, Total Loss: 0.9008755683898926, Accuracy Rate: 64.65%,            Test_Total_Loss: 0.8505836129188538, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 1702, Total Loss: 0.8724566102027893, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8621460795402527, Test_Accuracy_Rate: 66.93%\n",
      "Epoch 1703, Total Loss: 0.8863207101821899, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8965487480163574, Test_Accuracy_Rate: 66.10%\n",
      "Epoch 1704, Total Loss: 0.9494715332984924, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.9646934866905212, Test_Accuracy_Rate: 64.94%\n",
      "Epoch 1705, Total Loss: 0.9932888150215149, Accuracy Rate: 62.50%,            Test_Total_Loss: 0.9745278358459473, Test_Accuracy_Rate: 64.20%\n",
      "Epoch 1706, Total Loss: 0.9612837433815002, Accuracy Rate: 63.95%,            Test_Total_Loss: 0.9020168781280518, Test_Accuracy_Rate: 65.85%\n",
      "Epoch 1707, Total Loss: 0.9285792112350464, Accuracy Rate: 63.82%,            Test_Total_Loss: 0.8627364635467529, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 1708, Total Loss: 0.9453418254852295, Accuracy Rate: 63.95%,            Test_Total_Loss: 0.9244681000709534, Test_Accuracy_Rate: 66.31%\n",
      "Epoch 1709, Total Loss: 0.9885022044181824, Accuracy Rate: 63.82%,            Test_Total_Loss: 0.8949646353721619, Test_Accuracy_Rate: 66.10%\n",
      "Epoch 1710, Total Loss: 1.0578103065490723, Accuracy Rate: 62.91%,            Test_Total_Loss: 0.9093559384346008, Test_Accuracy_Rate: 65.69%\n",
      "Epoch 1711, Total Loss: 0.9530339241027832, Accuracy Rate: 63.53%,            Test_Total_Loss: 0.9112544059753418, Test_Accuracy_Rate: 66.39%\n",
      "Epoch 1712, Total Loss: 0.9282974600791931, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8535125851631165, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 1713, Total Loss: 0.8937276601791382, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8527617454528809, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 1714, Total Loss: 0.8752071857452393, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8278586864471436, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1715, Total Loss: 0.8584871888160706, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8311513066291809, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1716, Total Loss: 0.8546722531318665, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8231480121612549, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1717, Total Loss: 0.8582085967063904, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8366778492927551, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1718, Total Loss: 0.8636177182197571, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8411191701889038, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1719, Total Loss: 0.8607902526855469, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.8197411894798279, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1720, Total Loss: 0.8519424796104431, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.819353461265564, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1721, Total Loss: 0.8457374572753906, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8137891888618469, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1722, Total Loss: 0.8378146290779114, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.81416255235672, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1723, Total Loss: 0.8421735167503357, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8120484352111816, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1724, Total Loss: 0.8391938805580139, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8129671812057495, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1725, Total Loss: 0.8410815000534058, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8109646439552307, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1726, Total Loss: 0.8401992321014404, Accuracy Rate: 66.85%,            Test_Total_Loss: 0.8124104738235474, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1727, Total Loss: 0.8361039757728577, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8087025284767151, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1728, Total Loss: 0.8379306793212891, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8091616630554199, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1729, Total Loss: 0.8368154764175415, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8083933591842651, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1730, Total Loss: 0.8343502879142761, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8072525858879089, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1731, Total Loss: 0.8364745378494263, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8078792691230774, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1732, Total Loss: 0.8350487351417542, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8065052628517151, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1733, Total Loss: 0.8373379707336426, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.8060821294784546, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1734, Total Loss: 0.8350217938423157, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8087329268455505, Test_Accuracy_Rate: 68.54%\n",
      "Epoch 1735, Total Loss: 0.8348251581192017, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8061422109603882, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1736, Total Loss: 0.8404015302658081, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8075944185256958, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1737, Total Loss: 0.8357622027397156, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.8153127431869507, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1738, Total Loss: 0.8457446098327637, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8119158148765564, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1739, Total Loss: 0.8403308987617493, Accuracy Rate: 66.43%,            Test_Total_Loss: 0.8143153190612793, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1740, Total Loss: 0.842578113079071, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8074231147766113, Test_Accuracy_Rate: 68.54%\n",
      "Epoch 1741, Total Loss: 0.8387324213981628, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8064570426940918, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1742, Total Loss: 0.8360028266906738, Accuracy Rate: 66.43%,            Test_Total_Loss: 0.8073108196258545, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1743, Total Loss: 0.8362936973571777, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8092935681343079, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1744, Total Loss: 0.8357968926429749, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8073398470878601, Test_Accuracy_Rate: 68.54%\n",
      "Epoch 1745, Total Loss: 0.8363033533096313, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8076370358467102, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1746, Total Loss: 0.8358156085014343, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8060182332992554, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1747, Total Loss: 0.8376553058624268, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.807648241519928, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1748, Total Loss: 0.8371485471725464, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8081628680229187, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1749, Total Loss: 0.836284875869751, Accuracy Rate: 66.60%,            Test_Total_Loss: 0.805398166179657, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1750, Total Loss: 0.835476815700531, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8100277781486511, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1751, Total Loss: 0.8379186987876892, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8071691393852234, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1752, Total Loss: 0.8377006649971008, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8065730929374695, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1753, Total Loss: 0.8379423022270203, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8076643347740173, Test_Accuracy_Rate: 68.54%\n",
      "Epoch 1754, Total Loss: 0.8396044969558716, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8073758482933044, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1755, Total Loss: 0.8359702825546265, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.805439829826355, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1756, Total Loss: 0.8397576808929443, Accuracy Rate: 66.43%,            Test_Total_Loss: 0.8108057975769043, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1757, Total Loss: 0.8397815823554993, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8063270449638367, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1758, Total Loss: 0.836348831653595, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8064048886299133, Test_Accuracy_Rate: 68.42%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1759, Total Loss: 0.8363129496574402, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.807265043258667, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1760, Total Loss: 0.8404488563537598, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8142654299736023, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1761, Total Loss: 0.8590822815895081, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8235123753547668, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1762, Total Loss: 0.8610759377479553, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.824224591255188, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1763, Total Loss: 0.8506776690483093, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8130752444267273, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1764, Total Loss: 0.8413125872612, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.814016580581665, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1765, Total Loss: 0.8478116393089294, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8160305619239807, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1766, Total Loss: 0.8473045229911804, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8139427900314331, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1767, Total Loss: 0.8463241457939148, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8122108578681946, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1768, Total Loss: 0.8426516652107239, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8093568086624146, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1769, Total Loss: 0.8438864350318909, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8136013150215149, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1770, Total Loss: 0.8486968874931335, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8124089241027832, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1771, Total Loss: 0.8436808586120605, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.808057427406311, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1772, Total Loss: 0.8469323515892029, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.8122410178184509, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1773, Total Loss: 0.8460694551467896, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8088501691818237, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1774, Total Loss: 0.8438836932182312, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8099049925804138, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1775, Total Loss: 0.841160774230957, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8074387311935425, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1776, Total Loss: 0.8418300747871399, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8088482022285461, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1777, Total Loss: 0.8429073691368103, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8141725659370422, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1778, Total Loss: 0.8441312909126282, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8206316232681274, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1779, Total Loss: 0.8470068573951721, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8081837892532349, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1780, Total Loss: 0.8432954549789429, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8169801831245422, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1781, Total Loss: 0.8437969088554382, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8125498294830322, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1782, Total Loss: 0.841064989566803, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8108283281326294, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1783, Total Loss: 0.8420326113700867, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8185514211654663, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1784, Total Loss: 0.8458348512649536, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8192735314369202, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1785, Total Loss: 0.848050594329834, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8264443278312683, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 1786, Total Loss: 0.8465669751167297, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8194540143013, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1787, Total Loss: 0.8499817848205566, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8244139552116394, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1788, Total Loss: 0.8542134165763855, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8158096671104431, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1789, Total Loss: 0.8499243855476379, Accuracy Rate: 66.60%,            Test_Total_Loss: 0.8122578263282776, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1790, Total Loss: 0.8474565744400024, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.814855694770813, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1791, Total Loss: 0.8514621257781982, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8247714042663574, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1792, Total Loss: 0.8542580008506775, Accuracy Rate: 65.44%,            Test_Total_Loss: 0.8168419003486633, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1793, Total Loss: 0.849158763885498, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8100767135620117, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1794, Total Loss: 0.8440386056900024, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.807585597038269, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1795, Total Loss: 0.8398136496543884, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8070763349533081, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1796, Total Loss: 0.8442240357398987, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8153722882270813, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1797, Total Loss: 0.8473799824714661, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8114266395568848, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1798, Total Loss: 0.8407491445541382, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8112252950668335, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1799, Total Loss: 0.8459739685058594, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8088590502738953, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1800, Total Loss: 0.8391808867454529, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8117931485176086, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1801, Total Loss: 0.846280038356781, Accuracy Rate: 66.60%,            Test_Total_Loss: 0.8125753402709961, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1802, Total Loss: 0.8397291302680969, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.8219061493873596, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1803, Total Loss: 0.8423411846160889, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8089444637298584, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1804, Total Loss: 0.8418205380439758, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8110086917877197, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1805, Total Loss: 0.8409218192100525, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8081338405609131, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1806, Total Loss: 0.8385707139968872, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8117162585258484, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1807, Total Loss: 0.8393065929412842, Accuracy Rate: 66.43%,            Test_Total_Loss: 0.8218223452568054, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1808, Total Loss: 0.8428040742874146, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8113197684288025, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1809, Total Loss: 0.8444752097129822, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8199979662895203, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1810, Total Loss: 0.8412508964538574, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8111986517906189, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1811, Total Loss: 0.8539533019065857, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8148425817489624, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1812, Total Loss: 0.854189395904541, Accuracy Rate: 65.44%,            Test_Total_Loss: 1.390630841255188, Test_Accuracy_Rate: 59.31%\n",
      "Epoch 1813, Total Loss: 1.0200673341751099, Accuracy Rate: 63.25%,            Test_Total_Loss: 0.9958833456039429, Test_Accuracy_Rate: 63.99%\n",
      "Epoch 1814, Total Loss: 1.0181080102920532, Accuracy Rate: 62.42%,            Test_Total_Loss: 0.9257184863090515, Test_Accuracy_Rate: 65.15%\n",
      "Epoch 1815, Total Loss: 0.9936724305152893, Accuracy Rate: 62.75%,            Test_Total_Loss: 0.9136707186698914, Test_Accuracy_Rate: 65.60%\n",
      "Epoch 1816, Total Loss: 0.9449283480644226, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.8992620706558228, Test_Accuracy_Rate: 66.31%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1817, Total Loss: 0.9032789468765259, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.8406438827514648, Test_Accuracy_Rate: 67.34%\n",
      "Epoch 1818, Total Loss: 0.9015021324157715, Accuracy Rate: 65.07%,            Test_Total_Loss: 0.8563469052314758, Test_Accuracy_Rate: 67.18%\n",
      "Epoch 1819, Total Loss: 0.8890103697776794, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8370493054389954, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 1820, Total Loss: 0.8642792105674744, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8323613405227661, Test_Accuracy_Rate: 67.80%\n",
      "Epoch 1821, Total Loss: 0.868111789226532, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8217044472694397, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1822, Total Loss: 0.853160560131073, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8172500133514404, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1823, Total Loss: 0.8453687429428101, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8146979212760925, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1824, Total Loss: 0.8448468446731567, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8138940930366516, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1825, Total Loss: 0.8457731008529663, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8120522499084473, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1826, Total Loss: 0.8428996205329895, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8137701153755188, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1827, Total Loss: 0.8415350317955017, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8116214275360107, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1828, Total Loss: 0.839640200138092, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8129233121871948, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1829, Total Loss: 0.8419699668884277, Accuracy Rate: 66.56%,            Test_Total_Loss: 0.8124502301216125, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1830, Total Loss: 0.8394010663032532, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8077681064605713, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1831, Total Loss: 0.8386648893356323, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.8068006634712219, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1832, Total Loss: 0.8381546139717102, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.806880533695221, Test_Accuracy_Rate: 68.58%\n",
      "Epoch 1833, Total Loss: 0.8361280560493469, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8098790049552917, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1834, Total Loss: 0.8500968813896179, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8569077849388123, Test_Accuracy_Rate: 67.30%\n",
      "Epoch 1835, Total Loss: 0.8804070353507996, Accuracy Rate: 64.94%,            Test_Total_Loss: 0.8330128788948059, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1836, Total Loss: 0.8603966236114502, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8269883990287781, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1837, Total Loss: 0.8706108927726746, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8230233192443848, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1838, Total Loss: 0.8476030230522156, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8175145983695984, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1839, Total Loss: 0.8457095623016357, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8169716596603394, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1840, Total Loss: 0.8450537323951721, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8079571723937988, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1841, Total Loss: 0.8368080258369446, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.808731734752655, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1842, Total Loss: 0.8375734686851501, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.8064420223236084, Test_Accuracy_Rate: 68.58%\n",
      "Epoch 1843, Total Loss: 0.8347973823547363, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8051120042800903, Test_Accuracy_Rate: 68.58%\n",
      "Epoch 1844, Total Loss: 0.8333579301834106, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8047924041748047, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1845, Total Loss: 0.8355275988578796, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8074557185173035, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1846, Total Loss: 0.8347073793411255, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.8093398809432983, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1847, Total Loss: 0.8409887552261353, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.809951663017273, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1848, Total Loss: 0.8402078747749329, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.808763325214386, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1849, Total Loss: 0.8432011008262634, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8099503517150879, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1850, Total Loss: 0.8391115665435791, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.815636396408081, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1851, Total Loss: 0.8537009358406067, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8228244781494141, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1852, Total Loss: 0.8634858131408691, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8195219039916992, Test_Accuracy_Rate: 67.96%\n",
      "Epoch 1853, Total Loss: 0.8474342226982117, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8226467370986938, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1854, Total Loss: 0.855246901512146, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8243284225463867, Test_Accuracy_Rate: 67.84%\n",
      "Epoch 1855, Total Loss: 0.8555241823196411, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8118359446525574, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1856, Total Loss: 0.8445003628730774, Accuracy Rate: 66.64%,            Test_Total_Loss: 0.809062659740448, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1857, Total Loss: 0.8384973406791687, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8060351610183716, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1858, Total Loss: 0.8344398140907288, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8095653653144836, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1859, Total Loss: 0.8417789936065674, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8092080950737, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1860, Total Loss: 0.8366116881370544, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8072728514671326, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1861, Total Loss: 0.8359233140945435, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8103848099708557, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1862, Total Loss: 0.8397772908210754, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8089801669120789, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1863, Total Loss: 0.8348512649536133, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8108870983123779, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1864, Total Loss: 0.8378900289535522, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8056125640869141, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1865, Total Loss: 0.831819474697113, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8041313290596008, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1866, Total Loss: 0.8349052667617798, Accuracy Rate: 65.73%,            Test_Total_Loss: 0.809043288230896, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1867, Total Loss: 0.8386792540550232, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8137688636779785, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1868, Total Loss: 0.8391535878181458, Accuracy Rate: 65.19%,            Test_Total_Loss: 0.8103013038635254, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1869, Total Loss: 0.8428857922554016, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8116980791091919, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1870, Total Loss: 0.8375716209411621, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8065619468688965, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1871, Total Loss: 0.8351712226867676, Accuracy Rate: 67.01%,            Test_Total_Loss: 0.8114811182022095, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1872, Total Loss: 0.841915488243103, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8084220886230469, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1873, Total Loss: 0.8376588225364685, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8071208000183105, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1874, Total Loss: 0.8375673890113831, Accuracy Rate: 66.27%,            Test_Total_Loss: 0.8069354295730591, Test_Accuracy_Rate: 68.13%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1875, Total Loss: 0.8367888927459717, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8052323460578918, Test_Accuracy_Rate: 68.63%\n",
      "Epoch 1876, Total Loss: 0.8408229351043701, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8061327338218689, Test_Accuracy_Rate: 68.58%\n",
      "Epoch 1877, Total Loss: 0.8416199684143066, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8078007698059082, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1878, Total Loss: 0.8343505859375, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8036569952964783, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1879, Total Loss: 0.8335385322570801, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8052473068237305, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1880, Total Loss: 0.8358489871025085, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8043157458305359, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1881, Total Loss: 0.8346988558769226, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8086549639701843, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1882, Total Loss: 0.8379470705986023, Accuracy Rate: 65.48%,            Test_Total_Loss: 0.8051229119300842, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1883, Total Loss: 0.840167760848999, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.805698812007904, Test_Accuracy_Rate: 68.63%\n",
      "Epoch 1884, Total Loss: 0.835854709148407, Accuracy Rate: 66.43%,            Test_Total_Loss: 0.8060294985771179, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1885, Total Loss: 0.8381203413009644, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8066792488098145, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1886, Total Loss: 0.8393682241439819, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8099811673164368, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1887, Total Loss: 0.8524815440177917, Accuracy Rate: 65.40%,            Test_Total_Loss: 0.8166510462760925, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1888, Total Loss: 0.8658910989761353, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8393023014068604, Test_Accuracy_Rate: 67.51%\n",
      "Epoch 1889, Total Loss: 0.9275364279747009, Accuracy Rate: 63.58%,            Test_Total_Loss: 0.9458927512168884, Test_Accuracy_Rate: 65.44%\n",
      "Epoch 1890, Total Loss: 1.0372419357299805, Accuracy Rate: 62.00%,            Test_Total_Loss: 0.8946307301521301, Test_Accuracy_Rate: 66.18%\n",
      "Epoch 1891, Total Loss: 0.9561641812324524, Accuracy Rate: 63.53%,            Test_Total_Loss: 0.919347882270813, Test_Accuracy_Rate: 65.56%\n",
      "Epoch 1892, Total Loss: 0.9084061980247498, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8392395973205566, Test_Accuracy_Rate: 67.59%\n",
      "Epoch 1893, Total Loss: 0.8723257184028625, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8287714123725891, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1894, Total Loss: 0.8593363761901855, Accuracy Rate: 65.31%,            Test_Total_Loss: 0.820877730846405, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1895, Total Loss: 0.8578298687934875, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8401116132736206, Test_Accuracy_Rate: 67.43%\n",
      "Epoch 1896, Total Loss: 0.8665887117385864, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8231819272041321, Test_Accuracy_Rate: 67.88%\n",
      "Epoch 1897, Total Loss: 0.850252628326416, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.8159388303756714, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1898, Total Loss: 0.8448944091796875, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8098637461662292, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1899, Total Loss: 0.8390483856201172, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.808338463306427, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1900, Total Loss: 0.8370903134346008, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8082900643348694, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1901, Total Loss: 0.8399286866188049, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8098545074462891, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1902, Total Loss: 0.8364230990409851, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8116278052330017, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1903, Total Loss: 0.836707353591919, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8109346032142639, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1904, Total Loss: 0.8373191952705383, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8064802289009094, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1905, Total Loss: 0.8343825936317444, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8048979043960571, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1906, Total Loss: 0.8327993154525757, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.8042072653770447, Test_Accuracy_Rate: 68.54%\n",
      "Epoch 1907, Total Loss: 0.8338469862937927, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.804135262966156, Test_Accuracy_Rate: 68.54%\n",
      "Epoch 1908, Total Loss: 0.8318599462509155, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8048476576805115, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1909, Total Loss: 0.8325448632240295, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8035086393356323, Test_Accuracy_Rate: 68.63%\n",
      "Epoch 1910, Total Loss: 0.8327100872993469, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8167724609375, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1911, Total Loss: 0.8439426422119141, Accuracy Rate: 66.51%,            Test_Total_Loss: 0.8162801265716553, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1912, Total Loss: 0.8509718179702759, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8217610120773315, Test_Accuracy_Rate: 68.13%\n",
      "Epoch 1913, Total Loss: 0.854333758354187, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8233383297920227, Test_Accuracy_Rate: 68.09%\n",
      "Epoch 1914, Total Loss: 0.8496702313423157, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8115509152412415, Test_Accuracy_Rate: 67.92%\n",
      "Epoch 1915, Total Loss: 0.840597927570343, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.813849925994873, Test_Accuracy_Rate: 68.00%\n",
      "Epoch 1916, Total Loss: 0.8432742953300476, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8100975751876831, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1917, Total Loss: 0.8390044569969177, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.8076987862586975, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1918, Total Loss: 0.8347377777099609, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.8090788722038269, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1919, Total Loss: 0.8344510793685913, Accuracy Rate: 66.56%,            Test_Total_Loss: 0.8085227608680725, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1920, Total Loss: 0.8367209434509277, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8079565167427063, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1921, Total Loss: 0.837954044342041, Accuracy Rate: 65.89%,            Test_Total_Loss: 0.8095167279243469, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1922, Total Loss: 0.8367307782173157, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8063716888427734, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1923, Total Loss: 0.8347708582878113, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8071643114089966, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1924, Total Loss: 0.8342736959457397, Accuracy Rate: 65.60%,            Test_Total_Loss: 0.8086791038513184, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1925, Total Loss: 0.8356760740280151, Accuracy Rate: 66.51%,            Test_Total_Loss: 0.805543065071106, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1926, Total Loss: 0.8375040292739868, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8092977404594421, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1927, Total Loss: 0.8464383482933044, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8237230181694031, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1928, Total Loss: 0.8598920702934265, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8362022638320923, Test_Accuracy_Rate: 67.72%\n",
      "Epoch 1929, Total Loss: 0.8547841310501099, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8336755633354187, Test_Accuracy_Rate: 67.63%\n",
      "Epoch 1930, Total Loss: 0.8737514615058899, Accuracy Rate: 65.56%,            Test_Total_Loss: 0.8548698425292969, Test_Accuracy_Rate: 67.01%\n",
      "Epoch 1931, Total Loss: 0.8744984865188599, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.832461416721344, Test_Accuracy_Rate: 67.67%\n",
      "Epoch 1932, Total Loss: 0.8648988008499146, Accuracy Rate: 65.27%,            Test_Total_Loss: 0.8472009897232056, Test_Accuracy_Rate: 67.38%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1933, Total Loss: 0.9080163836479187, Accuracy Rate: 65.23%,            Test_Total_Loss: 0.8825581073760986, Test_Accuracy_Rate: 66.68%\n",
      "Epoch 1934, Total Loss: 1.2252581119537354, Accuracy Rate: 60.26%,            Test_Total_Loss: 1.1038333177566528, Test_Accuracy_Rate: 62.33%\n",
      "Epoch 1935, Total Loss: 1.1020634174346924, Accuracy Rate: 61.09%,            Test_Total_Loss: 1.0933376550674438, Test_Accuracy_Rate: 61.51%\n",
      "Epoch 1936, Total Loss: 1.1209312677383423, Accuracy Rate: 59.81%,            Test_Total_Loss: 1.048132300376892, Test_Accuracy_Rate: 62.96%\n",
      "Epoch 1937, Total Loss: 1.1245681047439575, Accuracy Rate: 60.72%,            Test_Total_Loss: 1.0280051231384277, Test_Accuracy_Rate: 63.08%\n",
      "Epoch 1938, Total Loss: 1.0335384607315063, Accuracy Rate: 62.38%,            Test_Total_Loss: 0.9520912170410156, Test_Accuracy_Rate: 64.74%\n",
      "Epoch 1939, Total Loss: 0.985898494720459, Accuracy Rate: 63.91%,            Test_Total_Loss: 0.887610673904419, Test_Accuracy_Rate: 66.56%\n",
      "Epoch 1940, Total Loss: 0.9359312653541565, Accuracy Rate: 64.53%,            Test_Total_Loss: 0.868503749370575, Test_Accuracy_Rate: 67.26%\n",
      "Epoch 1941, Total Loss: 0.8828628659248352, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8344758749008179, Test_Accuracy_Rate: 67.76%\n",
      "Epoch 1942, Total Loss: 0.8630269765853882, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8241397142410278, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1943, Total Loss: 0.8477270007133484, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8147085309028625, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1944, Total Loss: 0.8390241861343384, Accuracy Rate: 66.60%,            Test_Total_Loss: 0.8114911913871765, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1945, Total Loss: 0.836951494216919, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8111106157302856, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1946, Total Loss: 0.8344526290893555, Accuracy Rate: 66.31%,            Test_Total_Loss: 0.8090684413909912, Test_Accuracy_Rate: 68.63%\n",
      "Epoch 1947, Total Loss: 0.8317044973373413, Accuracy Rate: 66.60%,            Test_Total_Loss: 0.8077773451805115, Test_Accuracy_Rate: 68.58%\n",
      "Epoch 1948, Total Loss: 0.8313072919845581, Accuracy Rate: 66.43%,            Test_Total_Loss: 0.8064799308776855, Test_Accuracy_Rate: 68.17%\n",
      "Epoch 1949, Total Loss: 0.8314595222473145, Accuracy Rate: 66.27%,            Test_Total_Loss: 0.8061470985412598, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1950, Total Loss: 0.8308310508728027, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8071131706237793, Test_Accuracy_Rate: 68.58%\n",
      "Epoch 1951, Total Loss: 0.8309934735298157, Accuracy Rate: 66.72%,            Test_Total_Loss: 0.8060488700866699, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1952, Total Loss: 0.8319920301437378, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.8053489327430725, Test_Accuracy_Rate: 68.54%\n",
      "Epoch 1953, Total Loss: 0.8300414085388184, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8053999543190002, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1954, Total Loss: 0.8293837308883667, Accuracy Rate: 66.27%,            Test_Total_Loss: 0.8054496645927429, Test_Accuracy_Rate: 68.54%\n",
      "Epoch 1955, Total Loss: 0.8281089663505554, Accuracy Rate: 65.81%,            Test_Total_Loss: 0.8026041984558105, Test_Accuracy_Rate: 68.63%\n",
      "Epoch 1956, Total Loss: 0.8285651803016663, Accuracy Rate: 66.43%,            Test_Total_Loss: 0.8037973642349243, Test_Accuracy_Rate: 68.58%\n",
      "Epoch 1957, Total Loss: 0.8297445178031921, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8037740588188171, Test_Accuracy_Rate: 68.67%\n",
      "Epoch 1958, Total Loss: 0.8282265067100525, Accuracy Rate: 66.85%,            Test_Total_Loss: 0.8052471876144409, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1959, Total Loss: 0.8298333883285522, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8038135766983032, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1960, Total Loss: 0.8269357085227966, Accuracy Rate: 66.43%,            Test_Total_Loss: 0.8025194406509399, Test_Accuracy_Rate: 68.75%\n",
      "Epoch 1961, Total Loss: 0.8297406435012817, Accuracy Rate: 66.27%,            Test_Total_Loss: 0.8025659918785095, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1962, Total Loss: 0.8264920115470886, Accuracy Rate: 66.23%,            Test_Total_Loss: 0.803394079208374, Test_Accuracy_Rate: 68.54%\n",
      "Epoch 1963, Total Loss: 0.8304803371429443, Accuracy Rate: 65.52%,            Test_Total_Loss: 0.803779125213623, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1964, Total Loss: 0.8288512229919434, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.803307294845581, Test_Accuracy_Rate: 68.63%\n",
      "Epoch 1965, Total Loss: 0.8282815217971802, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8016960620880127, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1966, Total Loss: 0.8296782970428467, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8041741251945496, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1967, Total Loss: 0.8274979591369629, Accuracy Rate: 66.27%,            Test_Total_Loss: 0.803586483001709, Test_Accuracy_Rate: 68.63%\n",
      "Epoch 1968, Total Loss: 0.8270242214202881, Accuracy Rate: 65.85%,            Test_Total_Loss: 0.8015459179878235, Test_Accuracy_Rate: 68.34%\n",
      "Epoch 1969, Total Loss: 0.8294807076454163, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8011969327926636, Test_Accuracy_Rate: 68.63%\n",
      "Epoch 1970, Total Loss: 0.8284353613853455, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8010796308517456, Test_Accuracy_Rate: 68.71%\n",
      "Epoch 1971, Total Loss: 0.8279771208763123, Accuracy Rate: 66.85%,            Test_Total_Loss: 0.8029672503471375, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1972, Total Loss: 0.8294408917427063, Accuracy Rate: 66.27%,            Test_Total_Loss: 0.8021237254142761, Test_Accuracy_Rate: 68.58%\n",
      "Epoch 1973, Total Loss: 0.8273839354515076, Accuracy Rate: 66.27%,            Test_Total_Loss: 0.8032836318016052, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1974, Total Loss: 0.8277461528778076, Accuracy Rate: 66.64%,            Test_Total_Loss: 0.8002346158027649, Test_Accuracy_Rate: 68.71%\n",
      "Epoch 1975, Total Loss: 0.8279300928115845, Accuracy Rate: 66.85%,            Test_Total_Loss: 0.8000012636184692, Test_Accuracy_Rate: 68.71%\n",
      "Epoch 1976, Total Loss: 0.8279516696929932, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8018571734428406, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1977, Total Loss: 0.8280562162399292, Accuracy Rate: 65.77%,            Test_Total_Loss: 0.8032423853874207, Test_Accuracy_Rate: 68.54%\n",
      "Epoch 1978, Total Loss: 0.8279586434364319, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8017607927322388, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1979, Total Loss: 0.8293653130531311, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.8051788210868835, Test_Accuracy_Rate: 68.38%\n",
      "Epoch 1980, Total Loss: 0.8329030871391296, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8053768277168274, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1981, Total Loss: 0.8296014666557312, Accuracy Rate: 66.51%,            Test_Total_Loss: 0.8023648262023926, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1982, Total Loss: 0.830483078956604, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8050876259803772, Test_Accuracy_Rate: 68.05%\n",
      "Epoch 1983, Total Loss: 0.8331570625305176, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8089986443519592, Test_Accuracy_Rate: 68.21%\n",
      "Epoch 1984, Total Loss: 0.836814284324646, Accuracy Rate: 66.14%,            Test_Total_Loss: 0.8036226630210876, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1985, Total Loss: 0.831057071685791, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8011437654495239, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1986, Total Loss: 0.8294945955276489, Accuracy Rate: 65.65%,            Test_Total_Loss: 0.8020536303520203, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1987, Total Loss: 0.8272289633750916, Accuracy Rate: 66.64%,            Test_Total_Loss: 0.802487313747406, Test_Accuracy_Rate: 68.50%\n",
      "Epoch 1988, Total Loss: 0.8295902609825134, Accuracy Rate: 66.18%,            Test_Total_Loss: 0.8039739727973938, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1989, Total Loss: 0.8297535181045532, Accuracy Rate: 65.69%,            Test_Total_Loss: 0.8031314015388489, Test_Accuracy_Rate: 68.42%\n",
      "Epoch 1990, Total Loss: 0.8288597464561462, Accuracy Rate: 66.35%,            Test_Total_Loss: 0.8006895780563354, Test_Accuracy_Rate: 68.63%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1991, Total Loss: 0.8287538886070251, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8019277453422546, Test_Accuracy_Rate: 68.71%\n",
      "Epoch 1992, Total Loss: 0.8290338516235352, Accuracy Rate: 66.47%,            Test_Total_Loss: 0.8012670874595642, Test_Accuracy_Rate: 68.54%\n",
      "Epoch 1993, Total Loss: 0.8282462358474731, Accuracy Rate: 66.10%,            Test_Total_Loss: 0.8035615682601929, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 1994, Total Loss: 0.8294286727905273, Accuracy Rate: 65.94%,            Test_Total_Loss: 0.8011434078216553, Test_Accuracy_Rate: 68.25%\n",
      "Epoch 1995, Total Loss: 0.8278380036354065, Accuracy Rate: 65.98%,            Test_Total_Loss: 0.8006498217582703, Test_Accuracy_Rate: 68.67%\n",
      "Epoch 1996, Total Loss: 0.82840496301651, Accuracy Rate: 66.51%,            Test_Total_Loss: 0.8032488822937012, Test_Accuracy_Rate: 68.54%\n",
      "Epoch 1997, Total Loss: 0.8298474550247192, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8009476065635681, Test_Accuracy_Rate: 68.54%\n",
      "Epoch 1998, Total Loss: 0.8258996605873108, Accuracy Rate: 66.02%,            Test_Total_Loss: 0.8024095892906189, Test_Accuracy_Rate: 68.29%\n",
      "Epoch 1999, Total Loss: 0.8293195962905884, Accuracy Rate: 66.39%,            Test_Total_Loss: 0.8024134635925293, Test_Accuracy_Rate: 68.46%\n",
      "Epoch 2000, Total Loss: 0.8323285579681396, Accuracy Rate: 66.06%,            Test_Total_Loss: 0.8019675612449646, Test_Accuracy_Rate: 68.50%\n"
     ]
    }
   ],
   "source": [
    "class test_model(Model):\n",
    "    def __init__(self):\n",
    "        super(test_model, self).__init__()\n",
    "        self.emb = Embedding(vocabulary_size+1, emb_dim,input_length=max_length\n",
    "                             ,trainable=True,name='embedding')\n",
    "        self.rnn = LSTM(hidden_dim,return_sequences=True,return_state=False,name='common_extract'\n",
    "                      ,trainable=True)\n",
    "        self.bn1 = BatchNormalization(name='bn1')\n",
    "        self.bn2 = BatchNormalization(name='bn2')\n",
    "        self.att = Attention(name='selfatt')\n",
    "        self.rnn2 = GRU(int(hidden_dim/2),name='common_extract2'\n",
    "                      ,trainable=True)\n",
    "        self.dn = Dense(int(hidden_dim/4),activation='selu',\n",
    "                        kernel_initializer=tf.keras.initializers.lecun_normal(), name='dn')\n",
    "        \n",
    "        self.cnn = Conv1D(int(hidden_dim/2),strides=2,kernel_size=4)\n",
    "#         self.out = Dense(max(y_train)+1,activation='softmax',name='clf')\n",
    "        self.out = Dense(max(train_Y)+1,activation='softmax',name='clf')\n",
    "        self.flat = Flatten()\n",
    "    def call(self,x):\n",
    "        #x1 = self.emb(x)\n",
    "        x = self.cnn(x)\n",
    "        x = self.rnn(x)\n",
    "        x = self.bn1(x)\n",
    "        #x = self.att([x,x1,x1])\n",
    "        #x = self.att([x,x,x])\n",
    "        x = self.rnn2(x)\n",
    "        x = self.bn2(x)\n",
    "        #x = self.flat(x)\n",
    "        x = self.dn(x)\n",
    "        y2 = self.out(x)\n",
    "        return y2\n",
    "model = test_model()\n",
    "# batch_size = 32\n",
    "try:\n",
    "    train_ds = tf.data.Dataset.from_tensor_slices((X_train,y_train)).shuffle(X_train.shape[0]).batch(batch_size)\n",
    "    valid_ds = tf.data.Dataset.from_tensor_slices((X_test,y_test)).batch(batch_size)\n",
    "except NameError:\n",
    "    train_ds = tf.data.Dataset.from_tensor_slices((train_X,train_Y)).shuffle(train_X.shape[0]).batch(batch_size)\n",
    "    valid_ds = tf.data.Dataset.from_tensor_slices((train_X,train_Y)).batch(batch_size)\n",
    "\n",
    "loss_object = tf.keras.losses.SparseCategoricalCrossentropy()\n",
    "loss_object2 = tf.keras.losses.SparseCategoricalCrossentropy()\n",
    "optimizer = tf.keras.optimizers.Nadam()\n",
    "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
    "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') \n",
    "test_loss = tf.keras.metrics.Mean(name='test_loss')\n",
    "test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')\n",
    "\n",
    "@tf.function\n",
    "def train_step(x,yc):\n",
    "    with tf.GradientTape() as tape:\n",
    "        pred_cat = model(x) #weight: pred_imp; 0/1binary: pred_imp2 。#pahse1: pred_imp; phase2; pred_imp2\n",
    "        loss = loss_object(yc, pred_cat)\n",
    "    trainable_variable = model.trainable_variables\n",
    "    gradients = tape.gradient(loss,trainable_variable)\n",
    "    optimizer.apply_gradients(zip(gradients,trainable_variable))\n",
    "    \n",
    "    train_loss(loss) #total_loss\n",
    "    train_accuracy(yc, pred_cat) #acc_rate\n",
    "\n",
    "@tf.function\n",
    "def test_step(x,yc):\n",
    "    pred_cat = model(x) #weight: pred_imp ; binary:pred_imp2\n",
    "    t_loss = loss_object2(yc, pred_cat)\n",
    "    \n",
    "    test_loss(t_loss)\n",
    "    test_accuracy(yc, pred_cat)\n",
    "\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    for text, labels in train_ds:\n",
    "        train_step(text, labels)\n",
    "\n",
    "    for test_text, test_labels in valid_ds:\n",
    "        test_step(test_text, test_labels) #with restore words\n",
    "    \n",
    "    template = 'Epoch {}, Total Loss: {}, Accuracy Rate: {:5.2f}%,\\\n",
    "            Test_Total_Loss: {}, Test_Accuracy_Rate: {:5.2f}%'\n",
    "    print(template.format(epoch+1,train_loss.result(),\n",
    "                        train_accuracy.result()*100,\n",
    "                        test_loss.result(),\n",
    "                        test_accuracy.result()*100\n",
    "                        ))\n",
    "\n",
    "    train_loss.reset_states()\n",
    "    train_accuracy.reset_states()\n",
    "    \n",
    "    test_loss.reset_states()\n",
    "    test_accuracy.reset_states()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor: shape=(2, 182, 128), dtype=float32, numpy=\n",
       " array([[[ 0.05026103, -0.03759918, -0.0075792 , ..., -0.05715944,\n",
       "          -0.00929288,  0.07889917],\n",
       "         [ 0.04546504, -0.01330616, -0.08277178, ...,  0.0956112 ,\n",
       "           0.01197832,  0.04695564],\n",
       "         [ 0.05026103, -0.03759918, -0.0075792 , ..., -0.05715944,\n",
       "          -0.00929288,  0.07889917],\n",
       "         ...,\n",
       "         [-0.05260951,  0.04872397, -0.03031672, ...,  0.00519623,\n",
       "          -0.02559018, -0.03490983],\n",
       "         [-0.05260951,  0.04872397, -0.03031672, ...,  0.00519623,\n",
       "          -0.02559018, -0.03490983],\n",
       "         [-0.05260951,  0.04872397, -0.03031672, ...,  0.00519623,\n",
       "          -0.02559018, -0.03490983]],\n",
       " \n",
       "        [[-0.03128414,  0.03349531, -0.0571312 , ...,  0.00304027,\n",
       "           0.00620059, -0.00962268],\n",
       "         [ 0.05026103, -0.03759918, -0.0075792 , ..., -0.05715944,\n",
       "          -0.00929288,  0.07889917],\n",
       "         [ 0.09278463,  0.00644924,  0.01887746, ...,  0.08126906,\n",
       "          -0.07023067,  0.0265457 ],\n",
       "         ...,\n",
       "         [-0.05260951,  0.04872397, -0.03031672, ...,  0.00519623,\n",
       "          -0.02559018, -0.03490983],\n",
       "         [-0.05260951,  0.04872397, -0.03031672, ...,  0.00519623,\n",
       "          -0.02559018, -0.03490983],\n",
       "         [-0.05260951,  0.04872397, -0.03031672, ...,  0.00519623,\n",
       "          -0.02559018, -0.03490983]]], dtype=float32)>,\n",
       " <tf.Tensor: shape=(2, 182, 1), dtype=float32, numpy=\n",
       " array([[[0.69214606],\n",
       "         [0.69168156],\n",
       "         [0.69193673],\n",
       "         [0.69214463],\n",
       "         [0.69228816],\n",
       "         [0.69238   ],\n",
       "         [0.69243497],\n",
       "         [0.69142264],\n",
       "         [0.6910454 ],\n",
       "         [0.6915591 ],\n",
       "         [0.6907061 ],\n",
       "         [0.6908472 ],\n",
       "         [0.6914505 ],\n",
       "         [0.69065773],\n",
       "         [0.69117075],\n",
       "         [0.6914711 ],\n",
       "         [0.69069684],\n",
       "         [0.6912888 ],\n",
       "         [0.6908518 ],\n",
       "         [0.6912708 ],\n",
       "         [0.69103026],\n",
       "         [0.68843305],\n",
       "         [0.6891277 ],\n",
       "         [0.6896567 ],\n",
       "         [0.6907783 ],\n",
       "         [0.69151527],\n",
       "         [0.6916537 ],\n",
       "         [0.69190425],\n",
       "         [0.69211817],\n",
       "         [0.69226897],\n",
       "         [0.692181  ],\n",
       "         [0.691805  ],\n",
       "         [0.69137156],\n",
       "         [0.69222045],\n",
       "         [0.69124943],\n",
       "         [0.69225097],\n",
       "         [0.69227904],\n",
       "         [0.69157606],\n",
       "         [0.6918166 ],\n",
       "         [0.692147  ],\n",
       "         [0.6915486 ],\n",
       "         [0.6916647 ],\n",
       "         [0.69223547],\n",
       "         [0.69164485],\n",
       "         [0.69141555],\n",
       "         [0.6924881 ],\n",
       "         [0.6920124 ],\n",
       "         [0.69275475],\n",
       "         [0.69215775],\n",
       "         [0.69284934],\n",
       "         [0.6922255 ],\n",
       "         [0.6928975 ],\n",
       "         [0.69225985],\n",
       "         [0.6929211 ],\n",
       "         [0.69227636],\n",
       "         [0.69293225],\n",
       "         [0.6922839 ],\n",
       "         [0.692937  ],\n",
       "         [0.692287  ],\n",
       "         [0.6929391 ],\n",
       "         [0.692288  ],\n",
       "         [0.6929401 ],\n",
       "         [0.6922887 ],\n",
       "         [0.6929403 ],\n",
       "         [0.6922889 ],\n",
       "         [0.69294035],\n",
       "         [0.69228894],\n",
       "         [0.69294035],\n",
       "         [0.69228894],\n",
       "         [0.6929404 ],\n",
       "         [0.6922255 ],\n",
       "         [0.6920951 ],\n",
       "         [0.6925165 ],\n",
       "         [0.6917709 ],\n",
       "         [0.69156945],\n",
       "         [0.691386  ],\n",
       "         [0.69115484],\n",
       "         [0.6909711 ],\n",
       "         [0.692018  ],\n",
       "         [0.6916751 ],\n",
       "         [0.6913805 ],\n",
       "         [0.69115597],\n",
       "         [0.69098824],\n",
       "         [0.69087255],\n",
       "         [0.69058853],\n",
       "         [0.69100726],\n",
       "         [0.69115746],\n",
       "         [0.6910864 ],\n",
       "         [0.6909703 ],\n",
       "         [0.69203633],\n",
       "         [0.69116414],\n",
       "         [0.69106287],\n",
       "         [0.69072855],\n",
       "         [0.69064367],\n",
       "         [0.6914714 ],\n",
       "         [0.6908736 ],\n",
       "         [0.6909024 ],\n",
       "         [0.69102526],\n",
       "         [0.6911241 ],\n",
       "         [0.69091326],\n",
       "         [0.69134736],\n",
       "         [0.69101906],\n",
       "         [0.6907902 ],\n",
       "         [0.6904172 ],\n",
       "         [0.69027954],\n",
       "         [0.69082373],\n",
       "         [0.6910589 ],\n",
       "         [0.69116855],\n",
       "         [0.6912194 ],\n",
       "         [0.6912408 ],\n",
       "         [0.69124705],\n",
       "         [0.69124687],\n",
       "         [0.6912437 ],\n",
       "         [0.6912401 ],\n",
       "         [0.69123644],\n",
       "         [0.69123393],\n",
       "         [0.6912321 ],\n",
       "         [0.6912305 ],\n",
       "         [0.6912292 ],\n",
       "         [0.69122887],\n",
       "         [0.69122803],\n",
       "         [0.69122785],\n",
       "         [0.69122756],\n",
       "         [0.6912274 ],\n",
       "         [0.6912273 ],\n",
       "         [0.69122726],\n",
       "         [0.69122726],\n",
       "         [0.6912272 ],\n",
       "         [0.69122714],\n",
       "         [0.69122726],\n",
       "         [0.6912272 ],\n",
       "         [0.6912272 ],\n",
       "         [0.6912272 ],\n",
       "         [0.6912272 ],\n",
       "         [0.6912272 ],\n",
       "         [0.6912272 ],\n",
       "         [0.6912272 ],\n",
       "         [0.69122714],\n",
       "         [0.6912272 ],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714],\n",
       "         [0.69122714]],\n",
       " \n",
       "        [[0.7123216 ],\n",
       "         [0.71249956],\n",
       "         [0.71163934],\n",
       "         [0.71159124],\n",
       "         [0.7120533 ],\n",
       "         [0.71126944],\n",
       "         [0.7113326 ],\n",
       "         [0.7118592 ],\n",
       "         [0.7123859 ],\n",
       "         [0.71261597],\n",
       "         [0.71252334],\n",
       "         [0.7122388 ],\n",
       "         [0.7125617 ],\n",
       "         [0.71287787],\n",
       "         [0.71208394],\n",
       "         [0.71236295],\n",
       "         [0.7121548 ],\n",
       "         [0.7122527 ],\n",
       "         [0.71198934],\n",
       "         [0.71013755],\n",
       "         [0.7111206 ],\n",
       "         [0.711564  ],\n",
       "         [0.7117813 ],\n",
       "         [0.71221834],\n",
       "         [0.7114817 ],\n",
       "         [0.71193933],\n",
       "         [0.71145505],\n",
       "         [0.7113308 ],\n",
       "         [0.71234506],\n",
       "         [0.712947  ],\n",
       "         [0.71295136],\n",
       "         [0.712799  ],\n",
       "         [0.7117362 ],\n",
       "         [0.71129745],\n",
       "         [0.7120154 ],\n",
       "         [0.7122224 ],\n",
       "         [0.712533  ],\n",
       "         [0.71211636],\n",
       "         [0.7126842 ],\n",
       "         [0.7116541 ],\n",
       "         [0.71173483],\n",
       "         [0.7123398 ],\n",
       "         [0.71253556],\n",
       "         [0.71256685],\n",
       "         [0.7125422 ],\n",
       "         [0.7125034 ],\n",
       "         [0.71246624],\n",
       "         [0.7124381 ],\n",
       "         [0.71241695],\n",
       "         [0.71240264],\n",
       "         [0.712394  ],\n",
       "         [0.71238756],\n",
       "         [0.712384  ],\n",
       "         [0.71238166],\n",
       "         [0.71238005],\n",
       "         [0.7123791 ],\n",
       "         [0.712379  ],\n",
       "         [0.7123786 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123785 ],\n",
       "         [0.7123783 ],\n",
       "         [0.71237826],\n",
       "         [0.7123783 ],\n",
       "         [0.7123783 ],\n",
       "         [0.71237826],\n",
       "         [0.7123784 ],\n",
       "         [0.71237826],\n",
       "         [0.71237826],\n",
       "         [0.71237826],\n",
       "         [0.71237826],\n",
       "         [0.71237826],\n",
       "         [0.71237826],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ],\n",
       "         [0.7123784 ]]], dtype=float32)>)"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class base_model_1(Model):\n",
    "    def __init__(self):\n",
    "        super(base_model_1, self).__init__()\n",
    "        self.mask = Masking(mask_value=0)\n",
    "        self.emb = Embedding(vocabulary_size, emb_dim,input_length=max_length\n",
    "                             ,trainable=True,name='embedding')\n",
    "        self.rnn1 = GRU(emb_dim,return_sequences=True,return_state=False,name='common_extract'\n",
    "                      ,trainable=True)\n",
    "        self.att = Attention(name='selfatt')\n",
    "        self.bn1 = BatchNormalization(name='bn1')\n",
    "        #self.fil = Dense(1,activation=onezero,name='filter_out')\n",
    "        self.fil = TimeDistributed(Dense(1,activation=onezero,kernel_initializer=init_w,bias_initializer=init_b, name='filter_out'),name='TD2') #relu/linear/step function\n",
    "\n",
    "    def call(self,x):\n",
    "        x = self.mask(x)\n",
    "        x1 = self.emb(x)\n",
    "        #x = self.att([x1,x1])\n",
    "        x = self.rnn1(x1)\n",
    "        x = self.bn1(x)\n",
    "        x = self.att([x,x1,x1])\n",
    "        y = self.fil(x)\n",
    "        return x1,y\n",
    "\n",
    "model1 = base_model_1()\n",
    "model1.load_weights(saveP1)\n",
    "model1(X_train[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "class base_model_2(Model):\n",
    "    def __init__(self):\n",
    "        super(base_model_2, self).__init__()\n",
    "        self.mul = Multiply()\n",
    "        #self.rnn2 = Bidirectional(LSTM(int(hidden_dim),dropout=do,recurrent_dropout=do,name='lstm'))\n",
    "        self.rnn3 = GRU(int(hidden_dim/2))\n",
    "        self.bn2 = BatchNormalization(name='bn2')\n",
    "        self.dn = Dense(int(hidden_dim/4),activation='selu',\n",
    "                        kernel_initializer=tf.keras.initializers.lecun_normal(), name='dn')\n",
    "        self.out = Dense(max(y_train)+1,activation='softmax',name='clf')\n",
    "\n",
    "    def call(self,x1,y1):\n",
    "        x2 = self.mul([y1,x1])\n",
    "        x = self.rnn3(x2) #x2 #y1=weight|binary\n",
    "        x = self.bn2(x)\n",
    "        x = self.dn(x)\n",
    "        y2 = self.out(x)\n",
    "        return y2\n",
    "    \n",
    "model2 = base_model_2()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1, 182, 181), dtype=float32, numpy=\n",
       "array([[[1., 0., 0., ..., 0., 0., 0.],\n",
       "        [1., 1., 0., ..., 0., 0., 0.],\n",
       "        [0., 1., 1., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 1., 1., 0.],\n",
       "        [0., 0., 0., ..., 0., 1., 1.],\n",
       "        [0., 0., 0., ..., 0., 0., 1.]]], dtype=float32)>"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seq_num = 2 #連續幾個才叫做連續，要改modify_idx看有幾個\n",
    "\n",
    "arr_len = max_length - seq_num + 1\n",
    "seq_arr = []\n",
    "for i in range(arr_len):\n",
    "    ori_np = np.array([0]*max_length)\n",
    "    modify_idx = [i,i+1] #要跟著seq_num改\n",
    "    ori_np[modify_idx]=1\n",
    "    seq_arr.append(ori_np)\n",
    "seq_arr = np.array(seq_arr)\n",
    "seq_mask = tf.Variable(seq_arr.T,dtype='float32')\n",
    "seq_mask = tf.expand_dims(seq_mask,axis=0)\n",
    "seq_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_object1(predictions): #filter loss\n",
    "    mask = tf.math.logical_not(tf.math.equal(predictions, 0))\n",
    "    loss_ = tf.reduce_mean(predictions)\n",
    "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
    "    loss_ *= mask\n",
    "    return tf.reduce_mean(loss_)\n",
    "def one_percentage(predictions): #1 num\n",
    "    mask = tf.math.logical_not(tf.math.equal(predictions, 0))\n",
    "    loss_ = tf.reduce_mean(predictions)\n",
    "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
    "    loss_ *= mask\n",
    "    return tf.reduce_mean(loss_)\n",
    "def seq_loss(predictions):\n",
    "    mask = tf.math.logical_not(tf.math.equal(predictions, 0))\n",
    "    predictions = tf.squeeze(predictions,axis=-1)\n",
    "    results = tf.matmul(predictions,seq_mask)\n",
    "    results = tf.where(results==seq_num,1.0,0.0)\n",
    "    loss_ = tf.reduce_mean(results)\n",
    "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
    "    loss_ *= mask\n",
    "    return tf.reduce_mean(loss_)\n",
    "def word_reproduce1(prediction_int):\n",
    "    return prediction_int\n",
    "def word_reproduce2(prediction_int):\n",
    "    return prediction_int\n",
    "loss_object2 = tf.keras.losses.SparseCategoricalCrossentropy() #clf loss\n",
    "\n",
    "# optimizer1 = tf.keras.optimizers.SGD(learning_rate=0.0044, momentum=0.5, nesterov=True)\n",
    "optimizer1 = tf.keras.optimizers.Nadam()#Adam(learning_rate=0.0003)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loss = tf.keras.metrics.Mean(name='train_loss') #total_loss\n",
    "train_accloss = tf.keras.metrics.Mean(name='train_accloss')#loss_acc\n",
    "train_filloss = tf.keras.metrics.Mean(name='train_filloss') #loss_filter\n",
    "train_seqloss = tf.keras.metrics.Mean(name='train_seqloss') #loss_seq\n",
    "train_ones = tf.keras.metrics.Mean(name='train_ones') #ones_num\n",
    "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') #acc_rate\n",
    "\n",
    "test_loss = tf.keras.metrics.Mean(name='test_loss') #total_loss\n",
    "test_accloss = tf.keras.metrics.Mean(name='test_accloss')#loss_acc\n",
    "test_filloss = tf.keras.metrics.Mean(name='test_filloss') #loss_filter\n",
    "test_seqloss = tf.keras.metrics.Mean(name='test_seqloss')\n",
    "test_ones = tf.keras.metrics.Mean(name='test_ones') #ones_num\n",
    "test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha = 0.0000001#0.000002 #pahse1: -0.1 / 0.0 ; phase2: 0.01~0.05~0.1 有1-就是希望1越多，沒1-就是希望0越多1越少\n",
    "beta = 1.0 #clf loss 越大越要求分好\n",
    "gamma = 0.0 #seqloss 越大越要求連續\n",
    "#-0.001 / 1.0 / 1.0\n",
    "\n",
    "@tf.function\n",
    "def train_step(x,yc):\n",
    "    with tf.GradientTape(persistent=False) as tape:\n",
    "        emb, pred_imp = model1(x)\n",
    "        #loss1 = alpha*loss_object1(pred_imp) #phase1\n",
    "        #pred_imp2 = tf.math.round(pred_imp)\n",
    "        #pred_imp3 = tf.clip_by_value(pred_imp,clip_value_max=1,clip_value_min=0)\n",
    "        pred_imp2 = tf.math.round(pred_imp)\n",
    "        loss1 = 1-loss_object1(pred_imp) #有1-就是希望1越多，沒1-就是希望0越多1越少 #pahse2:alpha*loss_object1(pred_imp) ; phase1: alpha*(1-loss_object1(pred_imp))\n",
    "        pred_cat = model2(emb,pred_imp2) #weight: pred_imp; 0/1binary: pred_imp2 。#pahse1: pred_imp; phase2; pred_imp2\n",
    "        loss2 = loss_object2(yc, pred_cat)\n",
    "        loss3 = 1-seq_loss(pred_imp2)\n",
    "        loss = alpha*loss1 + beta*loss2 + gamma*loss3\n",
    "    trainable_variable = model1.trainable_variables\n",
    "    trainable_variable.extend(model2.trainable_variables)\n",
    "    gradients = tape.gradient(loss,trainable_variable)\n",
    "    optimizer1.apply_gradients(zip(gradients,trainable_variable))\n",
    "    \n",
    "    train_loss(loss) #total_loss\n",
    "    train_filloss(loss1)\n",
    "    train_accloss(loss2)\n",
    "    train_seqloss(loss3) #loss_seq\n",
    "    train_accuracy(yc, pred_cat) #acc_rate\n",
    "    ones = one_percentage(pred_imp2) #pred_imp2\n",
    "    train_ones(ones) #ones_num\n",
    "    \n",
    "    \n",
    "@tf.function\n",
    "def test_step(x,yc):\n",
    "    emb, pred_imp = model1(x)\n",
    "    #loss1 = alpha*loss_object1(pred_imp) #phase1\n",
    "    #pred_imp2 = tf.math.round(pred_imp)\n",
    "    #pred_imp3 = tf.clip_by_value(pred_imp,clip_value_max=1,clip_value_min=0)\n",
    "    pred_imp2 = tf.math.round(pred_imp)\n",
    "    loss1 = loss_object1(pred_imp) #phase2\n",
    "    pred_cat = model2(emb,pred_imp2) #weight: pred_imp ; binary:pred_imp2\n",
    "    loss2 = loss_object2(yc, pred_cat)\n",
    "    loss3 = 1-seq_loss(pred_imp2)\n",
    "    #t_loss = loss1 + loss2\n",
    "    t_loss = alpha*loss1 + beta*loss2 + gamma*loss3\n",
    "    \n",
    "    test_loss(t_loss)\n",
    "    test_filloss(loss1)\n",
    "    test_accloss(loss2)\n",
    "    test_seqloss(loss3)\n",
    "    test_accuracy(yc, pred_cat)\n",
    "    t_ones = one_percentage(pred_imp2) #pred_imp2\n",
    "    test_ones(t_ones)\n",
    "\n",
    "    \n",
    "# word_idx_pd = pd.DataFrame(data=word_index,index=[\"ID\"]).T\n",
    "# word_idx_pd['ori_word'] = word_idx_pd.index \n",
    "# word_idx_pd = word_idx_pd.set_index([\"ID\"])\n",
    "# print(word_idx_pd)\n",
    "\n",
    "ep = -1\n",
    "@tf.function(experimental_relax_shapes=True)\n",
    "def test_step_inf(x,yc,epoch):\n",
    "    #ori_text = getKeysByValues(list(x))\n",
    "    emb, pred_imp = model1(x)\n",
    "    #loss1 = alpha*loss_object1(pred_imp) #phase1\n",
    "    #pred_imp2 = tf.math.round(pred_imp)\n",
    "    #pred_imp3 = tf.clip_by_value(pred_imp,clip_value_max=1,clip_value_min=0)\n",
    "    pred_imp2 = tf.math.round(pred_imp)\n",
    "    #if ep != epoch:\n",
    "        #ep = epoch\n",
    "        #x = tf.dtypes.cast(x,dtype=tf.float32)\n",
    "    pred_imp3_ = tf.squeeze(pred_imp2)\n",
    "    pred_imp3 = tf.dtypes.cast(pred_imp3_,dtype=tf.int32)\n",
    "    used_x = tf.math.multiply(x,pred_imp3)\n",
    "    filter_emb = tf.math.multiply(emb,pred_imp2)\n",
    "        #word_reproduce1(x)\n",
    "        #word_reproduce2(used_x)\n",
    "        #used_text = getKeysByValues(list(used_x))\n",
    "        #tf.print(used_x)\n",
    "        #print(emb.numpy(),pred_imp, x,pred_imp3,used_x)\n",
    "        #int_ori_text.append(x)\n",
    "        #int_fil_text.append(used_x)\n",
    "    \n",
    "    loss1 = loss_object1(pred_imp) #phase2\n",
    "    pred_cat = model2(emb,pred_imp2) #weight: pred_imp ; binary:pred_imp2\n",
    "    loss2 = loss_object2(yc, pred_cat)\n",
    "    loss3 = 1-seq_loss(pred_imp2)\n",
    "    #t_loss = loss1 + loss2\n",
    "    t_loss = alpha*loss1 + beta*loss2 + gamma*loss3\n",
    "    \n",
    "    test_loss(t_loss)\n",
    "    test_filloss(loss1)\n",
    "    test_accloss(loss2)\n",
    "    test_seqloss(loss3)\n",
    "    test_accuracy(yc, pred_cat)\n",
    "    t_ones = one_percentage(pred_imp2) #pred_imp2\n",
    "    test_ones(t_ones)\n",
    "    return x, used_x, pred_cat, emb, filter_emb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Total Loss: 0.8048461675643921, Clf Loss: 0.0, Filter Loss: 0.0, Seq Loss: 0.0, Accuracy Rate: 69.15%, Ones Portion: 0.0,             Test_Total_Loss: 4.283809185028076, Test_Clf_Loss: 4.283809185028076, Test_Filter_Loss: 0.9998735189437866, TEST_Seq_Loss: 0.0, Test_Accuracy_Rate: 30.38%, Test_Ones_Portion: 1.0\n",
      "===MODEL WEIGHTS SAVED=== ./model/Hooklog/2019122601/model1 ./model/Hooklog/2019122601/model2\n",
      "Epoch 2, Total Loss: 0.8320643305778503, Clf Loss: 0.0, Filter Loss: 0.0, Seq Loss: 0.0, Accuracy Rate: 68.14%, Ones Portion: 0.0,             Test_Total_Loss: 4.283809185028076, Test_Clf_Loss: 4.283809185028076, Test_Filter_Loss: 0.9998735189437866, TEST_Seq_Loss: 0.0, Test_Accuracy_Rate: 30.38%, Test_Ones_Portion: 1.0\n",
      "===MODEL WEIGHTS SAVED=== ./model/Hooklog/2019122601/model1 ./model/Hooklog/2019122601/model2\n",
      "Epoch 3, Total Loss: 0.8325566053390503, Clf Loss: 0.0, Filter Loss: 0.0, Seq Loss: 0.0, Accuracy Rate: 68.33%, Ones Portion: 0.0,             Test_Total_Loss: 4.283809185028076, Test_Clf_Loss: 4.283809185028076, Test_Filter_Loss: 0.9998735189437866, TEST_Seq_Loss: 0.0, Test_Accuracy_Rate: 30.38%, Test_Ones_Portion: 1.0\n",
      "===MODEL WEIGHTS SAVED=== ./model/Hooklog/2019122601/model1 ./model/Hooklog/2019122601/model2\n",
      "Epoch 4, Total Loss: 0.8727174401283264, Clf Loss: 0.0, Filter Loss: 0.0, Seq Loss: 0.0, Accuracy Rate: 67.90%, Ones Portion: 0.0,             Test_Total_Loss: 4.283809185028076, Test_Clf_Loss: 4.283809185028076, Test_Filter_Loss: 0.9998735189437866, TEST_Seq_Loss: 0.0, Test_Accuracy_Rate: 30.38%, Test_Ones_Portion: 1.0\n",
      "===MODEL WEIGHTS SAVED=== ./model/Hooklog/2019122601/model1 ./model/Hooklog/2019122601/model2\n",
      "Epoch 5, Total Loss: 0.9017594456672668, Clf Loss: 0.0, Filter Loss: 0.0, Seq Loss: 0.0, Accuracy Rate: 66.64%, Ones Portion: 0.0,             Test_Total_Loss: 4.283809185028076, Test_Clf_Loss: 4.283809185028076, Test_Filter_Loss: 0.9998735189437866, TEST_Seq_Loss: 0.0, Test_Accuracy_Rate: 30.38%, Test_Ones_Portion: 1.0\n",
      "===MODEL WEIGHTS SAVED=== ./model/Hooklog/2019122601/model1 ./model/Hooklog/2019122601/model2\n",
      "Epoch 6, Total Loss: 0.9170001149177551, Clf Loss: 0.0, Filter Loss: 0.0, Seq Loss: 0.0, Accuracy Rate: 66.07%, Ones Portion: 0.0,             Test_Total_Loss: 4.283809185028076, Test_Clf_Loss: 4.283809185028076, Test_Filter_Loss: 0.9998735189437866, TEST_Seq_Loss: 0.0, Test_Accuracy_Rate: 30.38%, Test_Ones_Portion: 1.0\n",
      "===MODEL WEIGHTS SAVED=== ./model/Hooklog/2019122601/model1 ./model/Hooklog/2019122601/model2\n",
      "Epoch 7, Total Loss: 0.9004886746406555, Clf Loss: 0.0, Filter Loss: 0.0, Seq Loss: 0.0, Accuracy Rate: 66.73%, Ones Portion: 0.0,             Test_Total_Loss: 4.283809185028076, Test_Clf_Loss: 4.283809185028076, Test_Filter_Loss: 0.9998735189437866, TEST_Seq_Loss: 0.0, Test_Accuracy_Rate: 30.38%, Test_Ones_Portion: 1.0\n",
      "===MODEL WEIGHTS SAVED=== ./model/Hooklog/2019122601/model1 ./model/Hooklog/2019122601/model2\n",
      "Epoch 8, Total Loss: 0.8528181910514832, Clf Loss: 0.0, Filter Loss: 0.0, Seq Loss: 0.0, Accuracy Rate: 68.00%, Ones Portion: 0.0,             Test_Total_Loss: 4.283809185028076, Test_Clf_Loss: 4.283809185028076, Test_Filter_Loss: 0.9998735189437866, TEST_Seq_Loss: 0.0, Test_Accuracy_Rate: 30.38%, Test_Ones_Portion: 1.0\n",
      "===MODEL WEIGHTS SAVED=== ./model/Hooklog/2019122601/model1 ./model/Hooklog/2019122601/model2\n",
      "Epoch 9, Total Loss: 0.8446152210235596, Clf Loss: 0.0, Filter Loss: 0.0, Seq Loss: 0.0, Accuracy Rate: 67.90%, Ones Portion: 0.0,             Test_Total_Loss: 4.283809185028076, Test_Clf_Loss: 4.283809185028076, Test_Filter_Loss: 0.9998735189437866, TEST_Seq_Loss: 0.0, Test_Accuracy_Rate: 30.38%, Test_Ones_Portion: 1.0\n",
      "===MODEL WEIGHTS SAVED=== ./model/Hooklog/2019122601/model1 ./model/Hooklog/2019122601/model2\n",
      "Epoch 10, Total Loss: 0.8383711576461792, Clf Loss: 0.0, Filter Loss: 0.0, Seq Loss: 0.0, Accuracy Rate: 68.56%, Ones Portion: 0.0,             Test_Total_Loss: 4.283809185028076, Test_Clf_Loss: 4.283809185028076, Test_Filter_Loss: 0.9998735189437866, TEST_Seq_Loss: 0.0, Test_Accuracy_Rate: 30.38%, Test_Ones_Portion: 1.0\n",
      "===MODEL WEIGHTS SAVED=== ./model/Hooklog/2019122601/model1 ./model/Hooklog/2019122601/model2\n",
      "Epoch 11, Total Loss: 0.8207250833511353, Clf Loss: 0.0, Filter Loss: 0.0, Seq Loss: 0.0, Accuracy Rate: 68.28%, Ones Portion: 0.0,             Test_Total_Loss: 4.283809185028076, Test_Clf_Loss: 4.283809185028076, Test_Filter_Loss: 0.9998735189437866, TEST_Seq_Loss: 0.0, Test_Accuracy_Rate: 30.38%, Test_Ones_Portion: 1.0\n",
      "===MODEL WEIGHTS SAVED=== ./model/Hooklog/2019122601/model1 ./model/Hooklog/2019122601/model2\n",
      "Epoch 12, Total Loss: 0.7937115430831909, Clf Loss: 0.0, Filter Loss: 0.0, Seq Loss: 0.0, Accuracy Rate: 68.89%, Ones Portion: 0.0,             Test_Total_Loss: 4.283809185028076, Test_Clf_Loss: 4.283809185028076, Test_Filter_Loss: 0.9998735189437866, TEST_Seq_Loss: 0.0, Test_Accuracy_Rate: 30.38%, Test_Ones_Portion: 1.0\n",
      "===MODEL WEIGHTS SAVED=== ./model/Hooklog/2019122601/model1 ./model/Hooklog/2019122601/model2\n",
      "Epoch 13, Total Loss: 0.7880153656005859, Clf Loss: 0.0, Filter Loss: 0.0, Seq Loss: 0.0, Accuracy Rate: 69.97%, Ones Portion: 0.0,             Test_Total_Loss: 4.283809185028076, Test_Clf_Loss: 4.283809185028076, Test_Filter_Loss: 0.9998735189437866, TEST_Seq_Loss: 0.0, Test_Accuracy_Rate: 30.38%, Test_Ones_Portion: 1.0\n",
      "===MODEL WEIGHTS SAVED=== ./model/Hooklog/2019122601/model1 ./model/Hooklog/2019122601/model2\n",
      "Epoch 14, Total Loss: 0.7830978035926819, Clf Loss: 0.0, Filter Loss: 0.0, Seq Loss: 0.0, Accuracy Rate: 69.08%, Ones Portion: 0.0,             Test_Total_Loss: 4.283809185028076, Test_Clf_Loss: 4.283809185028076, Test_Filter_Loss: 0.9998735189437866, TEST_Seq_Loss: 0.0, Test_Accuracy_Rate: 30.38%, Test_Ones_Portion: 1.0\n",
      "===MODEL WEIGHTS SAVED=== ./model/Hooklog/2019122601/model1 ./model/Hooklog/2019122601/model2\n",
      "Epoch 15, Total Loss: 0.768572986125946, Clf Loss: 0.0, Filter Loss: 0.0, Seq Loss: 0.0, Accuracy Rate: 68.94%, Ones Portion: 0.0,             Test_Total_Loss: 4.283809185028076, Test_Clf_Loss: 4.283809185028076, Test_Filter_Loss: 0.9998735189437866, TEST_Seq_Loss: 0.0, Test_Accuracy_Rate: 30.38%, Test_Ones_Portion: 1.0\n",
      "===MODEL WEIGHTS SAVED=== ./model/Hooklog/2019122601/model1 ./model/Hooklog/2019122601/model2\n",
      "Epoch 16, Total Loss: 0.7659801244735718, Clf Loss: 0.0, Filter Loss: 0.0, Seq Loss: 0.0, Accuracy Rate: 69.50%, Ones Portion: 0.0,             Test_Total_Loss: 4.283809185028076, Test_Clf_Loss: 4.283809185028076, Test_Filter_Loss: 0.9998735189437866, TEST_Seq_Loss: 0.0, Test_Accuracy_Rate: 30.38%, Test_Ones_Portion: 1.0\n",
      "===MODEL WEIGHTS SAVED=== ./model/Hooklog/2019122601/model1 ./model/Hooklog/2019122601/model2\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-135-6cfb3144826b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     34\u001b[0m     \u001b[0mepoch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconvert_to_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint64\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtrain_ds\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 36\u001b[0;31m         \u001b[0mtrain_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mtest_text\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_labels\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mvalid_ds\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    566\u001b[0m         \u001b[0mxla_context\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    567\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 568\u001b[0;31m       \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    569\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    570\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    597\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    598\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 599\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    600\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    601\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2361\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2362\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2363\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2364\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2365\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[0;34m(self, args, kwargs)\u001b[0m\n\u001b[1;32m   1609\u001b[0m          if isinstance(t, (ops.Tensor,\n\u001b[1;32m   1610\u001b[0m                            resource_variable_ops.BaseResourceVariable))),\n\u001b[0;32m-> 1611\u001b[0;31m         self.captured_inputs)\n\u001b[0m\u001b[1;32m   1612\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1613\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1690\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1691\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1692\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1693\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1694\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    543\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    544\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"executor_type\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexecutor_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"config_proto\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 545\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    547\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tensorflow.TFE_Py_Execute(ctx._handle, device_name,\n\u001b[1;32m     60\u001b[0m                                                \u001b[0mop_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m                                                num_outputs)\n\u001b[0m\u001b[1;32m     62\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "EPOCHS = 2000\n",
    "DateID = '2019122601'\n",
    "\n",
    "saveP1 = './model/Hooklog/'+DateID+'/model1'\n",
    "saveP2 = './model/Hooklog/'+DateID+'/model2'\n",
    "train_loss_acc = []\n",
    "train_loss_filter = []\n",
    "train_loss_seq = []\n",
    "train_weighted_loss = []\n",
    "train_acc_rate = []\n",
    "train_ones_num = []\n",
    "\n",
    "test_loss_acc = []\n",
    "test_loss_filter = []\n",
    "test_loss_seq = []\n",
    "test_weighted_loss = []\n",
    "test_acc_rate = []\n",
    "test_ones_num = []\n",
    "\n",
    "int_ori_text =[]\n",
    "int_fil_text = []\n",
    "int_label = []\n",
    "pred_label = []\n",
    "ori_emb = []\n",
    "fil_emb = []\n",
    "epoch_num = []\n",
    "fam_lab = []\n",
    "pred_lab = []\n",
    "# signature_dict = {'att':model1.att}\n",
    "\n",
    "gc.collect()\n",
    "best_clf = 0.0\n",
    "for epoch in range(EPOCHS):\n",
    "    epoch = tf.convert_to_tensor(epoch, dtype=tf.int64)\n",
    "    for text, labels in train_ds:\n",
    "        train_step(text, labels)\n",
    "\n",
    "    for test_text, test_labels in valid_ds:\n",
    "        ori,fil,pred_cat,emb,filter_emb = test_step_inf(test_text, test_labels,epoch) #with restore words\n",
    "        ori_emb.append(emb.numpy()) #一個epoch有很多個batches\n",
    "        fil_emb.append(filter_emb.numpy()) #一個epoch有很多個batches\n",
    "        epoch_num.append(epoch.numpy()) #一個epoch一個數字\n",
    "        fam_lab.append(test_labels.numpy())#一個epoch有很多個batches\n",
    "        pred_lab.append(pred_cat.numpy())#一個epoch有很多個batches\n",
    "        #print(k2)\n",
    "        #test_step(test_text, test_labels) #original\n",
    "    int_ori_text.append(ori.numpy())\n",
    "    int_fil_text.append(fil.numpy())\n",
    "    int_label.append(test_labels.numpy()) #true label\n",
    "    pred_label.append(pred_cat.numpy())\n",
    "\n",
    "    \n",
    "    template = 'Epoch {}, Total Loss: {}, Clf Loss: {}, Filter Loss: {}, Seq Loss: {}, Accuracy Rate: {:5.2f}%, Ones Portion: {}, \\\n",
    "            Test_Total_Loss: {}, Test_Clf_Loss: {}, Test_Filter_Loss: {}, TEST_Seq_Loss: {}, Test_Accuracy_Rate: {:5.2f}%, Test_Ones_Portion: {}'\n",
    "    print(template.format(epoch+1,train_loss.result(),\n",
    "                          train_accloss.result(),train_filloss.result(),train_seqloss.result(),\n",
    "                        train_accuracy.result()*100,train_ones.result(),\n",
    "                        test_loss.result(),\n",
    "                        test_accloss.result(),test_filloss.result(),test_seqloss.result(),\n",
    "                        test_accuracy.result()*100,test_ones.result(),\n",
    "                        ))\n",
    "    #int_ori_text.append(word_reproduce1.result().numpy())\n",
    "    #int_fil_text.append(word_reproduce2.result().numpy())\n",
    "    train_loss_acc.append( train_accloss.result().numpy())\n",
    "    train_loss_filter.append( train_filloss.result().numpy())\n",
    "    train_loss_seq.append( train_seqloss.result().numpy())\n",
    "    train_weighted_loss.append( train_loss.result().numpy())\n",
    "    train_acc_rate.append( train_accuracy.result().numpy())\n",
    "    train_ones_num.append( train_ones.result().numpy())\n",
    "    \n",
    "    test_loss_acc.append( test_accloss.result().numpy())\n",
    "    test_loss_filter.append( test_filloss.result().numpy())\n",
    "    test_loss_seq.append( test_seqloss.result().numpy())\n",
    "    test_weighted_loss.append( test_loss.result().numpy())\n",
    "    test_acc_rate.append( test_accuracy.result().numpy())\n",
    "    test_ones_num.append( test_ones.result().numpy())\n",
    "    if best_clf<=test_accuracy.result()*100:\n",
    "        #tf.saved_model.save(model1,saveP1+'_all')\n",
    "        #model1.save(saveP1,save_format='h5')\n",
    "        #model2.save(saveP2,save_format='h5')\n",
    "        #tf.saved_model.save(model2,saveP2+'_all')\n",
    "        model1.save_weights(saveP1,save_format='tf')\n",
    "        model2.save_weights(saveP2,save_format='tf')\n",
    "        best_clf = test_accuracy.result()*100\n",
    "        print('===MODEL WEIGHTS SAVED===',saveP1,saveP2)\n",
    "    # Reset the metrics for the next epoch\n",
    "    #word_reproduce1.reset_states()\n",
    "    #word_reproduce2.reset_states()\n",
    "    train_loss.reset_states()\n",
    "    train_accloss.reset_states()\n",
    "    train_filloss.reset_states()\n",
    "    train_seqloss.reset_states()\n",
    "    train_ones.reset_states()\n",
    "    train_accuracy.reset_states()\n",
    "    \n",
    "    test_loss.reset_states()\n",
    "    test_accloss.reset_states()\n",
    "    test_filloss.reset_states()\n",
    "    test_seqloss.reset_states()\n",
    "    test_ones.reset_states()\n",
    "    test_accuracy.reset_states()\n",
    "pred_lab2 = []\n",
    "for pred in pred_lab:\n",
    "    pred_lab2.append(list(np.argmax(pred,axis=-1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "128"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_lab2 = []\n",
    "for pred in pred_lab:\n",
    "    pred_lab2.append(list(np.argmax(pred,axis=-1)))\n",
    "len(pred_lab2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 279,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# len(ep)\n",
    "ep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EP: 0\n",
      "samples#: 32\n",
      "samples#: 64\n",
      "samples#: 96\n",
      "samples#: 128\n",
      "samples#: 160\n",
      "samples#: 192\n",
      "samples#: 224\n",
      "samples#: 237\n",
      "EP: 1\n",
      "samples#: 32\n",
      "samples#: 64\n",
      "samples#: 96\n",
      "samples#: 128\n",
      "samples#: 160\n",
      "samples#: 192\n",
      "samples#: 224\n",
      "samples#: 237\n",
      "EP: 2\n",
      "samples#: 32\n",
      "samples#: 64\n",
      "samples#: 96\n",
      "samples#: 128\n",
      "samples#: 160\n",
      "samples#: 192\n",
      "samples#: 224\n",
      "samples#: 237\n",
      "EP: 3\n",
      "samples#: 32\n",
      "samples#: 64\n",
      "samples#: 96\n",
      "samples#: 128\n",
      "samples#: 160\n",
      "samples#: 192\n",
      "samples#: 224\n",
      "samples#: 237\n",
      "EP: 4\n",
      "samples#: 32\n",
      "samples#: 64\n",
      "samples#: 96\n",
      "samples#: 128\n",
      "samples#: 160\n",
      "samples#: 192\n",
      "samples#: 224\n",
      "samples#: 237\n",
      "EP: 5\n",
      "samples#: 32\n",
      "samples#: 64\n",
      "samples#: 96\n",
      "samples#: 128\n",
      "samples#: 160\n",
      "samples#: 192\n",
      "samples#: 224\n",
      "samples#: 237\n",
      "EP: 6\n",
      "samples#: 32\n",
      "samples#: 64\n",
      "samples#: 96\n",
      "samples#: 128\n",
      "samples#: 160\n",
      "samples#: 192\n",
      "samples#: 224\n",
      "samples#: 237\n",
      "EP: 7\n",
      "samples#: 32\n",
      "samples#: 64\n",
      "samples#: 96\n",
      "samples#: 128\n",
      "samples#: 160\n",
      "samples#: 192\n",
      "samples#: 224\n",
      "samples#: 237\n",
      "EP: 8\n",
      "samples#: 32\n",
      "samples#: 64\n",
      "samples#: 96\n",
      "samples#: 128\n",
      "samples#: 160\n",
      "samples#: 192\n",
      "samples#: 224\n",
      "samples#: 237\n",
      "EP: 9\n",
      "samples#: 32\n",
      "samples#: 64\n",
      "samples#: 96\n",
      "samples#: 128\n",
      "samples#: 160\n",
      "samples#: 192\n",
      "samples#: 224\n",
      "samples#: 237\n",
      "EP: 10\n",
      "samples#: 32\n",
      "samples#: 64\n",
      "samples#: 96\n",
      "samples#: 128\n",
      "samples#: 160\n",
      "samples#: 192\n",
      "samples#: 224\n",
      "samples#: 237\n",
      "EP: 11\n",
      "samples#: 32\n",
      "samples#: 64\n",
      "samples#: 96\n",
      "samples#: 128\n",
      "samples#: 160\n",
      "samples#: 192\n",
      "samples#: 224\n",
      "samples#: 237\n",
      "EP: 12\n",
      "samples#: 32\n",
      "samples#: 64\n",
      "samples#: 96\n",
      "samples#: 128\n",
      "samples#: 160\n",
      "samples#: 192\n",
      "samples#: 224\n",
      "samples#: 237\n",
      "EP: 13\n",
      "samples#: 32\n",
      "samples#: 64\n",
      "samples#: 96\n",
      "samples#: 128\n",
      "samples#: 160\n",
      "samples#: 192\n",
      "samples#: 224\n",
      "samples#: 237\n",
      "EP: 14\n",
      "samples#: 32\n",
      "samples#: 64\n",
      "samples#: 96\n",
      "samples#: 128\n",
      "samples#: 160\n",
      "samples#: 192\n",
      "samples#: 224\n",
      "samples#: 237\n",
      "EP: 15\n",
      "samples#: 32\n",
      "samples#: 64\n",
      "samples#: 96\n",
      "samples#: 128\n",
      "samples#: 160\n",
      "samples#: 192\n",
      "samples#: 224\n",
      "samples#: 237\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "237"
      ]
     },
     "execution_count": 283,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recent_ep = -1\n",
    "for ep, o_e, f_e, p_l, t_l in zip(epoch_num,ori_emb,fil_emb,pred_lab2,fam_lab): #take put each epoch\n",
    "    if ep != recent_ep:\n",
    "        whole_dict_ori = {}\n",
    "        whole_dict_fil = {}\n",
    "#         print('EP:',ep)\n",
    "        recent_ep = ep\n",
    "        count =  0\n",
    "    for oe,fe,pl,tl in zip(o_e, f_e, p_l, t_l): #take out each process\n",
    "        count+=1\n",
    "        try:\n",
    "            temp1 = whole_dict_ori[tl] #put by true label\n",
    "            temp1.append(oe)\n",
    "            temp2 = whole_dict_fil[tl]\n",
    "            temp2.append(fe)\n",
    "            whole_dict_ori[tl] = temp1 #original embedding\n",
    "            whole_dict_fil[tl] = temp2 #filter embedding\n",
    "        except KeyError:\n",
    "            temp1 = [oe]\n",
    "            temp2 = [fe]\n",
    "            whole_dict_ori[tl] = temp1 \n",
    "            whole_dict_fil[tl] = temp2\n",
    "#     print('samples#:',count)\n",
    "#             if tl == 2:\n",
    "#                 print([list(oe),])\n",
    "#                 break            \n",
    "#         except AttributeError:\n",
    "#             temp1 = list([oe,])\n",
    "#             temp2 = list([fe,])\n",
    "#             whole_dict_ori[tl] = temp1 #tl==2 有問題@@\n",
    "#             whole_dict_fil[tl] = temp2#tl==2 有問題@@\n",
    "#             if tl == 2:\n",
    "#                 print([list(oe),])\n",
    "#                 break           \n",
    "# count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13, 182, 128)"
      ]
     },
     "execution_count": 306,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(whole_dict_fil[0][1:]).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.02495082,  1.        ,  1.        , ..., -0.30626938,\n",
       "        -0.30626938, -0.30626938],\n",
       "       [-0.15718652, -0.42860642, -0.42860642, ...,  0.20198658,\n",
       "         0.20198658,  0.20198658],\n",
       "       [ 0.13184482,  0.4822547 ,  0.4822547 , ...,  0.04879938,\n",
       "         0.04879938,  0.04879938],\n",
       "       ...,\n",
       "       [-0.10193527,  0.24838135,  0.24838135, ...,  0.10296886,\n",
       "         0.10296886,  0.10296886],\n",
       "       [-0.04273163,  0.62445796,  0.62445796, ..., -0.32532468,\n",
       "        -0.32532468, -0.32532468],\n",
       "       [ 0.06389179, -0.11722743, -0.11722743, ..., -0.07050402,\n",
       "        -0.07050402, -0.07050402]], dtype=float32)"
      ]
     },
     "execution_count": 308,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = np.expand_dims(whole_dict_fil[0][0],axis=0)\n",
    "cosine_similarity(test[0],np.array(whole_dict_fil[0][1:])[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{7: [array([[-0.05921727, -0.22285984, -0.28284442, ...,  0.16880853,\n",
       "           0.02067087,  0.13023372],\n",
       "         [-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         [ 0.03967505,  0.11265814,  0.13778308, ..., -0.16289778,\n",
       "           0.05585124,  0.5513557 ],\n",
       "         ...,\n",
       "         [ 0.25461718, -0.28900287,  0.17536671, ..., -0.01272231,\n",
       "          -0.0379485 ,  0.0289838 ],\n",
       "         [ 0.08445907,  0.2207433 , -0.27071974, ..., -0.02107348,\n",
       "           0.17360754, -0.25596514],\n",
       "         [-0.00855983, -0.48185816, -0.20542417, ...,  0.1833916 ,\n",
       "           0.11007628, -0.4651002 ]], dtype=float32),\n",
       "  array([[-0.05921727, -0.22285984, -0.28284442, ...,  0.16880853,\n",
       "           0.02067087,  0.13023372],\n",
       "         [-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         [ 0.03967505,  0.11265814,  0.13778308, ..., -0.16289778,\n",
       "           0.05585124,  0.5513557 ],\n",
       "         ...,\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903]], dtype=float32),\n",
       "  array([[-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         [ 0.1538219 ,  0.08651822, -0.05850999, ...,  0.17292847,\n",
       "           0.0122154 , -0.16761358],\n",
       "         [ 0.10163565,  0.11445903, -0.09826617, ...,  0.22165859,\n",
       "          -0.40742022,  0.02286944],\n",
       "         ...,\n",
       "         [ 0.18049093,  0.04728775,  0.04551627, ..., -0.31711292,\n",
       "          -0.15048902,  0.13286895],\n",
       "         [ 0.18049093,  0.04728775,  0.04551627, ..., -0.31711292,\n",
       "          -0.15048902,  0.13286895],\n",
       "         [ 0.18049093,  0.04728775,  0.04551627, ..., -0.31711292,\n",
       "          -0.15048902,  0.13286895]], dtype=float32)],\n",
       " 4: [array([[-0.05921727, -0.22285984, -0.28284442, ...,  0.16880853,\n",
       "           0.02067087,  0.13023372],\n",
       "         [-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         [-0.1027265 , -0.27635664, -0.2477804 , ...,  0.17568733,\n",
       "          -0.4284471 ,  0.08035108],\n",
       "         ...,\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903]], dtype=float32)],\n",
       " 2: [array([[-0.05921727, -0.22285984, -0.28284442, ...,  0.16880853,\n",
       "           0.02067087,  0.13023372],\n",
       "         [-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         [ 0.03967505,  0.11265814,  0.13778308, ..., -0.16289778,\n",
       "           0.05585124,  0.5513557 ],\n",
       "         ...,\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903]], dtype=float32),\n",
       "  array([[-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         [-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         [-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         ...,\n",
       "         [ 0.1480549 ,  0.00072115, -0.14540538, ...,  0.39226478,\n",
       "          -0.012312  ,  0.022385  ],\n",
       "         [ 0.18049093,  0.04728775,  0.04551627, ..., -0.31711292,\n",
       "          -0.15048902,  0.13286895],\n",
       "         [ 0.18049093,  0.04728775,  0.04551627, ..., -0.31711292,\n",
       "          -0.15048902,  0.13286895]], dtype=float32)],\n",
       " 11: [array([[-0.05921727, -0.22285984, -0.28284442, ...,  0.16880853,\n",
       "           0.02067087,  0.13023372],\n",
       "         [-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         [ 0.06986269,  0.21129695,  0.04954508, ...,  0.01387486,\n",
       "          -0.11838187, -0.14526248],\n",
       "         ...,\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903]], dtype=float32)],\n",
       " 1: [array([[-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         [-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         [-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         ...,\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903]], dtype=float32)],\n",
       " 9: [array([[-0.05921727, -0.22285984, -0.28284442, ...,  0.16880853,\n",
       "           0.02067087,  0.13023372],\n",
       "         [-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         [ 0.03967505,  0.11265814,  0.13778308, ..., -0.16289778,\n",
       "           0.05585124,  0.5513557 ],\n",
       "         ...,\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903]], dtype=float32),\n",
       "  array([[-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         [ 0.1538219 ,  0.08651822, -0.05850999, ...,  0.17292847,\n",
       "           0.0122154 , -0.16761358],\n",
       "         [ 0.10163565,  0.11445903, -0.09826617, ...,  0.22165859,\n",
       "          -0.40742022,  0.02286944],\n",
       "         ...,\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903]], dtype=float32),\n",
       "  array([[-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         [-0.1027265 , -0.27635664, -0.2477804 , ...,  0.17568733,\n",
       "          -0.4284471 ,  0.08035108],\n",
       "         [-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         ...,\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903]], dtype=float32)],\n",
       " 0: [array([[-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         [-0.1027265 , -0.27635664, -0.2477804 , ...,  0.17568733,\n",
       "          -0.4284471 ,  0.08035108],\n",
       "         [-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         ...,\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903],\n",
       "         [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "          -0.05969607,  0.01120903]], dtype=float32)],\n",
       " 3: [array([[ 0.1538219 ,  0.08651822, -0.05850999, ...,  0.17292847,\n",
       "           0.0122154 , -0.16761358],\n",
       "         [-0.05921727, -0.22285984, -0.28284442, ...,  0.16880853,\n",
       "           0.02067087,  0.13023372],\n",
       "         [-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "          -0.22155283,  0.06595514],\n",
       "         ...,\n",
       "         [ 0.17120706, -0.07097782, -0.05042176, ...,  0.1894411 ,\n",
       "           0.01535186, -0.2281921 ],\n",
       "         [-0.16201018, -0.07334327,  0.01504868, ..., -0.07045063,\n",
       "          -0.6137164 ,  0.09685928],\n",
       "         [ 0.18049093,  0.04728775,  0.04551627, ..., -0.31711292,\n",
       "          -0.15048902,  0.13286895]], dtype=float32)]}"
      ]
     },
     "execution_count": 258,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "whole_dict_ori"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "         -0.22155283,  0.06595514],\n",
       "        [ 0.1538219 ,  0.08651822, -0.05850999, ...,  0.17292847,\n",
       "          0.0122154 , -0.16761358],\n",
       "        [ 0.10163565,  0.11445903, -0.09826617, ...,  0.22165859,\n",
       "         -0.40742022,  0.02286944],\n",
       "        ...,\n",
       "        [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "         -0.05969607,  0.01120903],\n",
       "        [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "         -0.05969607,  0.01120903],\n",
       "        [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "         -0.05969607,  0.01120903]], dtype=float32),\n",
       " array([[-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "         -0.22155283,  0.06595514],\n",
       "        [ 0.1538219 ,  0.08651822, -0.05850999, ...,  0.17292847,\n",
       "          0.0122154 , -0.16761358],\n",
       "        [ 0.10163565,  0.11445903, -0.09826617, ...,  0.22165859,\n",
       "         -0.40742022,  0.02286944],\n",
       "        ...,\n",
       "        [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "         -0.05969607,  0.01120903],\n",
       "        [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "         -0.05969607,  0.01120903],\n",
       "        [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "         -0.05969607,  0.01120903]], dtype=float32)]"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "whole_dict_ori[tl] = [fe,]\n",
    "whole_dict_ori[tl].append(fe)\n",
    "whole_dict_ori[tl]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "         -0.22155283,  0.06595514],\n",
       "        [ 0.1538219 ,  0.08651822, -0.05850999, ...,  0.17292847,\n",
       "          0.0122154 , -0.16761358],\n",
       "        [ 0.10163565,  0.11445903, -0.09826617, ...,  0.22165859,\n",
       "         -0.40742022,  0.02286944],\n",
       "        ...,\n",
       "        [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "         -0.05969607,  0.01120903],\n",
       "        [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "         -0.05969607,  0.01120903],\n",
       "        [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "         -0.05969607,  0.01120903]], dtype=float32),\n",
       " array([[-0.04800555, -0.06131621, -0.07549804, ..., -0.10737687,\n",
       "         -0.22155283,  0.06595514],\n",
       "        [ 0.1538219 ,  0.08651822, -0.05850999, ...,  0.17292847,\n",
       "          0.0122154 , -0.16761358],\n",
       "        [ 0.10163565,  0.11445903, -0.09826617, ...,  0.22165859,\n",
       "         -0.40742022,  0.02286944],\n",
       "        ...,\n",
       "        [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "         -0.05969607,  0.01120903],\n",
       "        [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "         -0.05969607,  0.01120903],\n",
       "        [-0.01391036,  0.05648555,  0.05279909, ..., -0.00105697,\n",
       "         -0.05969607,  0.01120903]], dtype=float32)]"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[fe,fe]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "128"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pred_lab2 = list(np.argmax(pred_lab,axis=-1))\n",
    "len(epoch_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ori_emb[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ori_emb \n",
    "fil_emb \n",
    "epoch_num \n",
    "fam_lab \n",
    "pred_lab2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Store process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./results/Hooklog/2019122601/losses_metrics.xlsx\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train total loss</th>\n",
       "      <th>train acc loss</th>\n",
       "      <th>train filter loss</th>\n",
       "      <th>train seq loss</th>\n",
       "      <th>train acc rate</th>\n",
       "      <th>train ones num</th>\n",
       "      <th>test total loss</th>\n",
       "      <th>test acc loss</th>\n",
       "      <th>test filter loss</th>\n",
       "      <th>test seq loss</th>\n",
       "      <th>test acc rate</th>\n",
       "      <th>test ones num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [train total loss, train acc loss, train filter loss, train seq loss, train acc rate, train ones num, test total loss, test acc loss, test filter loss, test seq loss, test acc rate, test ones num]\n",
       "Index: []"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "save_dir = './results/Hooklog/'+DateID+'/'\n",
    "if not os.path.isdir(save_dir):\n",
    "    os.makedirs(save_dir, exist_ok=True)\n",
    "saveR = save_dir + 'losses_metrics.xlsx'\n",
    "data = {'train total loss':train_weighted_loss, 'train acc loss':train_loss_acc,\n",
    "        'train filter loss':train_loss_filter,'train seq loss':train_loss_seq,\n",
    "        'train acc rate':train_acc_rate, 'train ones num':train_ones_num,\n",
    "        'test total loss':test_weighted_loss, 'test acc loss':test_loss_acc,\n",
    "        'test filter loss':test_loss_filter, 'test seq loss': test_loss_seq,\n",
    "        'test acc rate':test_acc_rate, 'test ones num':test_ones_num\n",
    "       }\n",
    "df = pd.DataFrame(data)\n",
    "df.to_excel(saveR)\n",
    "print(saveR)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                            ori_word\n",
      "ID                                                  \n",
      "1               (DeleteFile, DeleteFile, DeleteFile)\n",
      "2                     (CopyFile, CopyFile, CopyFile)\n",
      "3               (RegOpenKey, RegOpenKey, RegOpenKey)\n",
      "4            (CloseHandle, CloseHandle, CloseHandle)\n",
      "5           (RegOpenKey, RegQueryValue, RegCloseKey)\n",
      "...                                              ...\n",
      "1373          (CloseHandle, RegSetValue, CreateFile)\n",
      "1374     (HttpSendRequest, CloseHandle, LoadLibrary)\n",
      "1375    (LoadLibrary, InternetOpen, InternetConnect)\n",
      "1376       (RegQueryValue, LoadLibrary, CloseHandle)\n",
      "1377  (LoadLibrary, CreateRemoteThread, CloseHandle)\n",
      "\n",
      "[1377 rows x 1 columns]\n"
     ]
    }
   ],
   "source": [
    "labelID_dict = pickle.load(open(\"./data/Hooklog/trace_picked_fam/fam_dict_enc.pkl\",\"rb\"))\n",
    "IDlabel_dict = {v: k for k, v in labelID_dict.items()}\n",
    "word_index = pickle.load(open(\"./data/Hooklog/trace_picked_fam/tri_dict_enc.pkl\",\"rb\"))\n",
    "word_idx_pd = pd.DataFrame(data=word_index,index=[\"ID\"]).T\n",
    "word_idx_pd['ori_word'] = word_idx_pd.index \n",
    "word_idx_pd = word_idx_pd.set_index([\"ID\"])\n",
    "print(word_idx_pd)\n",
    "\n",
    "pred_label = list(np.argmax(pred_label,axis=-1))\n",
    "def getKeysByValues(listOfValues,dfOfElements=word_idx_pd):\n",
    "    ori_word_li = []\n",
    "    for item in listOfValues:\n",
    "        key = item\n",
    "#         print(key)\n",
    "        if key == 0:\n",
    "            continue\n",
    "        ori_word = dfOfElements.loc[key][0]\n",
    "        ori_word_li.append(ori_word)\n",
    "    return ori_word_li"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/2000 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "1383",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2896\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2897\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2898\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.Int64HashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.Int64HashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 1383",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-155-bcc0e65f2c67>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mtemp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m#t = one sample\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m         \u001b[0mtemp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgetKeysByValues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     \u001b[0mword_ori_text\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtext\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mint_fil_text\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-154-3e7ee4253430>\u001b[0m in \u001b[0;36mgetKeysByValues\u001b[0;34m(listOfValues, dfOfElements)\u001b[0m\n\u001b[1;32m     15\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m             \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m         \u001b[0mori_word\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdfOfElements\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m         \u001b[0mori_word_li\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mori_word\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mori_word_li\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   1422\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1423\u001b[0m             \u001b[0mmaybe_callable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_if_callable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1424\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_axis\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaybe_callable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1425\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1426\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_is_scalar_access\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_getitem_axis\u001b[0;34m(self, key, axis)\u001b[0m\n\u001b[1;32m   1848\u001b[0m         \u001b[0;31m# fall thru to straight lookup\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1849\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_key\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1850\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_label\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1851\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1852\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/pandas/core/indexing.py\u001b[0m in \u001b[0;36m_get_label\u001b[0;34m(self, label, axis)\u001b[0m\n\u001b[1;32m    158\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mIndexingError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"no slices here, handle elsewhere\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 160\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_xs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    161\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_get_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36mxs\u001b[0;34m(self, key, axis, level, drop_level)\u001b[0m\n\u001b[1;32m   3735\u001b[0m             \u001b[0mloc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc_level\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdrop_level\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdrop_level\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3736\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3737\u001b[0;31m             \u001b[0mloc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3738\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3739\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2897\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2898\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2899\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2900\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2901\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.Int64HashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.Int64HashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 1383"
     ]
    }
   ],
   "source": [
    "word_fil_text = []\n",
    "word_ori_text = []\n",
    "from tqdm import tqdm\n",
    "for text in tqdm(int_ori_text): #text = one epoch\n",
    "    temp = []\n",
    "    for t in text: #t = one sample\n",
    "        temp.append(getKeysByValues(list(t)))\n",
    "    word_ori_text.append(temp)\n",
    "for text in tqdm(int_fil_text):\n",
    "    temp = []\n",
    "    for t in text:\n",
    "        temp.append(getKeysByValues(list(t)))\n",
    "    word_fil_text.append(temp)\n",
    "\n",
    "save_path = save_dir + \"original_words.pkl\"\n",
    "pickle.dump(file=open(save_path,'wb'),obj=word_ori_text)\n",
    "save_path = save_dir + \"filterUsed_words.pkl\"\n",
    "pickle.dump(file=open(save_path,'wb'),obj=word_fil_text)\n",
    "print(save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2000it [00:11, 174.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).emb.embeddings\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).rnn1.state_spec\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).bn1.axis\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).bn1.gamma\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).bn1.beta\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).bn1.moving_mean\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).bn1.moving_variance\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).rnn1.cell.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).rnn1.cell.recurrent_kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).rnn1.cell.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).fil.layer.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).fil.layer.bias\n",
      "WARNING:tensorflow:A checkpoint was restored (e.g. tf.train.Checkpoint.restore or tf.keras.Model.load_weights) but not all checkpointed values were used. See above for specific issues. Use expect_partial() on the load status object, e.g. tf.train.Checkpoint.restore(...).expect_partial(), to silence these warnings, or use assert_consumed() to make the check explicit. See https://www.tensorflow.org/guide/checkpoint#loading_mechanics for details.\n",
      "./results/Hooklog/2019122301/caseStudy_result.xlsx\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch#</th>\n",
       "      <th>text_id</th>\n",
       "      <th>ori_text</th>\n",
       "      <th>filter_text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>('RegOpenKey', 'LoadLibrary', 'LoadLibrary') (...</td>\n",
       "      <td>('RegOpenKey', 'LoadLibrary', 'LoadLibrary') (...</td>\n",
       "      <td>domaiq</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...</td>\n",
       "      <td>('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...</td>\n",
       "      <td>installcore</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...</td>\n",
       "      <td>('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...</td>\n",
       "      <td>browsefox</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>('RegOpenKey', 'LoadLibrary', 'LoadLibrary') (...</td>\n",
       "      <td>('RegOpenKey', 'LoadLibrary', 'LoadLibrary') (...</td>\n",
       "      <td>domaiq</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...</td>\n",
       "      <td>('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...</td>\n",
       "      <td>installcore</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5995</th>\n",
       "      <td>1998</td>\n",
       "      <td>1</td>\n",
       "      <td>('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...</td>\n",
       "      <td>('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...</td>\n",
       "      <td>installcore</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5996</th>\n",
       "      <td>1998</td>\n",
       "      <td>2</td>\n",
       "      <td>('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...</td>\n",
       "      <td>('LoadLibrary', 'LoadLibrary', 'RegOpenKey') (...</td>\n",
       "      <td>browsefox</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5997</th>\n",
       "      <td>1999</td>\n",
       "      <td>0</td>\n",
       "      <td>('RegOpenKey', 'LoadLibrary', 'LoadLibrary') (...</td>\n",
       "      <td>('RegOpenKey', 'LoadLibrary', 'LoadLibrary') (...</td>\n",
       "      <td>domaiq</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5998</th>\n",
       "      <td>1999</td>\n",
       "      <td>1</td>\n",
       "      <td>('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...</td>\n",
       "      <td>('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...</td>\n",
       "      <td>installcore</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5999</th>\n",
       "      <td>1999</td>\n",
       "      <td>2</td>\n",
       "      <td>('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...</td>\n",
       "      <td>('LoadLibrary', 'LoadLibrary', 'RegOpenKey') (...</td>\n",
       "      <td>browsefox</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6000 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     epoch# text_id                                           ori_text  \\\n",
       "0         0       0  ('RegOpenKey', 'LoadLibrary', 'LoadLibrary') (...   \n",
       "1         0       1  ('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...   \n",
       "2         0       2  ('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...   \n",
       "3         1       0  ('RegOpenKey', 'LoadLibrary', 'LoadLibrary') (...   \n",
       "4         1       1  ('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...   \n",
       "...     ...     ...                                                ...   \n",
       "5995   1998       1  ('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...   \n",
       "5996   1998       2  ('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...   \n",
       "5997   1999       0  ('RegOpenKey', 'LoadLibrary', 'LoadLibrary') (...   \n",
       "5998   1999       1  ('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...   \n",
       "5999   1999       2  ('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...   \n",
       "\n",
       "                                            filter_text        label  \n",
       "0     ('RegOpenKey', 'LoadLibrary', 'LoadLibrary') (...       domaiq  \n",
       "1     ('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...  installcore  \n",
       "2     ('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...    browsefox  \n",
       "3     ('RegOpenKey', 'LoadLibrary', 'LoadLibrary') (...       domaiq  \n",
       "4     ('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...  installcore  \n",
       "...                                                 ...          ...  \n",
       "5995  ('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...  installcore  \n",
       "5996  ('LoadLibrary', 'LoadLibrary', 'RegOpenKey') (...    browsefox  \n",
       "5997  ('RegOpenKey', 'LoadLibrary', 'LoadLibrary') (...       domaiq  \n",
       "5998  ('LoadLibrary', 'LoadLibrary', 'LoadLibrary') ...  installcore  \n",
       "5999  ('LoadLibrary', 'LoadLibrary', 'RegOpenKey') (...    browsefox  \n",
       "\n",
       "[6000 rows x 5 columns]"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col = ['epoch#','text_id','ori_text','filter_text','label','predict']\n",
    "df = pd.DataFrame(columns=col)\n",
    "for epoch,(text_ori,text_fil,labelID,predID) in tqdm(enumerate(zip(word_ori_text,word_fil_text,int_label,pred_label))):\n",
    "    for idx,(t_o,t_f,label,pred) in enumerate(zip(text_ori,text_fil,labelID,predID)):\n",
    "        temp = pd.Series([epoch,idx,\" \".join([str(x) for x in t_o]),\" \".join([str(x) for x in t_f]),IDlabel_dict[label],IDlabel_dict[pred]],col)\n",
    "        df = df.append(temp,ignore_index=True)\n",
    "\n",
    "save_path = save_dir + \"caseStudy_result.xlsx\"\n",
    "df.to_excel(save_path,index=False)\n",
    "print(save_path)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:root] *",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}