CNN2Head_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import CNN2Head_input\n",
    "import os\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import BKNetStyle\n",
    "from const import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load smile image...................\n",
      "Done !\n",
      "Number of smile train data:  3000\n",
      "---------------------------------------------------------------\n",
      "Load gender image...................\n",
      "Done !\n",
      "Number of gender train data:  150000\n",
      "---------------------------------------------------------------\n",
      "Load age image...................\n",
      "Done !\n",
      "Number of age train data:  150000\n",
      "---------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "''' PREPARE DATA '''\n",
    "''' PREPARE DATA '''\n",
    "smile_train, smile_test = CNN2Head_input.getSmileImage()\n",
    "gender_train, gender_test = CNN2Head_input.getGenderImage()\n",
    "age_train, age_test = CNN2Head_input.getAgeImage()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot(index, num_classes):\n",
    "    assert index < num_classes and index >= 0\n",
    "    tmp = np.zeros(num_classes, dtype=np.float32)\n",
    "    tmp[index] = 1.0\n",
    "    return tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-4-f5c3e05590f1>:2: get_or_create_global_step (from tensorflow.contrib.framework.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please switch to tf.train.get_or_create_global_step\n"
     ]
    }
   ],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "global_step = tf.contrib.framework.get_or_create_global_step()\n",
    "\n",
    "x, y_, mask = BKNetStyle.Input()\n",
    "\n",
    "y_smile_conv, y_gender_conv, y_age_conv, phase_train, keep_prob = BKNetStyle.BKNetModel(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restoring existed model\n",
      "INFO:tensorflow:Restoring parameters from ./save/current/model.ckpt\n",
      "OK\n",
      "Epoch: 40\n",
      "Learning rate: 0.000059\n",
      "Smile task train accuracy: 99.9799979998\n",
      "Gender task train accuracy: 84.0072536718\n",
      "Age task train accuracy: 51.7660706428\n",
      "Total loss: 1.54786. L2-loss: 0.124232\n",
      "Smile loss: 0.00505291\n",
      "Gender loss: 0.351021\n",
      "Age loss: 1.06756\n",
      "\n",
      "\n",
      "Epoch: 41\n",
      "Learning rate: 0.000051\n",
      "Smile task train accuracy: 99.99\n",
      "Gender task train accuracy: 84.0520287478\n",
      "Age task train accuracy: 51.8654576972\n",
      "Total loss: 1.54084. L2-loss: 0.122288\n",
      "Smile loss: 0.00416632\n",
      "Gender loss: 0.3498\n",
      "Age loss: 1.06459\n",
      "\n",
      "\n",
      "Epoch: 42\n",
      "Learning rate: 0.000046\n",
      "Smile task train accuracy: 99.9899996667\n",
      "Gender task train accuracy: 84.0444823724\n",
      "Age task train accuracy: 52.0730967445\n",
      "Total loss: 1.53514. L2-loss: 0.12057\n",
      "Smile loss: 0.00357896\n",
      "Gender loss: 0.349225\n",
      "Age loss: 1.06177\n",
      "\n",
      "\n",
      "Epoch: 43\n",
      "Learning rate: 0.000039\n",
      "Smile task train accuracy: 99.9899993333\n",
      "Gender task train accuracy: 84.0821660255\n",
      "Age task train accuracy: 52.3423801809\n",
      "Total loss: 1.53053. L2-loss: 0.11908\n",
      "Smile loss: 0.0040685\n",
      "Gender loss: 0.348638\n",
      "Age loss: 1.05875\n",
      "\n",
      "\n",
      "Epoch: 44\n",
      "Learning rate: 0.000035\n",
      "Smile task train accuracy: 99.9933328889\n",
      "Gender task train accuracy: 84.2112631923\n",
      "Age task train accuracy: 52.334442274\n",
      "Total loss: 1.52374. L2-loss: 0.11774\n",
      "Smile loss: 0.00298509\n",
      "Gender loss: 0.346964\n",
      "Age loss: 1.05605\n",
      "\n",
      "\n",
      "Epoch: 45\n",
      "Learning rate: 0.000030\n",
      "Smile task train accuracy: 99.9833333333\n",
      "Gender task train accuracy: 84.1518214305\n",
      "Age task train accuracy: 52.3807936423\n",
      "Total loss: 1.52096. L2-loss: 0.116543\n",
      "Smile loss: 0.00266229\n",
      "Gender loss: 0.346797\n",
      "Age loss: 1.05495\n",
      "\n",
      "\n",
      "Epoch: 46\n",
      "Learning rate: 0.000026\n",
      "Smile task train accuracy: 99.9966663333\n",
      "Gender task train accuracy: 84.1081369379\n",
      "Age task train accuracy: 52.5148007894\n",
      "Total loss: 1.51739. L2-loss: 0.115502\n",
      "Smile loss: 0.00230276\n",
      "Gender loss: 0.345529\n",
      "Age loss: 1.05406\n",
      "\n",
      "\n",
      "Epoch: 47\n",
      "Learning rate: 0.000024\n",
      "Smile task train accuracy: 99.9799993333\n",
      "Gender task train accuracy: 84.1757172097\n",
      "Age task train accuracy: 52.5667693374\n",
      "Total loss: 1.51338. L2-loss: 0.114574\n",
      "Smile loss: 0.00269148\n",
      "Gender loss: 0.344677\n",
      "Age loss: 1.05144\n",
      "\n",
      "\n",
      "Epoch: 48\n",
      "Learning rate: 0.000020\n",
      "Smile task train accuracy: 100.0\n",
      "Gender task train accuracy: 84.2565986413\n",
      "Age task train accuracy: 52.5534695184\n",
      "Total loss: 1.51033. L2-loss: 0.113773\n",
      "Smile loss: 0.00197225\n",
      "Gender loss: 0.344232\n",
      "Age loss: 1.05035\n",
      "\n",
      "\n",
      "Epoch: 49\n",
      "Learning rate: 0.000018\n",
      "Smile task train accuracy: 99.9899986665\n",
      "Gender task train accuracy: 84.1707001613\n",
      "Age task train accuracy: 52.7461098444\n",
      "Total loss: 1.50777. L2-loss: 0.113059\n",
      "Smile loss: 0.00201439\n",
      "Gender loss: 0.344112\n",
      "Age loss: 1.04858\n",
      "\n",
      "\n",
      "Epoch: 50\n",
      "Learning rate: 0.000016\n",
      "Smile task train accuracy: 99.9833322221\n",
      "Gender task train accuracy: 84.2567036015\n",
      "Age task train accuracy: 52.7361459278\n",
      "Total loss: 1.50429. L2-loss: 0.112438\n",
      "Smile loss: 0.00219212\n",
      "Gender loss: 0.342759\n",
      "Age loss: 1.04691\n",
      "\n",
      "\n",
      "Epoch: 51\n",
      "Learning rate: 0.000014\n",
      "Smile task train accuracy: 99.9733324444\n",
      "Gender task train accuracy: 84.2198250573\n",
      "Age task train accuracy: 52.7644623416\n",
      "Total loss: 1.50145. L2-loss: 0.111907\n",
      "Smile loss: 0.00229195\n",
      "Gender loss: 0.342365\n",
      "Age loss: 1.04489\n",
      "\n",
      "\n",
      "Epoch: 52\n",
      "Learning rate: 0.000012\n",
      "Smile task train accuracy: 99.9899996667\n",
      "Gender task train accuracy: 84.2823902767\n",
      "Age task train accuracy: 52.9821192848\n",
      "Total loss: 1.50004. L2-loss: 0.111437\n",
      "Smile loss: 0.00182806\n",
      "Gender loss: 0.34291\n",
      "Age loss: 1.04386\n",
      "\n",
      "\n",
      "Epoch: 53\n",
      "Learning rate: 0.000010\n",
      "Smile task train accuracy: 99.976665111\n",
      "Gender task train accuracy: 84.1905955611\n",
      "Age task train accuracy: 52.905135573\n",
      "Total loss: 1.49906. L2-loss: 0.111018\n",
      "Smile loss: 0.00174026\n",
      "Gender loss: 0.342777\n",
      "Age loss: 1.04352\n",
      "\n",
      "\n",
      "Epoch: 54\n",
      "Learning rate: 0.000009\n",
      "Smile task train accuracy: 99.9799986666\n",
      "Gender task train accuracy: 84.2575802021\n",
      "Age task train accuracy: 53.0202013468\n",
      "Total loss: 1.49626. L2-loss: 0.110649\n",
      "Smile loss: 0.00192738\n",
      "Gender loss: 0.34154\n",
      "Age loss: 1.04215\n",
      "\n",
      "\n",
      "Epoch: 55\n",
      "Learning rate: 0.000008\n",
      "Smile task train accuracy: 99.9933333333\n",
      "Gender task train accuracy: 84.2623890767\n",
      "Age task train accuracy: 53.0058069377\n",
      "Total loss: 1.4952. L2-loss: 0.110324\n",
      "Smile loss: 0.0013705\n",
      "Gender loss: 0.341287\n",
      "Age loss: 1.04222\n",
      "\n",
      "\n",
      "Epoch: 56\n",
      "Learning rate: 0.000007\n",
      "Smile task train accuracy: 99.9866662222\n",
      "Gender task train accuracy: 84.296600508\n",
      "Age task train accuracy: 53.0221611819\n",
      "Total loss: 1.49412. L2-loss: 0.110044\n",
      "Smile loss: 0.00172862\n",
      "Gender loss: 0.341192\n",
      "Age loss: 1.04115\n",
      "\n",
      "\n",
      "Epoch: 57\n",
      "Learning rate: 0.000006\n",
      "Smile task train accuracy: 99.9833322221\n",
      "Gender task train accuracy: 84.2759328769\n",
      "Age task train accuracy: 53.05914276\n",
      "Total loss: 1.49237. L2-loss: 0.109797\n",
      "Smile loss: 0.00167086\n",
      "Gender loss: 0.340736\n",
      "Age loss: 1.04017\n",
      "\n",
      "\n",
      "Epoch: 58\n",
      "Learning rate: 0.000006\n",
      "Smile task train accuracy: 99.9966665556\n",
      "Gender task train accuracy: 84.3311643288\n",
      "Age task train accuracy: 53.1671478002\n",
      "Total loss: 1.4906. L2-loss: 0.109584\n",
      "Smile loss: 0.00157649\n",
      "Gender loss: 0.340271\n",
      "Age loss: 1.03917\n",
      "\n",
      "\n",
      "Epoch: 59\n",
      "Learning rate: 0.000005\n",
      "Smile task train accuracy: 99.97\n",
      "Gender task train accuracy: 84.2562837523\n",
      "Age task train accuracy: 53.114124565\n",
      "Total loss: 1.49146. L2-loss: 0.109394\n",
      "Smile loss: 0.00196512\n",
      "Gender loss: 0.340542\n",
      "Age loss: 1.03955\n",
      "\n",
      "\n",
      "Epoch: 60\n",
      "Learning rate: 0.000004\n",
      "Smile task train accuracy: 99.9799993333\n",
      "Gender task train accuracy: 84.2943909968\n",
      "Age task train accuracy: 53.0974572316\n",
      "Total loss: 1.49082. L2-loss: 0.109225\n",
      "Smile loss: 0.0018533\n",
      "Gender loss: 0.339818\n",
      "Age loss: 1.03993\n",
      "\n",
      "\n",
      "Epoch: 61\n",
      "Learning rate: 0.000004\n",
      "Smile task train accuracy: 99.9966664444\n",
      "Gender task train accuracy: 84.3640412283\n",
      "Age task train accuracy: 53.1395007734\n",
      "Total loss: 1.48834. L2-loss: 0.109081\n",
      "Smile loss: 0.00138639\n",
      "Gender loss: 0.339601\n",
      "Age loss: 1.03827\n",
      "\n",
      "\n",
      "Epoch: 62\n",
      "Learning rate: 0.000003\n",
      "Smile task train accuracy: 99.9799986666\n",
      "Gender task train accuracy: 84.3684996533\n",
      "Age task train accuracy: 53.1067909383\n",
      "Total loss: 1.48732. L2-loss: 0.108956\n",
      "Smile loss: 0.00174585\n",
      "Gender loss: 0.339175\n",
      "Age loss: 1.03744\n",
      "\n",
      "\n",
      "Epoch: 63\n",
      "Learning rate: 0.000003\n",
      "Smile task train accuracy: 99.9766666667\n",
      "Gender task train accuracy: 84.3448114937\n",
      "Age task train accuracy: 53.1518978058\n",
      "Total loss: 1.48733. L2-loss: 0.108845\n",
      "Smile loss: 0.00179635\n",
      "Gender loss: 0.33984\n",
      "Age loss: 1.03685\n",
      "\n",
      "\n",
      "Epoch: 64\n",
      "Learning rate: 0.000002\n",
      "Smile task train accuracy: 99.9799993333\n",
      "Gender task train accuracy: 84.2775807355\n",
      "Age task train accuracy: 53.1692324104\n",
      "Total loss: 1.48656. L2-loss: 0.108749\n",
      "Smile loss: 0.0019027\n",
      "Gender loss: 0.339373\n",
      "Age loss: 1.03653\n",
      "\n",
      "\n",
      "Epoch: 65\n",
      "Learning rate: 0.000002\n",
      "Smile task train accuracy: 99.9599986666\n",
      "Gender task train accuracy: 84.29618172\n",
      "Age task train accuracy: 53.2154190778\n",
      "Total loss: 1.48591. L2-loss: 0.108662\n",
      "Smile loss: 0.00191531\n",
      "Gender loss: 0.339123\n",
      "Age loss: 1.03621\n",
      "\n",
      "\n",
      "Epoch: 66\n",
      "Learning rate: 0.000002\n",
      "Smile task train accuracy: 99.98\n",
      "Gender task train accuracy: 84.3624993333\n",
      "Age task train accuracy: 53.2281721692\n",
      "Total loss: 1.4855. L2-loss: 0.108587\n",
      "Smile loss: 0.00165468\n",
      "Gender loss: 0.339331\n",
      "Age loss: 1.03593\n",
      "\n",
      "\n",
      "Epoch: 67\n",
      "Learning rate: 0.000002\n",
      "Smile task train accuracy: 99.9833333333\n",
      "Gender task train accuracy: 84.3658328444\n",
      "Age task train accuracy: 53.2135047203\n",
      "Total loss: 1.48282. L2-loss: 0.108522\n",
      "Smile loss: 0.00188606\n",
      "Gender loss: 0.337908\n",
      "Age loss: 1.03451\n",
      "\n",
      "\n",
      "Epoch: 68\n",
      "Learning rate: 0.000001\n",
      "Smile task train accuracy: 99.9699969997\n",
      "Gender task train accuracy: 84.2864952797\n",
      "Age task train accuracy: 53.1124370812\n",
      "Total loss: 1.48528. L2-loss: 0.108465\n",
      "Smile loss: 0.00193775\n",
      "Gender loss: 0.339438\n",
      "Age loss: 1.03544\n",
      "\n",
      "\n",
      "Epoch: 69\n",
      "Learning rate: 0.000001\n",
      "Smile task train accuracy: 99.9833333333\n",
      "Gender task train accuracy: 84.3416227749\n",
      "Age task train accuracy: 53.1534594717\n",
      "Total loss: 1.48497. L2-loss: 0.108413\n",
      "Smile loss: 0.00144518\n",
      "Gender loss: 0.338372\n",
      "Age loss: 1.03674\n",
      "\n",
      "\n",
      "Epoch: 70\n",
      "Learning rate: 0.000001\n",
      "Smile task train accuracy: 99.9799993333\n",
      "Gender task train accuracy: 84.3560409083\n",
      "Age task train accuracy: 53.1005193645\n",
      "Total loss: 1.48518. L2-loss: 0.108369\n",
      "Smile loss: 0.00155279\n",
      "Gender loss: 0.338682\n",
      "Age loss: 1.03657\n",
      "\n",
      "\n",
      "Epoch: 71\n",
      "Learning rate: 0.000001\n",
      "Smile task train accuracy: 99.9833327778\n",
      "Gender task train accuracy: 84.3416868337\n",
      "Age task train accuracy: 53.3222657813\n",
      "Total loss: 1.48415. L2-loss: 0.10833\n",
      "Smile loss: 0.00143813\n",
      "Gender loss: 0.339281\n",
      "Age loss: 1.0351\n",
      "\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 72\n",
      "Learning rate: 0.000001\n",
      "Smile task train accuracy: 99.9799993333\n",
      "Gender task train accuracy: 84.3209547303\n",
      "Age task train accuracy: 53.3751125038\n",
      "Total loss: 1.48401. L2-loss: 0.108296\n",
      "Smile loss: 0.00162613\n",
      "Gender loss: 0.338449\n",
      "Age loss: 1.03563\n",
      "\n",
      "\n",
      "Epoch: 73\n",
      "Learning rate: 0.000001\n",
      "Smile task train accuracy: 99.9833305551\n",
      "Gender task train accuracy: 84.3227062416\n",
      "Age task train accuracy: 53.2564418814\n",
      "Total loss: 1.48493. L2-loss: 0.108265\n",
      "Smile loss: 0.00225842\n",
      "Gender loss: 0.338801\n",
      "Age loss: 1.0356\n",
      "\n",
      "\n",
      "Epoch: 74\n",
      "Learning rate: 0.000001\n",
      "Smile task train accuracy: 99.9699969997\n",
      "Gender task train accuracy: 84.3450607036\n",
      "Age task train accuracy: 53.3554228113\n",
      "Total loss: 1.48242. L2-loss: 0.108239\n",
      "Smile loss: 0.00180874\n",
      "Gender loss: 0.337987\n",
      "Age loss: 1.03439\n",
      "\n",
      "\n",
      "Epoch: 75\n",
      "Learning rate: 0.000001\n",
      "Smile task train accuracy: 99.9899989999\n",
      "Gender task train accuracy: 84.4296885938\n",
      "Age task train accuracy: 53.2068804587\n",
      "Total loss: 1.48186. L2-loss: 0.108217\n",
      "Smile loss: 0.00158625\n",
      "Gender loss: 0.337795\n",
      "Age loss: 1.03427\n",
      "\n",
      "\n",
      "Epoch: 76\n",
      "Learning rate: 0.000001\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-6b70e79a0c73>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    105\u001b[0m                                               feed_dict={x: batch_img, y_: batch_label, mask: batch_mask,\n\u001b[1;32m    106\u001b[0m                                                          \u001b[0mphase_train\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 107\u001b[0;31m                                                          keep_prob: 0.5})\n\u001b[0m\u001b[1;32m    108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m         smile_nb_true_pred += sess.run(smile_true_pred, feed_dict={x: batch_img, y_: batch_label, mask: batch_mask,\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    887\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 889\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    890\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    891\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1118\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1119\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1120\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1121\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1122\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1315\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1316\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1317\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1318\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1319\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1321\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1322\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1323\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1324\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1325\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1300\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[1;32m   1301\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1302\u001b[0;31m                                    status, run_metadata)\n\u001b[0m\u001b[1;32m   1303\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1304\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "smile_loss, gender_loss, age_loss, l2_loss, loss = BKNetStyle.selective_loss(y_smile_conv, y_gender_conv,\n",
    "                                                                             y_age_conv, y_, mask)\n",
    "\n",
    "train_step = BKNetStyle.train_op(loss, global_step)\n",
    "\n",
    "smile_mask = tf.get_collection('smile_mask')[0]\n",
    "gender_mask = tf.get_collection('gender_mask')[0]\n",
    "age_mask = tf.get_collection('age_mask')[0]\n",
    "\n",
    "y_smile = tf.get_collection('y_smile')[0]\n",
    "y_gender = tf.get_collection('y_gender')[0]\n",
    "y_age = tf.get_collection('y_age')[0]\n",
    "\n",
    "smile_correct_prediction = tf.equal(tf.argmax(y_smile_conv, 1), tf.argmax(y_smile, 1))\n",
    "gender_correct_prediction = tf.equal(tf.argmax(y_gender_conv, 1), tf.argmax(y_gender, 1))\n",
    "age_correct_prediction = tf.equal(tf.argmax(y_age_conv, 1), tf.argmax(y_age, 1))\n",
    "\n",
    "smile_true_pred = tf.reduce_sum(tf.cast(smile_correct_prediction, dtype=tf.float32) * smile_mask)\n",
    "gender_true_pred = tf.reduce_sum(tf.cast(gender_correct_prediction, dtype=tf.float32) * gender_mask)\n",
    "age_true_pred = tf.reduce_sum(tf.cast(age_correct_prediction, dtype=tf.float32) * age_mask)\n",
    "\n",
    "train_data = []\n",
    "# Mask: Smile -> 0, Gender -> 1, Age -> 2\n",
    "for i in range(len(smile_train) * 10):\n",
    "    img = (smile_train[i % 3000][0] - 128) / 255.0\n",
    "    label = smile_train[i % 3000][1]\n",
    "    train_data.append((img, one_hot(label, 4), 0.0))\n",
    "for i in range(len(gender_train)):\n",
    "    img = (gender_train[i][0] - 128) / 255.0\n",
    "    label = (int)(gender_train[i][1])\n",
    "    train_data.append((img, one_hot(label, 4), 1.0))\n",
    "for i in range(len(age_train)):\n",
    "    img = (age_train[i][0] - 128) / 255.0\n",
    "    label = (int)(age_train[i][1])\n",
    "    train_data.append((img, one_hot(label, 4), 2.0))\n",
    "\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "if not os.path.isfile(SAVE_FOLDER + 'model.ckpt.index'):\n",
    "    print('Create new model')\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    print('OK')\n",
    "else:\n",
    "    print('Restoring existed model')\n",
    "    saver.restore(sess, SAVE_FOLDER + 'model.ckpt')\n",
    "    print('OK')\n",
    "\n",
    "loss_summary_placeholder = tf.placeholder(tf.float32)\n",
    "tf.summary.scalar('loss', loss_summary_placeholder)\n",
    "merge_summary = tf.summary.merge_all()\n",
    "writer = tf.summary.FileWriter(\"./summary/\")\n",
    "\n",
    "learning_rate = tf.get_collection('learning_rate')[0]\n",
    "\n",
    "current_epoch = (int)(global_step.eval() / (len(train_data) // BATCH_SIZE))\n",
    "for epoch in range(current_epoch + 1, NUM_EPOCHS):\n",
    "    print('Epoch:', str(epoch))\n",
    "    np.random.shuffle(train_data)\n",
    "    train_img = []\n",
    "    train_label = []\n",
    "    train_mask = []\n",
    "\n",
    "    for i in range(len(train_data)):\n",
    "        train_img.append(train_data[i][0])\n",
    "        train_label.append(train_data[i][1])\n",
    "        train_mask.append(train_data[i][2])\n",
    "\n",
    "    number_batch = len(train_data) // BATCH_SIZE\n",
    "\n",
    "    avg_ttl = []\n",
    "    avg_rgl = []\n",
    "    avg_smile_loss = []\n",
    "    avg_gender_loss = []\n",
    "    avg_age_loss = []\n",
    "\n",
    "    smile_nb_true_pred = 0\n",
    "    gender_nb_true_pred = 0\n",
    "    age_nb_true_pred = 0\n",
    "\n",
    "    smile_nb_train = 0\n",
    "    gender_nb_train = 0\n",
    "    age_nb_train = 0\n",
    "\n",
    "    print(\"Learning rate: %f\" % learning_rate.eval())\n",
    "    for batch in range(number_batch):\n",
    "#             print('Training on batch {0}/{1}'.format(str(batch + 1), str(number_batch)))\n",
    "        top = batch * BATCH_SIZE\n",
    "        bot = min((batch + 1) * BATCH_SIZE, len(train_data))\n",
    "        batch_img = np.asarray(train_img[top:bot])\n",
    "        batch_label = np.asarray(train_label[top:bot])\n",
    "        batch_mask = np.asarray(train_mask[top:bot])\n",
    "\n",
    "        for i in range(BATCH_SIZE):\n",
    "            if batch_mask[i] == 0.0:\n",
    "                smile_nb_train += 1\n",
    "            else:\n",
    "                if batch_mask[i] == 1.0:\n",
    "                    gender_nb_train += 1                        \n",
    "                else:\n",
    "                    age_nb_train += 1\n",
    "\n",
    "        batch_img = CNN2Head_input.augmentation(batch_img, 48)\n",
    "\n",
    "        ttl, sml, gel, agl, l2l, _ = sess.run([loss, smile_loss, gender_loss, age_loss, l2_loss, train_step],\n",
    "                                              feed_dict={x: batch_img, y_: batch_label, mask: batch_mask,\n",
    "                                                         phase_train: True,\n",
    "                                                         keep_prob: 0.5})\n",
    "\n",
    "        smile_nb_true_pred += sess.run(smile_true_pred, feed_dict={x: batch_img, y_: batch_label, mask: batch_mask,\n",
    "                                                                   phase_train: True,\n",
    "                                                                   keep_prob: 0.5})\n",
    "\n",
    "        gender_nb_true_pred += sess.run(gender_true_pred,\n",
    "                                        feed_dict={x: batch_img, y_: batch_label, mask: batch_mask,\n",
    "                                                   phase_train: True,\n",
    "                                                   keep_prob: 0.5})\n",
    "\n",
    "        age_nb_true_pred += sess.run(age_true_pred,\n",
    "                                         feed_dict={x: batch_img, y_: batch_label, mask: batch_mask,\n",
    "                                                    phase_train: True,\n",
    "                                                    keep_prob: 0.5})\n",
    "\n",
    "        '''--------------------------------------- DEBUG -----------------------------------------------------'''\n",
    "        '''\n",
    "        sm_mask, em_mask, ge_mask = sess.run([smile_mask, gender_mask, age_mask],\n",
    "                                             feed_dict={x: batch_img, y_: batch_label, mask: batch_mask,\n",
    "                                                        phase_train: True,\n",
    "                                                        keep_prob: 0.5})\n",
    "        print('Smile mask: ', sm_mask)\n",
    "        print('Gender mask', ge_mask)\n",
    "        print('Age mask', ag_mask)\n",
    "        print('Batch mask', batch_mask)\n",
    "\n",
    "        y_true_sm, y_true_ge, y_true_ag = sess.run([y_smile, y_gender, y_age],\n",
    "                                                   feed_dict={x: batch_img, y_: batch_label, mask: batch_mask,\n",
    "                                                              phase_train: True,\n",
    "                                                              keep_prob: 0.5})\n",
    "        print('Smile label', y_true_sm)\n",
    "        print('Gender label', y_true_ge)\n",
    "        print('Age label', y_true_ag)\n",
    "        print('Batch label', batch_label)\n",
    "\n",
    "        y_conv_sm, y_conv_ge, y_conv_ag = sess.run([y_smile_conv, y_gender_conv, y_age_conv],\n",
    "                                                   feed_dict={x: batch_img, y_: batch_label, mask: batch_mask,\n",
    "                                                              phase_train: True,\n",
    "                                                              keep_prob: 0.5})\n",
    "        print('Smile conv', y_conv_sm)\n",
    "        print('Gender conv', y_conv_ge)\n",
    "        print('Age conv', y_conv_ag)\n",
    "\n",
    "        '''\n",
    "        '''---------------------------------- END OF DEBUG ----------------------------------------------------'''\n",
    "\n",
    "        avg_ttl.append(ttl)\n",
    "        avg_smile_loss.append(sml)\n",
    "        avg_gender_loss.append(gel)\n",
    "        avg_age_loss.append(agl)\n",
    "\n",
    "        avg_rgl.append(l2l)\n",
    "\n",
    "    smile_train_accuracy = smile_nb_true_pred * 1.0 / smile_nb_train\n",
    "    gender_train_accuracy = gender_nb_true_pred * 1.0 / gender_nb_train\n",
    "    age_train_accuracy = age_nb_true_pred * 1.0 / age_nb_train\n",
    "\n",
    "    avg_smile_loss = np.average(avg_smile_loss)\n",
    "    avg_gender_loss = np.average(avg_gender_loss)\n",
    "    avg_age_loss = np.average(avg_age_loss)\n",
    "\n",
    "    avg_rgl = np.average(avg_rgl)\n",
    "    avg_ttl = np.average(avg_ttl)\n",
    "\n",
    "#     print('Avg_ttl: ' + str(avg_ttl))\n",
    "#     print('loss_summary_placeholder: ' + str(loss_summary_placeholder))\n",
    "#     print('merge_summary: ' + str(merge_summary))\n",
    "\n",
    "    summary = sess.run(merge_summary, feed_dict={loss_summary_placeholder: avg_ttl})\n",
    "    writer.add_summary(summary, global_step=epoch)\n",
    "    \n",
    "    with open('log.csv', 'w+') as f:\n",
    "        # epochs, smile_train_accuracy, gender_train_accuracy, age_train_accuracy,\n",
    "        # avg_smile_loss, avg_gender_loss, avg_age_loss, avg_ttl, avg_rgl\n",
    "        f.write('{0},{1},{2},{3},{4},{5},{6},{7},{8}\\n'.format(current_epoch, smile_train_accuracy, gender_train_accuracy, age_train_accuracy, avg_smile_loss, avg_gender_loss, avg_age_loss, avg_ttl, avg_rgl))\n",
    "\n",
    "    print('Smile task train accuracy: ' + str(smile_train_accuracy * 100))\n",
    "    print('Gender task train accuracy: ' + str(gender_train_accuracy * 100))\n",
    "    print('Age task train accuracy: ' + str(age_train_accuracy * 100))\n",
    "\n",
    "    print('Total loss: ' + str(avg_ttl) + '. L2-loss: ' + str(avg_rgl))\n",
    "    print('Smile loss: ' + str(avg_smile_loss))\n",
    "    print('Gender loss: ' + str(avg_gender_loss))\n",
    "    print('Age loss: ' + str(avg_age_loss))\n",
    "    \n",
    "    print('\\n')\n",
    "\n",
    "    saver.save(sess, SAVE_FOLDER + 'model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}