Tensorflow - 07 RNN (mnist).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/abulbasar/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import h5py\n",
    "\n",
    "import math\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist_csv(path = \"/data/MNIST/\", one_hot = False, shape = None):\n",
    "    df_train = pd.read_csv(path + \"mnist_train.csv\", header=None)\n",
    "    df_test = pd.read_csv(path + \"mnist_test.csv\", header=None)\n",
    "    \n",
    "    X_train = df_train.iloc[:, 1:].values/255\n",
    "    X_test = df_test.iloc[:, 1:].values/255\n",
    "    y_train = df_train.iloc[:, 0].values\n",
    "    y_test = df_test.iloc[:, 0].values\n",
    "    \n",
    "    if shape == \"2D\":\n",
    "        X_train = X_train.reshape(-1, 28, 28)\n",
    "        X_test = X_test.reshape(-1, 28, 28)\n",
    "    \n",
    "    if one_hot:\n",
    "        eye = np.eye(len(np.unique(y_train)))\n",
    "        y_train, y_test = eye[y_train], eye[y_test]\n",
    "        \n",
    "    return X_train, X_test, y_train, y_test\n",
    "\n",
    "X_train, X_test, y_train, y_test = load_mnist_csv(shape = \"2D\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Outputs:  [<tf.Tensor 'rnn/basic_rnn_cell/Tanh:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_1:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_2:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_3:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_4:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_5:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_6:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_7:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_8:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_9:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_10:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_11:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_12:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_13:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_14:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_15:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_16:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_17:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_18:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_19:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_20:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_21:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_22:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_23:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_24:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_25:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_26:0' shape=(?, 400) dtype=float32>, <tf.Tensor 'rnn/basic_rnn_cell/Tanh_27:0' shape=(?, 400) dtype=float32>]\n",
      "States:  Tensor(\"rnn/basic_rnn_cell/Tanh_27:0\", shape=(?, 400), dtype=float32)\n",
      "epoch:  0, progress: 100%, cost: 0.30333, train acc: 0.9000\n",
      "Test accuracy: 0.9240\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "tf.set_random_seed(1)\n",
    "\n",
    "n_steps = 28\n",
    "n_inputs = 28\n",
    "state_size = 400\n",
    "n_outputs = 10\n",
    "\n",
    "batch_size = 32\n",
    "\n",
    "epochs = 1\n",
    "\n",
    "learning_rate = 0.001\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "y = tf.placeholder(tf.int32, [None])\n",
    "\n",
    "X_sequence = tf.unstack(X, axis=-1)\n",
    "\n",
    "cell = tf.contrib.rnn.BasicRNNCell(num_units=state_size, activation=tf.nn.tanh)\n",
    "\n",
    "nrows = tf.shape(X)[0]\n",
    "\n",
    "#initial_state = cell.zero_state(batch_size=nrows, dtype=tf.float32)\n",
    "initial_state = tf.truncated_normal(shape=(nrows, state_size), dtype=tf.float32, stddev=0.1)\n",
    "\n",
    "\n",
    "outputs, states = tf.nn.static_rnn(cell, X_sequence, initial_state=initial_state,  dtype=tf.float32)\n",
    "print(\"Outputs: \", outputs)\n",
    "print(\"States: \", states)\n",
    "\n",
    "logits = tf.layers.dense(states, n_outputs)\n",
    "\n",
    "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "cost = tf.reduce_mean(xentropy)\n",
    "opt = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n",
    "y_pred = tf.argmax(logits, axis=1, output_type=tf.int32)\n",
    "correct = tf.equal(y, y_pred)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "\n",
    "n_epochs = 100\n",
    "batch_size = 150\n",
    "\n",
    "m = len(X_train)\n",
    "num_batches = math.ceil(m/batch_size)\n",
    "\n",
    "\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for i in range(epochs):\n",
    "        indices = np.arange(m)\n",
    "        np.random.shuffle(indices)\n",
    "        X_train = X_train[indices]\n",
    "        y_train = y_train[indices]\n",
    "        for j in range(num_batches):\n",
    "            X_batch = X_train[j * batch_size: (j+1) * batch_size]\n",
    "            y_batch = y_train[j * batch_size: (j+1) * batch_size]\n",
    "            _, cost_, acc_train = sess.run([opt, cost, accuracy], feed_dict={X: X_batch, y: y_batch})\n",
    "            progress = (j+1)*100//num_batches\n",
    "            print(\"epoch: %2d, progress: %3d%%, cost: %.5f, train acc: %.4f\" \n",
    "                  % (i, progress, cost_, acc_train), end=\"\\r\")\n",
    "\n",
    "        acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})\n",
    "        print(\"\\nTest accuracy: %.4f\" % acc_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "States:  Tensor(\"rnn/basic_lstm_cell/Mul_83:0\", shape=(?, 400), dtype=float32)\n",
      "epoch:  0, progress: 100%, cost: 0.14906, train acc: 0.9600\n",
      "Test accuracy: 0.9514\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "tf.set_random_seed(1)\n",
    "\n",
    "n_steps = 28\n",
    "n_inputs = 28\n",
    "state_size = 400\n",
    "n_outputs = 10\n",
    "\n",
    "batch_size = 32\n",
    "\n",
    "epochs = 1\n",
    "\n",
    "learning_rate = 0.001\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "y = tf.placeholder(tf.int32, [None])\n",
    "\n",
    "X_sequence = tf.unstack(X, axis=-1)\n",
    "\n",
    "cell = tf.contrib.rnn.BasicLSTMCell(num_units=state_size, activation=tf.nn.tanh, state_is_tuple=True)\n",
    "outputs, (states_c, states_h) = tf.nn.static_rnn(cell, X_sequence, dtype=tf.float32)\n",
    "print(\"States: \", states_h)\n",
    "\n",
    "logits = tf.layers.dense(states_h, n_outputs)\n",
    "\n",
    "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "cost = tf.reduce_mean(xentropy)\n",
    "opt = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n",
    "y_pred = tf.argmax(logits, axis=1, output_type=tf.int32)\n",
    "correct = tf.equal(y, y_pred)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "\n",
    "n_epochs = 100\n",
    "batch_size = 150\n",
    "\n",
    "m = len(X_train)\n",
    "num_batches = math.ceil(m/batch_size)\n",
    "\n",
    "\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for i in range(epochs):\n",
    "        indices = np.arange(m)\n",
    "        np.random.shuffle(indices)\n",
    "        X_train = X_train[indices]\n",
    "        y_train = y_train[indices]\n",
    "        for j in range(num_batches):\n",
    "            X_batch = X_train[j * batch_size: (j+1) * batch_size]\n",
    "            y_batch = y_train[j * batch_size: (j+1) * batch_size]\n",
    "            _, cost_, acc_train = sess.run([opt, cost, accuracy], feed_dict={X: X_batch, y: y_batch})\n",
    "            progress = (j+1)*100//num_batches\n",
    "            print(\"epoch: %2d, progress: %3d%%, cost: %.5f, train acc: %.4f\" \n",
    "                  % (i, progress, cost_, acc_train), end=\"\\r\")\n",
    "\n",
    "        acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})\n",
    "        print(\"\\nTest accuracy: %.4f\" % acc_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "States:  Tensor(\"rnn/while/Exit_4:0\", shape=(?, 400), dtype=float32)\n",
      "epoch:  0, progress: 100%, cost: 0.14378, train acc: 0.9667\n",
      "Test accuracy: 0.9517\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "tf.set_random_seed(1)\n",
    "\n",
    "n_steps = 28\n",
    "n_inputs = 28\n",
    "state_size = 400\n",
    "n_outputs = 10\n",
    "\n",
    "batch_size = 32\n",
    "\n",
    "epochs = 1\n",
    "\n",
    "learning_rate = 0.001\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])\n",
    "y = tf.placeholder(tf.int32, [None])\n",
    "\n",
    "X_sequence = tf.unstack(X, axis=-1)\n",
    "\n",
    "cell = tf.contrib.rnn.BasicLSTMCell(num_units=state_size, activation=tf.nn.tanh, state_is_tuple=True)\n",
    "outputs, (states_c, states_h) = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32, )\n",
    "print(\"States: \", states_h)\n",
    "\n",
    "logits = tf.layers.dense(states_h, n_outputs)\n",
    "\n",
    "xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)\n",
    "cost = tf.reduce_mean(xentropy)\n",
    "opt = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n",
    "y_pred = tf.argmax(logits, axis=1, output_type=tf.int32)\n",
    "correct = tf.equal(y, y_pred)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "\n",
    "n_epochs = 100\n",
    "batch_size = 150\n",
    "\n",
    "m = len(X_train)\n",
    "num_batches = math.ceil(m/batch_size)\n",
    "\n",
    "\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for i in range(epochs):\n",
    "        indices = np.arange(m)\n",
    "        np.random.shuffle(indices)\n",
    "        X_train = X_train[indices]\n",
    "        y_train = y_train[indices]\n",
    "        for j in range(num_batches):\n",
    "            X_batch = X_train[j * batch_size: (j+1) * batch_size]\n",
    "            y_batch = y_train[j * batch_size: (j+1) * batch_size]\n",
    "            _, cost_, acc_train = sess.run([opt, cost, accuracy], feed_dict={X: X_batch, y: y_batch})\n",
    "            progress = (j+1)*100//num_batches\n",
    "            print(\"epoch: %2d, progress: %3d%%, cost: %.5f, train acc: %.4f\" \n",
    "                  % (i, progress, cost_, acc_train), end=\"\\r\")\n",
    "\n",
    "        acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})\n",
    "        print(\"\\nTest accuracy: %.4f\" % acc_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}