host.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2 align=\"center\"> Deploy Models with TensorFlow Serving and Docker</h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 1: Load and Preprocess Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "colab_type": "code",
    "id": "ZQhSNW-CZUZh",
    "outputId": "0a0dd94e-b78f-4ffb-c819-5eae6efb6255"
   },
   "outputs": [],
   "source": [
    "#%%writefile -a train.py\n",
    "import os\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Id,ProductId,UserId,ProfileName,HelpfulnessNumerator,HelpfulnessDenominator,Score,Time,Summary,Text\n",
      "184502,B001BCVY4W,A1JMR1N9NBYJ1X,Mad Ethyl Flint,0,0,4,1228176000,Doesn't look like catfood!,\"When you first open the can, it looks like something you would eat.  And no catfood smell! Nice sized chunks of chicken and vegetables in a lot of gravy.<br /><br />That being said, Ms Casiopia lapped up all the gravy and left the rest.  This however is not the product's fault as she has done this before with other catfoods<br /><br />I would have given it 5 stars, but since I won't be purchasing it, I gave it 4.  If your cat will eat chunks and vegetables, this product is for you.<br /><br />I have donated the remainder of the package to a less fortunate friend.<br /><br />Thank you.\"\n"
     ]
    }
   ],
   "source": [
    "#Souce: https://www.kaggle.com/snap/amazon-fine-food-reviews/data\n",
    "!head -n 2 train.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%writefile -a train.py\n",
    "\n",
    "def load_dataset(file_path, num_samples):\n",
    "    df = pd.read_csv(file_path, usecols=[6, 9], nrows=num_samples)\n",
    "    df.columns = ['rating', 'title']\n",
    "\n",
    "    text = df['title'].tolist()\n",
    "    text = [str(t).encode('ascii', 'replace') for t in text]\n",
    "    text = np.array(text, dtype=object)[:]\n",
    "    \n",
    "    labels = df['rating'].tolist()\n",
    "    labels = [1 if i>=4 else 0 if i==3 else -1 for i in labels]\n",
    "    labels = np.array(pd.get_dummies(labels), dtype=int)[:] \n",
    "\n",
    "    return labels, text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100,)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp_labels, tmp_text = load_dataset('train.csv', 100)\n",
    "tmp_text.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 2: Build the Classification Model using TF Hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%writefile -a train.py\n",
    "##https://tfhub.dev/google/tf2-preview/nnlm-en-dim50/1\n",
    "##https://tfhub.dev/google/tf2-preview/nnlm-en-dim128/1\n",
    "\n",
    "def get_model():\n",
    "    hub_layer = hub.KerasLayer(\"https://tfhub.dev/google/tf2-preview/nnlm-en-dim50/1\", output_shape=[50], \n",
    "                           input_shape=[], dtype=tf.string, name='input', trainable=False)\n",
    "\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(hub_layer)\n",
    "    model.add(tf.keras.layers.Dense(64, activation='relu'))\n",
    "    model.add(tf.keras.layers.Dense(3, activation='softmax', name='output'))\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer='Adam', metrics=['accuracy'])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 50), dtype=float32, numpy=\n",
       "array([[ 0.05650096,  0.2567145 ,  0.24404189,  0.14395264, -0.05569138,\n",
       "        -0.10513686,  0.09544804,  0.3080969 , -0.218672  , -0.03048538,\n",
       "        -0.19036277,  0.01005417,  0.11541115, -0.14860378,  0.03914931,\n",
       "        -0.2561884 , -0.15442336,  0.12836292,  0.0469152 , -0.1500514 ,\n",
       "        -0.13068351, -0.01958708,  0.09192695,  0.1208052 , -0.12291992,\n",
       "        -0.04548305, -0.3679261 ,  0.05125156,  0.09797382, -0.10217863,\n",
       "        -0.1965521 ,  0.15523128, -0.05881735, -0.16426983,  0.06646369,\n",
       "         0.05789638,  0.15421619, -0.24014738,  0.11075415, -0.10756174,\n",
       "        -0.01679449, -0.01877424,  0.18602087,  0.2623015 , -0.3829217 ,\n",
       "        -0.34895867, -0.0868978 ,  0.02295742,  0.03787762, -0.02646483],\n",
       "       [-0.01533648,  0.2517981 ,  0.15771465,  0.10011643, -0.03027005,\n",
       "        -0.09655963,  0.10035348, -0.13405894, -0.13515756,  0.15999079,\n",
       "        -0.0257801 ,  0.01482286,  0.17336626,  0.02416893, -0.02589497,\n",
       "        -0.2256546 , -0.10834836,  0.05091727, -0.01329861, -0.1124052 ,\n",
       "        -0.04385714, -0.16535808,  0.07700986, -0.04862161,  0.0580256 ,\n",
       "         0.06278763, -0.10784025,  0.11745881,  0.07221924, -0.15103991,\n",
       "        -0.07155664, -0.00210649, -0.00439631, -0.24547555, -0.16631113,\n",
       "        -0.20567422, -0.05564746, -0.14419016, -0.12905043, -0.056013  ,\n",
       "        -0.10155825, -0.18731159,  0.1180155 ,  0.277938  , -0.02531946,\n",
       "         0.02398384, -0.12459337, -0.03263708,  0.00738647, -0.07372268]],\n",
       "      dtype=float32)>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embed = hub.load(\"https://tfhub.dev/google/tf2-preview/nnlm-en-dim50/1\")\n",
    "embeddings = embed([\"this is a test\", \"look at the embeddings\"])\n",
    "embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "del embed, embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 3: Define Training Procedure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%writefile -a train.py\n",
    "\n",
    "def train(EPOCHS=5, BATCH_SIZE=32, TRAIN_FILE='train.csv', VAL_FILE='test.csv'):\n",
    "    WORKING_DIR = os.getcwd() #use to specify model checkpoint path\n",
    "    print(\"Loading training/validation data ...\")\n",
    "    y_train, x_train = load_dataset(TRAIN_FILE, num_samples=100000)\n",
    "    y_val, x_val = load_dataset(VAL_FILE, num_samples=10000)\n",
    "\n",
    "    print(\"Training the model ...\")\n",
    "    model = get_model()\n",
    "    model.fit(x_train, y_train, batch_size=BATCH_SIZE, epochs=EPOCHS, verbose=1,\n",
    "              validation_data=(x_val, y_val),\n",
    "              callbacks=[tf.keras.callbacks.ModelCheckpoint(os.path.join(WORKING_DIR,\n",
    "                                                                         'model_checkpoint'),\n",
    "                                                            monitor='val_loss', verbose=1,\n",
    "                                                            save_best_only=True,\n",
    "                                                            save_weights_only=False,\n",
    "                                                            mode='auto')])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 4: Train and Export Model as Protobuf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading training/validation data ...\n",
      "Training the model ...\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input (KerasLayer)           (None, 50)                48190600  \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 64)                3264      \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 3)                 195       \n",
      "=================================================================\n",
      "Total params: 48,194,059\n",
      "Trainable params: 3,459\n",
      "Non-trainable params: 48,190,600\n",
      "_________________________________________________________________\n",
      "Epoch 1/5\n",
      "3121/3125 [============================>.] - ETA: 0s - loss: 0.5838 - accuracy: 0.7859\n",
      "Epoch 00001: val_loss improved from inf to 0.56583, saving model to /home/rhyme/Desktop/Project/model_checkpoint\n",
      "WARNING:tensorflow:From /home/rhyme/.local/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py:1817: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/rhyme/.local/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py:1817: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: /home/rhyme/Desktop/Project/model_checkpoint/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: /home/rhyme/Desktop/Project/model_checkpoint/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3125/3125 [==============================] - 25s 8ms/step - loss: 0.5836 - accuracy: 0.7860 - val_loss: 0.5658 - val_accuracy: 0.7888\n",
      "Epoch 2/5\n",
      "3125/3125 [==============================] - ETA: 0s - loss: 0.5609 - accuracy: 0.7920\n",
      "Epoch 00002: val_loss improved from 0.56583 to 0.56067, saving model to /home/rhyme/Desktop/Project/model_checkpoint\n",
      "INFO:tensorflow:Assets written to: /home/rhyme/Desktop/Project/model_checkpoint/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: /home/rhyme/Desktop/Project/model_checkpoint/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3125/3125 [==============================] - 24s 8ms/step - loss: 0.5609 - accuracy: 0.7920 - val_loss: 0.5607 - val_accuracy: 0.7898\n",
      "Epoch 3/5\n",
      "3119/3125 [============================>.] - ETA: 0s - loss: 0.5549 - accuracy: 0.7936\n",
      "Epoch 00003: val_loss improved from 0.56067 to 0.55701, saving model to /home/rhyme/Desktop/Project/model_checkpoint\n",
      "INFO:tensorflow:Assets written to: /home/rhyme/Desktop/Project/model_checkpoint/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: /home/rhyme/Desktop/Project/model_checkpoint/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3125/3125 [==============================] - 24s 8ms/step - loss: 0.5549 - accuracy: 0.7936 - val_loss: 0.5570 - val_accuracy: 0.7907\n",
      "Epoch 4/5\n",
      "3125/3125 [==============================] - ETA: 0s - loss: 0.5514 - accuracy: 0.7951\n",
      "Epoch 00004: val_loss did not improve from 0.55701\n",
      "3125/3125 [==============================] - 22s 7ms/step - loss: 0.5514 - accuracy: 0.7951 - val_loss: 0.5575 - val_accuracy: 0.7914\n",
      "Epoch 5/5\n",
      "3118/3125 [============================>.] - ETA: 0s - loss: 0.5484 - accuracy: 0.7954\n",
      "Epoch 00005: val_loss improved from 0.55701 to 0.55509, saving model to /home/rhyme/Desktop/Project/model_checkpoint\n",
      "WARNING:tensorflow:5 out of the last 5 calls to <function recreate_function.<locals>.restored_function_body at 0x7f0c9afe3a60> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:5 out of the last 5 calls to <function recreate_function.<locals>.restored_function_body at 0x7f0c9afe3a60> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: /home/rhyme/Desktop/Project/model_checkpoint/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: /home/rhyme/Desktop/Project/model_checkpoint/assets\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3125/3125 [==============================] - 24s 8ms/step - loss: 0.5483 - accuracy: 0.7954 - val_loss: 0.5551 - val_accuracy: 0.7907\n",
      "WARNING:tensorflow:6 out of the last 6 calls to <function recreate_function.<locals>.restored_function_body at 0x7f0c9afe3a60> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:6 out of the last 6 calls to <function recreate_function.<locals>.restored_function_body at 0x7f0c9afe3a60> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings is likely due to passing python objects instead of tensors. Also, tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. Please refer to https://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args and https://www.tensorflow.org/api_docs/python/tf/function for more details.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: amazon_review/1598679107/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: amazon_review/1598679107/assets\n"
     ]
    }
   ],
   "source": [
    "#%%writefile -a train.py\n",
    "\n",
    "def export_model(model, base_path=\"amazon_review/\"):\n",
    "    path = os.path.join(base_path, str(int(time.time())))\n",
    "    tf.saved_model.save(model, path)\n",
    "\n",
    "\n",
    "if __name__== '__main__':\n",
    "    model = train()\n",
    "    export_model(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 5: Test Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Negative Review:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.5471361 , 0.05663168, 0.3962322 ]], dtype=float32)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_sentence = \"horrible book, waste of time\"\n",
    "model.predict([test_sentence])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Positive Review:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.02511285, 0.02534903, 0.9495381 ]], dtype=float32)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_sentence = \"Awesome product.\"\n",
    "model.predict([test_sentence])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 6: TensorFlow Serving with Docker"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`docker pull tensorflow/serving`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`docker run -p 8500:8500 \\\n",
    "            -p 8501:8501 \\\n",
    "            --mount type=bind,\\\n",
    "            source=/path/amazon_review/,\\\n",
    "            target=/models/amazon_review \\\n",
    "            -e MODEL_NAME=amazon_review \\\n",
    "            -t tensorflow/serving`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 7: Setup a REST Client to Perform Model Predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Perform Model Prediction\n",
    "\n",
    "##### Support for gRPC and REST\n",
    "\n",
    "- TensorFlow Serving supports\n",
    "    - Remote Procedure Protocal (gRPC)\n",
    "    - Representational State Transfer (REST)\n",
    "- Consistent API structures\n",
    "- Server supports both standards simultaneously\n",
    "- Default ports:\n",
    "    - RPC: 8500\n",
    "    - REST: 8501"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Predictions via REST\n",
    "\n",
    "- Standard HTTP POST requests\n",
    "- Response is a JSON body with the prediction\n",
    "- Request from the default or specific model\n",
    "\n",
    "Default URI scheme:\n",
    "\n",
    "`http://{HOST}:{PORT}/v1/models/{MODEL_NAME}`\n",
    "\n",
    "Specific model versions:\n",
    "\n",
    "`http://{HOST}:{PORT}/v1/models/{MODEL_NAME}[/versions/{MODEL_VERSION}]:predict`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3NND75SMZUsP"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting tf_serving_rest_client.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile tf_serving_rest_client.py\n",
    "import json\n",
    "import requests\n",
    "import sys\n",
    "\n",
    "def get_rest_url(model_name, host='127.0.0.1', port='8501', verb='predict', version=None):\n",
    "    \"\"\" generate the URL path\"\"\"\n",
    "    url = \"http://{host}:{port}/v1/models/{model_name}\".format(host=host, port=port, model_name=model_name)\n",
    "    if version:\n",
    "        url += 'versions/{version}'.format(version=version)\n",
    "    url += ':{verb}'.format(verb=verb)\n",
    "    return url\n",
    "\n",
    "\n",
    "def get_model_prediction(model_input, model_name='amazon_review', signature_name='serving_default'):\n",
    "    \"\"\" no error handling at all, just poc\"\"\"\n",
    "\n",
    "    url = get_rest_url(model_name)\n",
    "    #In the row format, inputs are keyed to instances key in the JSON request.\n",
    "    #When there is only one named input, specify the value of instances key to be the value of the input:\n",
    "    data = {\"instances\": [model_input]}\n",
    "    \n",
    "    rv = requests.post(url, data=json.dumps(data))\n",
    "    if rv.status_code != requests.codes.ok:\n",
    "        rv.raise_for_status()\n",
    "    \n",
    "    return rv.json()['predictions']\n",
    "\n",
    "if __name__ == '__main__':\n",
    "\n",
    "    print(\"\\nGenerate REST url ...\")\n",
    "    url = get_rest_url(model_name='amazon_review')\n",
    "    print(url)\n",
    "    \n",
    "    while True:\n",
    "        print(\"\\nEnter an Amazon review [:q for Quit]\")\n",
    "        if sys.version_info[0] <= 3:\n",
    "            sentence = input()\n",
    "        if sentence == ':q':\n",
    "            break\n",
    "        model_input = sentence\n",
    "        model_prediction = get_model_prediction(model_input)\n",
    "        print(\"The model predicted ...\")\n",
    "        print(model_prediction)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 8: Setup a gRPC Client to Perform Model Predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Modified from [https://github.com/tensorflow/serving/blob/master/tensorflow_serving/example/mnist_client.py](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/example/mnist_client.py#L152)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Predictions via gRPC\n",
    "\n",
    "More sophisticated client-server connections\n",
    "\n",
    "- Prediction data has to be converted to the Protobuf format\n",
    "- Request types have designated types, e.g. float, int, bytes\n",
    "- Payloads need to be converted to base64\n",
    "- Connect to the server via gRPC stubs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### gRPC vs REST: When to use which API standard\n",
    "\n",
    "- Rest is easy to implement and debug\n",
    "- RPC is more network efficient, smaller payloads\n",
    "- RPC can provide much faster inferences!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jKJVOjDlZUvc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting tf_serving_grpc_client.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile tf_serving_grpc_client.py\n",
    "import sys\n",
    "import grpc\n",
    "from grpc.beta import implementations\n",
    "import tensorflow as tf\n",
    "from tensorflow_serving.apis import predict_pb2\n",
    "from tensorflow_serving.apis import prediction_service_pb2, get_model_metadata_pb2\n",
    "from tensorflow_serving.apis import prediction_service_pb2_grpc\n",
    "\n",
    "\n",
    "def get_stub(host='127.0.0.1', port='8500'):\n",
    "    channel = grpc.insecure_channel('127.0.0.1:8500') \n",
    "    stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)\n",
    "    return stub\n",
    "\n",
    "\n",
    "def get_model_prediction(model_input, stub, model_name='amazon_review', signature_name='serving_default'):\n",
    "    \"\"\" no error handling at all, just poc\"\"\"\n",
    "    request = predict_pb2.PredictRequest()\n",
    "    request.model_spec.name = model_name\n",
    "    request.model_spec.signature_name = signature_name\n",
    "    request.inputs['input_input'].CopyFrom(tf.make_tensor_proto(model_input))\n",
    "    response = stub.Predict.future(request, 5.0)  # 5 seconds\n",
    "    return response.result().outputs[\"output\"].float_val\n",
    "\n",
    "\n",
    "def get_model_version(model_name, stub):\n",
    "    request = get_model_metadata_pb2.GetModelMetadataRequest()\n",
    "    request.model_spec.name = 'amazon_review'\n",
    "    request.metadata_field.append(\"signature_def\")\n",
    "    response = stub.GetModelMetadata(request, 10)\n",
    "    # signature of loaded model is available here: response.metadata['signature_def']\n",
    "    return response.model_spec.version.value\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    print(\"\\nCreate RPC connection ...\")\n",
    "    stub = get_stub()\n",
    "    while True:\n",
    "        print(\"\\nEnter an Amazon review [:q for Quit]\")\n",
    "        if sys.version_info[0] <= 3:\n",
    "            sentence = raw_input() if sys.version_info[0] < 3 else input()\n",
    "        if sentence == ':q':\n",
    "            break\n",
    "        model_input = [sentence]\n",
    "        model_prediction = get_model_prediction(model_input, stub)\n",
    "        print(\"The model predicted ...\")\n",
    "        print(model_prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TidfRe2VZU39"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "MetaGraphDef with tag-set: 'serve' contains the following SignatureDefs:\n",
      "\n",
      "signature_def['__saved_model_init_op']:\n",
      "  The given SavedModel SignatureDef contains the following input(s):\n",
      "  The given SavedModel SignatureDef contains the following output(s):\n",
      "    outputs['__saved_model_init_op'] tensor_info:\n",
      "        dtype: DT_INVALID\n",
      "        shape: unknown_rank\n",
      "        name: NoOp\n",
      "  Method name is: \n",
      "\n",
      "signature_def['serving_default']:\n",
      "  The given SavedModel SignatureDef contains the following input(s):\n",
      "    inputs['input_input'] tensor_info:\n",
      "        dtype: DT_STRING\n",
      "        shape: (-1)\n",
      "        name: serving_default_input_input:0\n",
      "  The given SavedModel SignatureDef contains the following output(s):\n",
      "    outputs['output'] tensor_info:\n",
      "        dtype: DT_FLOAT\n",
      "        shape: (-1, 3)\n",
      "        name: StatefulPartitionedCall_2:0\n",
      "  Method name is: tensorflow/serving/predict\n",
      "WARNING:tensorflow:From /home/rhyme/.local/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py:1817: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "2020-08-29 05:32:58.782500: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1\n",
      "2020-08-29 05:32:58.786410: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-08-29 05:32:58.786823: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1561] Found device 0 with properties: \n",
      "pciBusID: 0000:00:1e.0 name: Tesla K80 computeCapability: 3.7\n",
      "coreClock: 0.8235GHz coreCount: 13 deviceMemorySize: 11.17GiB deviceMemoryBandwidth: 223.96GiB/s\n",
      "2020-08-29 05:32:58.787083: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1\n",
      "2020-08-29 05:32:58.789618: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10\n",
      "2020-08-29 05:32:58.792044: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10\n",
      "2020-08-29 05:32:58.792461: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10\n",
      "2020-08-29 05:32:58.794512: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10\n",
      "2020-08-29 05:32:58.795708: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10\n",
      "2020-08-29 05:32:58.799978: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
      "2020-08-29 05:32:58.800170: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-08-29 05:32:58.800660: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-08-29 05:32:58.801027: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1703] Adding visible gpu devices: 0\n",
      "2020-08-29 05:32:58.801404: I tensorflow/core/platform/cpu_feature_guard.cc:143] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2020-08-29 05:32:58.828354: I tensorflow/core/platform/profile_utils/cpu_utils.cc:102] CPU Frequency: 2300020000 Hz\n",
      "2020-08-29 05:32:58.828802: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x7ff018000b20 initialized for platform Host (this does not guarantee that XLA will be used). Devices:\n",
      "2020-08-29 05:32:58.828841: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version\n",
      "2020-08-29 05:32:58.892844: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-08-29 05:32:58.893457: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5c59a30 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2020-08-29 05:32:58.893501: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Tesla K80, Compute Capability 3.7\n",
      "2020-08-29 05:32:58.893766: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-08-29 05:32:58.894240: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1561] Found device 0 with properties: \n",
      "pciBusID: 0000:00:1e.0 name: Tesla K80 computeCapability: 3.7\n",
      "coreClock: 0.8235GHz coreCount: 13 deviceMemorySize: 11.17GiB deviceMemoryBandwidth: 223.96GiB/s\n",
      "2020-08-29 05:32:58.894319: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1\n",
      "2020-08-29 05:32:58.894363: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10\n",
      "2020-08-29 05:32:58.894403: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10\n",
      "2020-08-29 05:32:58.894446: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10\n",
      "2020-08-29 05:32:58.894486: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10\n",
      "2020-08-29 05:32:58.894529: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10\n",
      "2020-08-29 05:32:58.894558: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
      "2020-08-29 05:32:58.894655: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-08-29 05:32:58.895024: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-08-29 05:32:58.895379: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1703] Adding visible gpu devices: 0\n",
      "2020-08-29 05:32:58.895438: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1\n",
      "2020-08-29 05:32:58.896652: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2020-08-29 05:32:58.896683: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1108]      0 \n",
      "2020-08-29 05:32:58.896704: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1121] 0:   N \n",
      "2020-08-29 05:32:58.896843: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-08-29 05:32:58.897282: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-08-29 05:32:58.897672: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1247] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 301 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:1e.0, compute capability: 3.7)\n",
      "\n",
      "Defined Functions:\n",
      "  Function Name: '__call__'\n",
      "    Option #1\n",
      "      Callable with:\n",
      "        Argument #1\n",
      "          input_input: TensorSpec(shape=(None,), dtype=tf.string, name='input_input')\n",
      "        Argument #2\n",
      "          DType: bool\n",
      "          Value: False\n",
      "        Argument #3\n",
      "          DType: NoneType\n",
      "          Value: None\n",
      "    Option #2\n",
      "      Callable with:\n",
      "        Argument #1\n",
      "          inputs: TensorSpec(shape=(None,), dtype=tf.string, name='inputs')\n",
      "        Argument #2\n",
      "          DType: bool\n",
      "          Value: False\n",
      "        Argument #3\n",
      "          DType: NoneType\n",
      "          Value: None\n",
      "    Option #3\n",
      "      Callable with:\n",
      "        Argument #1\n",
      "          inputs: TensorSpec(shape=(None,), dtype=tf.string, name='inputs')\n",
      "        Argument #2\n",
      "          DType: bool\n",
      "          Value: True\n",
      "        Argument #3\n",
      "          DType: NoneType\n",
      "          Value: None\n",
      "    Option #4\n",
      "      Callable with:\n",
      "        Argument #1\n",
      "          input_input: TensorSpec(shape=(None,), dtype=tf.string, name='input_input')\n",
      "        Argument #2\n",
      "          DType: bool\n",
      "          Value: True\n",
      "        Argument #3\n",
      "          DType: NoneType\n",
      "          Value: None\n",
      "\n",
      "  Function Name: '_default_save_signature'\n",
      "    Option #1\n",
      "      Callable with:\n",
      "        Argument #1\n",
      "          input_input: TensorSpec(shape=(None,), dtype=tf.string, name='input_input')\n",
      "\n",
      "  Function Name: 'call_and_return_all_conditional_losses'\n",
      "    Option #1\n",
      "      Callable with:\n",
      "        Argument #1\n",
      "          inputs: TensorSpec(shape=(None,), dtype=tf.string, name='inputs')\n",
      "        Argument #2\n",
      "          DType: bool\n",
      "          Value: False\n",
      "        Argument #3\n",
      "          DType: NoneType\n",
      "          Value: None\n",
      "    Option #2\n",
      "      Callable with:\n",
      "        Argument #1\n",
      "          inputs: TensorSpec(shape=(None,), dtype=tf.string, name='inputs')\n",
      "        Argument #2\n",
      "          DType: bool\n",
      "          Value: True\n",
      "        Argument #3\n",
      "          DType: NoneType\n",
      "          Value: None\n",
      "    Option #3\n",
      "      Callable with:\n",
      "        Argument #1\n",
      "          input_input: TensorSpec(shape=(None,), dtype=tf.string, name='input_input')\n",
      "        Argument #2\n",
      "          DType: bool\n",
      "          Value: False\n",
      "        Argument #3\n",
      "          DType: NoneType\n",
      "          Value: None\n",
      "    Option #4\n",
      "      Callable with:\n",
      "        Argument #1\n",
      "          input_input: TensorSpec(shape=(None,), dtype=tf.string, name='input_input')\n",
      "        Argument #2\n",
      "          DType: bool\n",
      "          Value: True\n",
      "        Argument #3\n",
      "          DType: NoneType\n",
      "          Value: None\n"
     ]
    }
   ],
   "source": [
    "!saved_model_cli show --dir `pwd`/amazon_review/1598679107 --all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "deploy-TF-Serving.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}