speech_recognition_CNN_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "from scipy.io import wavfile\n",
    "import scipy.io.wavfile as wav\n",
    "import librosa\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import tensorflow as tf\n",
    "import random\n",
    "\n",
    "\n",
    "from python_speech_features import mfcc\n",
    "from python_speech_features import logfbank\n",
    "\n",
    "\n",
    "from tensorflow.keras.models import Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Input, Conv1D, LSTM, Dense\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import keras\n",
    "# from keras.preprocessing.text import Tokenizer\n",
    "# from keras.preprocessing.sequence import pad_sequences\n",
    "# from keras.utils import to_categorical,np_utils\n",
    "# from keras.layers import Dense, Input, GlobalMaxPooling1D,Dropout, Activation, Flatten\n",
    "# from keras.layers import Conv1D, MaxPooling1D, Embedding, Convolution2D, MaxPooling2D,Conv2D, Reshape, GRU, TimeDistributed, Lambda\n",
    "# from keras.models import Model\n",
    "\n",
    "# from keras.preprocessing import sequence\n",
    "# from keras.models import Sequential\n",
    "# from keras.layers import Dense, Embedding\n",
    "# from keras.layers import LSTM , Bidirectional,Dropout, BatchNormalization\n",
    "# from keras import backend as K\n",
    "# from keras.layers.advanced_activations import LeakyReLU\n",
    "# from keras import regularizers\n",
    "# from keras.layers.merge import add, concatenate\n",
    "# from keras.callbacks import ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\maruf\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:5: WavFileWarning: Chunk (non-data) not understood, skipping it.\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "path = r\"data\\recordings\"\n",
    "files = os.listdir(path)\n",
    "audio_data=[]\n",
    "for filename in glob.glob(os.path.join(path, '*.wav')):\n",
    "    samplerate, data = wavfile.read(filename)\n",
    "    audio_data.append(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "label=[]\n",
    "for i in files:\n",
    "    label.append(i[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\maruf\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:3: WavFileWarning: Chunk (non-data) not understood, skipping it.\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "raw_aud_data=[]\n",
    "for filename in glob.glob(os.path.join(path, '*.wav')):\n",
    "    (rate,sig) = wav.read(filename)\n",
    "    raw_aud_data.append(sig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_audio_trans=label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5235,)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_aud_data[10].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_aud_data_flat=[]\n",
    "for i in raw_aud_data:\n",
    "    raw_aud_data_flat.append(i.reshape(1,-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 3142)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_aud_data_flat[100].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_audio_trans[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_int=[]\n",
    "for i in label:\n",
    "    label_int.append(int(i))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import inflect\n",
    "\n",
    "p = inflect. engine()\n",
    "final_audio_trans=[]\n",
    "for i in label_int:\n",
    "    final_audio_trans.append(p. number_to_words(i))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "samp_id=list(range(0,1500)) #1500 is the number of data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#shuffling multiple times\n",
    "for _ in range(3):\n",
    "    random.shuffle(samp_id)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving sample_id as text\n",
    "with open(\"samp_id.txt\", \"wb\") as fp:\n",
    "    pickle.dump(samp_id, fp)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading sample_id as text\n",
    "with open(\"samp_id.txt\", \"rb\") as fp:\n",
    "    samp_id = pickle.load(fp)    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#shuffled training data\n",
    "raw_aud_data_flat_shuffle=[]\n",
    "final_audio_trans_shuffled=[]\n",
    "final_audio_trans_shuffled=[]\n",
    "for i in samp_id:\n",
    "    raw_aud_data_flat_shuffle.append(raw_aud_data_flat[i])\n",
    "    final_audio_trans_shuffled.append(final_audio_trans[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_aud_list=final_audio_trans_shuffled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_texts = []                            \n",
    "output_vocabulary = set()\n",
    "start_token = '@'                                             \n",
    "stop_token = '#'\n",
    "\n",
    "for i in final_aud_list:\n",
    "    target_text = start_token + i + stop_token\n",
    "    target_texts.append(target_text)\n",
    "    \n",
    "    for char in target_text:\n",
    "        if char not in output_vocabulary:\n",
    "            output_vocabulary.add(char)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#function to get the actual word-characters till # appears in the text\n",
    "def actual_word(mytextlist):\n",
    "    revised_text=[]\n",
    "    for i in mytextlist:\n",
    "        ct=[]\n",
    "        if \"#\" in i:\n",
    "            for j in range(i.index(\"#\")):\n",
    "                ct.append(i[j])\n",
    "            revised_text.append(\"\".join(ct))\n",
    "        else:\n",
    "            revised_text.append(i)\n",
    "            \n",
    "        #revised_text.append(\"\".join(ct))\n",
    "    return revised_text   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "max_decoder_seq_length = max([len(txt) for txt in target_texts])\n",
    "target_token_index = dict([(char, i) for i, char in enumerate(output_vocabulary)])\n",
    "reverse_target_char_index = dict((i, char) for char, i in target_token_index.items())\n",
    "output_vocab_size=len(output_vocabulary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'r',\n",
       " 1: 'o',\n",
       " 2: 'w',\n",
       " 3: '@',\n",
       " 4: 'u',\n",
       " 5: '#',\n",
       " 6: 'f',\n",
       " 7: 'x',\n",
       " 8: 'z',\n",
       " 9: 'h',\n",
       " 10: 'v',\n",
       " 11: 't',\n",
       " 12: 'e',\n",
       " 13: 'n',\n",
       " 14: 's',\n",
       " 15: 'i',\n",
       " 16: 'g'}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reverse_target_char_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder_input_data = np.zeros((len(target_texts), max_decoder_seq_length, output_vocab_size), dtype='float32')\n",
    "decoder_target_data = np.zeros((len(target_texts), max_decoder_seq_length, output_vocab_size), dtype='float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#encoder_input_data = np.zeros((len(target_texts), max_decoder_seq_length, output_vocab_size), dtype='float32')\n",
    "decoder_input_data = np.zeros((len(target_texts), max_decoder_seq_length, output_vocab_size), dtype='float32')\n",
    "decoder_target_data = np.zeros((len(target_texts), max_decoder_seq_length, output_vocab_size), dtype='float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for i, text in enumerate(target_texts):  \n",
    "    for t, char in enumerate(text):\n",
    "            decoder_input_data[i, t, target_token_index[char]] = 1.\n",
    "            if t > 0:\n",
    "                decoder_target_data[i, t - 1, target_token_index[char]] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This is for input text\n",
    "#for i, text in enumerate(input_text):  \n",
    "#    for t, char in enumerate(text):\n",
    "#            decoder_input_data[i, t, target_token_index[char]] = 1.\n",
    "#            if t > 0:\n",
    "#                decoder_target_data[i, t - 1, target_token_index[char]] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving decoder_input_data as text\n",
    "\n",
    "with open(\"mfcc_data_shuffled.txt\", \"wb\") as fp:\n",
    "    pickle.dump(mfcc_data_shuffled, fp)\n",
    "\n",
    "with open(\"decoder_input_data.txt\", \"wb\") as fp:\n",
    "    pickle.dump(decoder_input_data, fp)\n",
    "    \n",
    "#Saving decoder_output_data as text\n",
    "with open(\"decoder_target_data.txt\", \"wb\") as fp:\n",
    "    pickle.dump(decoder_target_data, fp)\n",
    "    \n",
    "#Saving target_token_index as text\n",
    "with open(\"target_token_index.txt\", \"wb\") as fp:\n",
    "    pickle.dump(target_token_index, fp)\n",
    "    \n",
    "#Saving reverse_target_char_index as text\n",
    "with open(\"reverse_target_char_index.txt\", \"wb\") as fp:\n",
    "    pickle.dump(reverse_target_char_index, fp)\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading decoder_input_data as text\n",
    "with open(\"mfcc_data_shuffled.txt\", \"rb\") as fp:\n",
    "    mfcc_data_shuffled = pickle.load(fp)    \n",
    "\n",
    "\n",
    "#Loading decoder_input_data as text\n",
    "with open(\"decoder_input_data.txt\", \"rb\") as fp:\n",
    "    decoder_input_data = pickle.load(fp)    \n",
    "\n",
    "#Loading decoder_output_data as text\n",
    "with open(\"decoder_target_data.txt\", \"rb\") as fp:\n",
    "    decoder_target_data = pickle.load(fp)\n",
    "    \n",
    "#Loading decoder_output_data as text\n",
    "with open(\"target_token_index.txt\", \"rb\") as fp:\n",
    "    target_token_index = pickle.load(fp)\n",
    "    \n",
    "#Loading decoder_output_data as text\n",
    "with open(\"reverse_target_char_index.txt\", \"rb\") as fp:\n",
    "    reverse_target_char_index = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'mfcc_data_shuffled' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-32-e3c0b8d29d83>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#Transposing\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mmfcc_data_shuffled_transposed\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmfcc_data_shuffled\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m     \u001b[0mmfcc_data_shuffled_transposed\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmfcc_data_shuffled\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'mfcc_data_shuffled' is not defined"
     ]
    }
   ],
   "source": [
    "#Transposing \n",
    "mfcc_data_shuffled_transposed=[]\n",
    "for i in range(len(mfcc_data_shuffled)):\n",
    "    mfcc_data_shuffled_transposed.append(np.transpose(mfcc_data_shuffled[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "tc=[]\n",
    "for i in raw_aud_data_flat:\n",
    "    tc.append(i.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data=[]\n",
    "for i in range(len(raw_aud_data_flat_shuffle)):\n",
    "    all_data.append(raw_aud_data_flat_shuffle[i].tolist()[0])\n",
    "#    for j in range(len(a)):\n",
    "#        all_data.append(a[j])\n",
    "\n",
    "all_data_shuffled_padded = tf.keras.preprocessing.sequence.pad_sequences(all_data,\n",
    "                                                         padding='post')        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_aud_data_flat_shuffle_padded=[]\n",
    "for i in all_data_shuffled_padded:\n",
    "    raw_aud_data_flat_shuffle_padded.append(i.reshape(i.shape[0],1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18262, 1)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_aud_data_flat_shuffle_padded[0].shape    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "mfcc_data_shuffled_padded=[]\n",
    "for i in range(1500):\n",
    "    inter=all_data_shuffled_padded[13*i:13*(i+1)]\n",
    "    mfcc_data_shuffled_padded.append(np.array(inter))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([126, 126, 124, ...,   0,   0,   0])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data_shuffled_padded[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Transposing \n",
    "mfcc_data_shuffled_padded_transposed=[]\n",
    "for i in range(len(mfcc_data_shuffled_padded)):\n",
    "    mfcc_data_shuffled_padded_transposed.append(np.transpose(mfcc_data_shuffled_padded[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18262, 13)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mfcc_data_shuffled_padded_transposed[0].shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1500, 7, 17)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decoder_input_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_decoder_seq_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_inputs = tf.keras.Input(shape=(18262,1))\n",
    "conv1= Conv1D(filters=32, kernel_size = 4, padding='causal',activation='relu')(encoder_inputs)\n",
    "conv2= Conv1D(filters=64, kernel_size = 4, strides=2,padding='causal',activation='relu')(conv1)\n",
    "conv3= Conv1D(filters=128, kernel_size = 4, strides=2,padding='causal',activation='relu')(conv2)\n",
    "conv4= Conv1D(filters=256, kernel_size = 4, strides=2,padding='causal',activation='relu')(conv3)\n",
    "conv5= Conv1D(filters=512, kernel_size = 4, strides=3,padding='causal',activation='relu')(conv4)\n",
    "conv6= Conv1D(filters=1024, kernel_size = 5, strides=4,padding='causal',activation='relu')(conv5)\n",
    "conv7= Conv1D(filters=1024, kernel_size = 5, strides=5,padding='causal',activation='relu')(conv6)\n",
    "\n",
    "\n",
    "encoder = LSTM(256, return_state=True, return_sequences=True)\n",
    "encoder_outputs, state_h, state_c = encoder(conv7)\n",
    "encoder_states = [state_h, state_c]\n",
    " \n",
    "decoder_inputs = Input(shape=(max_decoder_seq_length, output_vocab_size))\n",
    "decoder_lstm = LSTM(256, return_sequences=True,\n",
    "                     return_state=True)\n",
    "decoder_outputs, _, _ = decoder_lstm(decoder_inputs,\n",
    "     initial_state=encoder_states)\n",
    "decoder_dense = Dense(output_vocab_size, activation='softmax')\n",
    "decoder_outputs = decoder_dense(decoder_outputs)\n",
    "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
    " \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_9 (InputLayer)            [(None, 18262, 1)]   0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_21 (Conv1D)              (None, 18262, 32)    160         input_9[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_22 (Conv1D)              (None, 9131, 64)     8256        conv1d_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_23 (Conv1D)              (None, 4566, 128)    32896       conv1d_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_24 (Conv1D)              (None, 2283, 256)    131328      conv1d_23[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_25 (Conv1D)              (None, 761, 512)     524800      conv1d_24[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_26 (Conv1D)              (None, 191, 1024)    2622464     conv1d_25[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_27 (Conv1D)              (None, 39, 1024)     5243904     conv1d_26[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "input_10 (InputLayer)           [(None, 7, 17)]      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lstm_4 (LSTM)                   [(None, 39, 256), (N 1311744     conv1d_27[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lstm_5 (LSTM)                   [(None, 7, 256), (No 280576      input_10[0][0]                   \n",
      "                                                                 lstm_4[0][1]                     \n",
      "                                                                 lstm_4[0][2]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 7, 17)        4369        lstm_5[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 10,160,497\n",
      "Trainable params: 10,160,497\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def correctness(y_true, y_pred):\n",
    "    vec_oh=[]\n",
    "    #a=y_pred\n",
    "\n",
    "    #vec_oh=[]\n",
    "    #a=train_y.copy()\n",
    "    if type(y_pred.shape[0]) != None:\n",
    "        for i in range(y_pred.shape[0]):\n",
    "            ppp=[]\n",
    "            for j in range(y_pred.shape[1]):\n",
    "\n",
    "                if y_pred[i][j].max(axis=0)!=0:\n",
    "                    ppp.append((y_pred[i][j] == y_pred[i][j].max(axis=0)).astype(int))\n",
    "                else:\n",
    "                    \n",
    "                    ppp.append((y_pred[i][j]).astype(int))\n",
    "            b=np.array(ppp)\n",
    "            vec_oh.append(b)\n",
    "            vec_oh_array=np.array(vec_oh)\n",
    "        \n",
    "        check=np.array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
    "        for i in range(vec_oh_array.shape[0]):\n",
    "            ind=[]\n",
    "            for j in range(vec_oh_array.shape[1]):\n",
    "                if np.array_equal(vec_oh_array[i][j],check)==True:\n",
    "                    ind.append(j)\n",
    "                    ind=sorted(ind)\n",
    "            for k in ind[1:]:\n",
    "                vec_oh_array[i][k][0]=0\n",
    "              \n",
    "        \n",
    "        \n",
    "        sentence_accuracy=[]\n",
    "        for i in range(vec_oh_array.shape[0]):\n",
    "                sentence_accuracy.append(np.array_equal(vec_oh_array[i],train_y[i]))\n",
    "        no_correct_sentence=sum(sentence_accuracy)\n",
    "        percn_correct_sentence=sum(sentence_accuracy)/len(sentence_accuracy)\n",
    "\n",
    "        character_accuracy=[]\n",
    "        for i in range(vec_oh_array.shape[0]):\n",
    "            for j in range(vec_oh_array.shape[1]):\n",
    "                character_accuracy.append(np.array_equal(vec_oh_array[i][j],train_y[i][j]))\n",
    "        no_correct_character=sum(character_accuracy)\n",
    "        percn_correct_character=sum(character_accuracy)/len(character_accuracy)\n",
    "        \n",
    "        return no_correct_sentence, percn_correct_sentence, no_correct_character,percn_correct_character\n",
    "        #return vec_oh_array\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.optimizers import Adam\n",
    "\n",
    "Optimizer = Adam(lr = .001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`period` argument is deprecated. Please use `save_freq` to specify the frequency in number of samples seen.\n",
      "WARNING:tensorflow:`period` argument is deprecated. Please use `save_freq` to specify the frequency in number of samples seen.\n"
     ]
    }
   ],
   "source": [
    "# filepath_best = \"weights\\weights.{epoch:02d}-{loss:.2f}-bigger_CNN-Raw.hdf5\" #this will save the model with best weights\n",
    "# checkpoint_best = ModelCheckpoint(filepath_best, monitor='loss', verbose=1,\n",
    "#     save_best_only=True, mode='auto', period=1)\n",
    "\n",
    "# filepath_latest = \"weights\\latest_weights_CNN-Raw.hdf5\" #this will save the model with latest weight updates (not necessarily the best weights)\n",
    "# checkpoint_latest = ModelCheckpoint(filepath_latest, monitor='loss', verbose=1,\n",
    "#     save_best_only=False, mode='auto', period=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Splitting into train and test sets-5% data as test set\n",
    "#Test sets\n",
    "test_size=len(raw_aud_data_flat_shuffle_padded)*.05\n",
    "\n",
    "raw_aud_data_flat_shuffle_padded_test=raw_aud_data_flat_shuffle_padded[0:round(test_size)]\n",
    "decoder_input_data_test=decoder_input_data[0:round(test_size)]\n",
    "decoder_target_data_test=decoder_target_data[0:round(test_size)]\n",
    "target_texts_test=target_texts[0:round(test_size)]\n",
    "\n",
    "#Training sets\n",
    "raw_aud_data_flat_shuffle_padded_train=raw_aud_data_flat_shuffle_padded[round(test_size):]\n",
    "decoder_input_data_train=decoder_input_data[round(test_size):]\n",
    "decoder_target_data_train=decoder_target_data[round(test_size):]\n",
    "target_texts_train=target_texts[round(test_size):]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving decoder_input_data as text\n",
    "\n",
    "with open(\"raw_aud_data_flat_shuffle_padded_test.txt\", \"wb\") as fp:\n",
    "    pickle.dump(raw_aud_data_flat_shuffle_padded_test, fp)\n",
    "\n",
    "with open(\"decoder_input_data_test.txt\", \"wb\") as fp:\n",
    "    pickle.dump(decoder_input_data_test, fp)\n",
    "    \n",
    "#Saving decoder_output_data as text\n",
    "with open(\"decoder_target_data_test.txt\", \"wb\") as fp:\n",
    "    pickle.dump(decoder_target_data_test, fp)\n",
    "    \n",
    "#Saving target_token_index as text\n",
    "with open(\"target_texts_test.txt\", \"wb\") as fp:\n",
    "    pickle.dump(target_texts_test, fp)\n",
    "    \n",
    "#Saving reverse_target_char_index as text\n",
    "with open(\"raw_aud_data_flat_shuffle_padded_train.txt\", \"wb\") as fp:\n",
    "    pickle.dump(raw_aud_data_flat_shuffle_padded_train, fp)\n",
    "    \n",
    "with open(\"decoder_input_data_train.txt\", \"wb\") as fp:\n",
    "    pickle.dump(decoder_input_data_train, fp)\n",
    "    \n",
    "with open(\"decoder_target_data_train.txt\", \"wb\") as fp:\n",
    "    pickle.dump(decoder_target_data_train, fp)\n",
    "    \n",
    "with open(\"target_texts_train.txt\", \"wb\") as fp:\n",
    "    pickle.dump(target_texts_train, fp)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading raw_aud_data_flat_shuffle_padded_test as text\n",
    "with open(\"raw_aud_data_flat_shuffle_padded_test.txt\", \"rb\") as fp:\n",
    "    raw_aud_data_flat_shuffle_padded_test = pickle.load(fp)    \n",
    "\n",
    "\n",
    "#Loading decoder_input_data_test as text\n",
    "with open(\"decoder_input_data_test.txt\", \"rb\") as fp:\n",
    "    decoder_input_data_test = pickle.load(fp)    \n",
    "\n",
    "#Loading decoder_target_data_test as text\n",
    "with open(\"decoder_target_data_test.txt\", \"rb\") as fp:\n",
    "    decoder_target_data_test = pickle.load(fp)\n",
    "    \n",
    "#Loading target_texts_test as text\n",
    "with open(\"target_texts_test.txt\", \"rb\") as fp:\n",
    "    target_texts_test = pickle.load(fp)\n",
    "    \n",
    "#Loading raw_aud_data_flat_shuffle_padded_train as text\n",
    "with open(\"raw_aud_data_flat_shuffle_padded_train.txt\", \"rb\") as fp:\n",
    "    raw_aud_data_flat_shuffle_padded_train = pickle.load(fp)\n",
    "    \n",
    "#Loading decoder_input_data_train as text\n",
    "with open(\"decoder_input_data_train.txt\", \"rb\") as fp:\n",
    "    decoder_input_data_train = pickle.load(fp)\n",
    "\n",
    "#Loading decoder_target_data_train as text\n",
    "with open(\"decoder_target_data_train.txt\", \"rb\") as fp:\n",
    "    decoder_target_data_train = pickle.load(fp)\n",
    "    \n",
    "#Loading target_texts_train as text\n",
    "with open(\"target_texts_train.txt\", \"rb\") as fp:\n",
    "    target_texts_train = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "#batch function\n",
    "def batch_custom(xdataset,size):\n",
    "    id=[]\n",
    "    for m in range(0,len(xdataset),size):\n",
    "        id.append(m)\n",
    "\n",
    "    sl=[]\n",
    "    for m in range(0,len(xdataset)):\n",
    "                   sl.append(m)\n",
    "    mybatch=[]\n",
    "    for k in range(len(id)-1):\n",
    "        mybatch.append(sl[id[k]:id[k+1]])\n",
    "        \n",
    "    return mybatch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#batch of 32\n",
    "batch_ids=batch_custom(raw_aud_data_flat_shuffle_padded_train,32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 10ms/sample - loss: 2.0518 - accuracy: 0.2143\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.8743 - accuracy: 0.1429\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.7868 - accuracy: 0.1518\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.7339 - accuracy: 0.1830\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.7818 - accuracy: 0.1562\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.5993 - accuracy: 0.2232\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 9ms/sample - loss: 1.6449 - accuracy: 0.1875\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 9ms/sample - loss: 1.6462 - accuracy: 0.2054\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.5425 - accuracy: 0.1920\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.5312 - accuracy: 0.1920\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.5413 - accuracy: 0.2277\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.4676 - accuracy: 0.2143\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.5611 - accuracy: 0.1964\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.4489 - accuracy: 0.2411\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.5395 - accuracy: 0.2366\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.5683 - accuracy: 0.2277\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 9ms/sample - loss: 1.4957 - accuracy: 0.2545\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 9ms/sample - loss: 1.4965 - accuracy: 0.2411\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 9ms/sample - loss: 1.4654 - accuracy: 0.2232\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.3259 - accuracy: 0.2589\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.5580 - accuracy: 0.1696\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.4596 - accuracy: 0.2455\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 8ms/sample - loss: 1.4015 - accuracy: 0.2589\n",
      "Train on 32 samples\n",
      "32/32 [==============================] - 0s 9ms/sample - loss: 1.4229 - accuracy: 0.2946\n",
      "Train on 32 samples\n",
      " 0/32 [..............................] - ETA: 0s"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-66-ea1510254cba>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      7\u001b[0m         \u001b[0mdecoder_output_batch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdecoder_target_data_train\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[1;31m#         history=model.fit([batch_input, decoder_input_batch], decoder_output_batch, batch_size=32,epochs=1, callbacks=[checkpoint_best,checkpoint_latest])\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m         \u001b[0mhistory\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mbatch_input\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdecoder_input_batch\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdecoder_output_batch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m32\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[0;32m    817\u001b[0m         \u001b[0mmax_queue_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmax_queue_size\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    818\u001b[0m         \u001b[0mworkers\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mworkers\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 819\u001b[1;33m         use_multiprocessing=use_multiprocessing)\n\u001b[0m\u001b[0;32m    820\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    821\u001b[0m   def evaluate(self,\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training_v2.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, model, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[0;32m    327\u001b[0m                 \u001b[0mtraining_data_iter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_initializer\u001b[0m  \u001b[1;31m# pylint: disable=pointless-statement\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    328\u001b[0m               \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 329\u001b[1;33m                 \u001b[0mtraining_data_iter\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0miter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtraining_dataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    330\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    331\u001b[0m             training_result = run_one_epoch(\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\data\\ops\\dataset_ops.py\u001b[0m in \u001b[0;36m__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    416\u001b[0m     if (context.executing_eagerly()\n\u001b[0;32m    417\u001b[0m         or ops.get_default_graph()._building_function):  # pylint: disable=protected-access\n\u001b[1;32m--> 418\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0miterator_ops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOwnedIterator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    419\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    420\u001b[0m       raise RuntimeError(\"__iter__() is only supported inside of tf.function \"\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\data\\ops\\iterator_ops.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, dataset, components, element_spec)\u001b[0m\n\u001b[0;32m    592\u001b[0m           context.context().device_spec.device_type != \"CPU\"):\n\u001b[0;32m    593\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"/cpu:0\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 594\u001b[1;33m           \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_create_iterator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    595\u001b[0m       \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    596\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_create_iterator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\data\\ops\\iterator_ops.py\u001b[0m in \u001b[0;36m_create_iterator\u001b[1;34m(self, dataset)\u001b[0m\n\u001b[0;32m    617\u001b[0m               \u001b[0moutput_types\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_flat_output_types\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    618\u001b[0m               output_shapes=self._flat_output_shapes))\n\u001b[1;32m--> 619\u001b[1;33m       \u001b[0mgen_dataset_ops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmake_iterator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mds_variant\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_iterator_resource\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    620\u001b[0m       \u001b[1;31m# Delete the resource when this object is deleted\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    621\u001b[0m       self._resource_deleter = IteratorResourceDeleter(\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow_core\\python\\ops\\gen_dataset_ops.py\u001b[0m in \u001b[0;36mmake_iterator\u001b[1;34m(dataset, iterator, name)\u001b[0m\n\u001b[0;32m   2694\u001b[0m       _result = _pywrap_tensorflow.TFE_Py_FastPathExecute(\n\u001b[0;32m   2695\u001b[0m         \u001b[0m_ctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_context_handle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtld\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"MakeIterator\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2696\u001b[1;33m         tld.op_callbacks, dataset, iterator)\n\u001b[0m\u001b[0;32m   2697\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0m_result\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2698\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0m_core\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_FallbackException\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for i in range(1000):\n",
    "    for k in batch_ids:            \n",
    "        batch_input=[raw_aud_data_flat_shuffle_padded_train[i] for i in k]\n",
    "        batch_input = np.array(batch_input)\n",
    "        batch_input = batch_input/np.max(batch_input)\n",
    "        decoder_input_batch=decoder_input_data_train[k[0]:k[-1]+1]        \n",
    "        decoder_output_batch=decoder_target_data_train[k[0]:k[-1]+1]        \n",
    "#         history=model.fit([batch_input, decoder_input_batch], decoder_output_batch, batch_size=32,epochs=1, callbacks=[checkpoint_best,checkpoint_latest])\n",
    "        history=model.fit([batch_input, decoder_input_batch], decoder_output_batch, batch_size=32,epochs=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# from tensorflow.keras.models import Sequential, load_model \n",
    "\n",
    "# model = load_model(\"weights\\latest_weights_CNN-Raw.hdf5\",compile=False) \n",
    "                   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#######################################Prediction experiment#######################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\maruf\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:10: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=[<tf.Tenso..., outputs=[<tf.Tenso...)`\n",
      "  # Remove the CWD from sys.path while we load stuff.\n"
     ]
    }
   ],
   "source": [
    "thought_input = [Input(shape=(256,)),\n",
    "     Input(shape=(256,))]                       \n",
    "decoder_outputs, state_h, state_c = decoder_lstm(\n",
    "     decoder_inputs, initial_state=thought_input)       \n",
    "decoder_states = [state_h, state_c]                    \n",
    "decoder_outputs = decoder_dense(decoder_outputs)       \n",
    " \n",
    "decoder_model = Model(                                 \n",
    "     inputs=[decoder_inputs] + thought_input,           \n",
    "     output=[decoder_outputs] + decoder_states)         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_model = Model(inputs=encoder_inputs, outputs=encoder_states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "thought_train = encoder_model.predict(train_x)   \n",
    "thought_test = encoder_model.predict(test_x)             \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_token_train, h, c = decoder_model.predict(\n",
    "         [train_y] + thought_train) \n",
    "output_token_test, h, c = decoder_model.predict(\n",
    "         [test_y] + thought_test) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving output_token_train as text\n",
    "with open(\"thought_train_cnn.txt\", \"wb\") as fp:\n",
    "    pickle.dump(thought_train, fp)\n",
    "\n",
    "with open(\"thought_test_cnn.txt\", \"wb\") as fp:\n",
    "    pickle.dump(thought_test, fp)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving output_token_train as text\n",
    "with open(\"output_token_train_cnn.txt\", \"wb\") as fp:\n",
    "    pickle.dump(output_token_train, fp)\n",
    "\n",
    "with open(\"output_token_test_cnn.txt\", \"wb\") as fp:\n",
    "    pickle.dump(output_token_test, fp)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading decoder_output_data as text\n",
    "with open(\"thought_train_cnn.txt\", \"rb\") as fp:\n",
    "    thought_train = pickle.load(fp)\n",
    "\n",
    "#Loading decoder_output_data as text\n",
    "with open(\"thought_test_cnn.txt\", \"rb\") as fp:\n",
    "    thought_test = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading decoder_output_data as text\n",
    "with open(\"output_token_test_cnn.txt\", \"rb\") as fp:\n",
    "    output_token_test = pickle.load(fp)\n",
    "\n",
    "#Loading decoder_output_data as text\n",
    "with open(\"output_token_train_cnn.txt\", \"rb\") as fp:\n",
    "    output_token_train = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1425, 18262, 1)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x=np.array(raw_aud_data_flat_shuffle_padded_train)\n",
    "train_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75, 18262, 1)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_x=np.array(raw_aud_data_flat_shuffle_padded_test)\n",
    "test_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1425, 7, 17)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_y=decoder_target_data_train\n",
    "train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75, 7, 17)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_y=decoder_target_data_test\n",
    "test_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentence_prediction(output_tokens, actual_label_vector):\n",
    "    \n",
    "    predicted_sentence=[]\n",
    "    actual_sentence=[]\n",
    "    for i in range(output_tokens.shape[0]):\n",
    "        generated_token_idx = np.argmax(output_tokens[i], axis=1)\n",
    "        g=generated_token_idx.tolist()\n",
    "        char=[]\n",
    "        for i in g:\n",
    "            char.append(reverse_target_char_index[i])\n",
    "        predicted_sentence.append(\"\".join(char))\n",
    "    \n",
    "    for i in range(actual_label_vector.shape[0]):\n",
    "        generated_token_idx_actual=np.argmax(actual_label_vector[i], axis=1)\n",
    "        ga=generated_token_idx_actual.tolist()\n",
    "        char_label=[]\n",
    "        for i in ga:\n",
    "            char_label.append(reverse_target_char_index[i])\n",
    "        actual_sentence.append(\"\".join(char_label))\n",
    "        #char_label.append(reverse_target_char_index[i])\n",
    "        #actual_sentence.append(\"\".join(char_label))\n",
    "    return actual_sentence,predicted_sentence\n",
    "\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "iteration_train=correctness(train_y, output_token_train)\n",
    "iteration_test=correctness(test_y, output_token_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "#words with tokens\n",
    "match_pred_act_train=sentence_prediction(output_token_train,train_y)\n",
    "match_pred_act_test=sentence_prediction(output_token_test,test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 0.0, 3121, 0.3128822055137845)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iteration_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 0.0, 167, 0.3180952380952381)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iteration_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#words without tokens\n",
    "actualword_train_word=actual_word(match_pred_act_train[0])\n",
    "predictedword__train_word=actual_word(match_pred_act_train[1])\n",
    "\n",
    "actualword_test_word=actual_word(match_pred_act_test[0])\n",
    "predictedword__test_word=actual_word(match_pred_act_test[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving actualword_train_word as text\n",
    "with open(\"actualword_train_word.txt\", \"wb\") as fp:\n",
    "    pickle.dump(actualword_train_word, fp)\n",
    "\n",
    "with open(\"predictedword__train_word.txt\", \"wb\") as fp:\n",
    "    pickle.dump(predictedword__train_word, fp)\n",
    "\n",
    "with open(\"actualword_test_word.txt\", \"wb\") as fp:\n",
    "    pickle.dump(actualword_test_word, fp)\n",
    "\n",
    "with open(\"predictedword__test_word.txt\", \"wb\") as fp:\n",
    "    pickle.dump(predictedword__test_word, fp)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving actualword_train_word as text\n",
    "with open(\"actualword_train_word.txt\", \"rb\") as fp:\n",
    "    actualword_train_word = pickle.load(fp)\n",
    "with open(\"predictedword__train_word.txt\", \"rb\") as fp:\n",
    "    predictedword__train_word = pickle.load(fp)\n",
    "with open(\"actualword_test_word.txt\", \"rb\") as fp:\n",
    "    actualword_test_word = pickle.load(fp)\n",
    "with open(\"predictedword__test_word.txt\", \"rb\") as fp:\n",
    "    predictedword__test_word = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_word_comparison = pd.DataFrame((actualword_train_word,predictedword__train_word), index=[\"actual\",\"predicted\"]).T\n",
    "test_word_comparison = pd.DataFrame((actualword_test_word,predictedword__test_word), index=[\"actual\",\"predicted\"]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>actual</th>\n",
       "      <th>predicted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>three</td>\n",
       "      <td>sree</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>nine</td>\n",
       "      <td>sne</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>two</td>\n",
       "      <td>so</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>five</td>\n",
       "      <td>ove</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>three</td>\n",
       "      <td>sree</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>seven</td>\n",
       "      <td>sven</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>one</td>\n",
       "      <td>nine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>six</td>\n",
       "      <td>sx</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>one</td>\n",
       "      <td>ni</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>one</td>\n",
       "      <td>nine</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>75 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   actual predicted\n",
       "0   three      sree\n",
       "1    nine       sne\n",
       "2     two        so\n",
       "3    five       ove\n",
       "4   three      sree\n",
       "..    ...       ...\n",
       "70  seven      sven\n",
       "71    one      nine\n",
       "72    six        sx\n",
       "73    one        ni\n",
       "74    one      nine\n",
       "\n",
       "[75 rows x 2 columns]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_word_comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_word_comparison.to_csv('train_word_comparison.csv')\n",
    "test_word_comparison.to_csv('test_word_comparison.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_accuracy_wordlevel(predword,actword):\n",
    "    val=[]\n",
    "    for i in range(len(actword)):\n",
    "        if actword[i]==predword[i]:\n",
    "            val.append(1)\n",
    "        else:\n",
    "            val.append(0)\n",
    "    return sum(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_accuracy_charevel(predword,actword):\n",
    "    val=[]\n",
    "    for i in range(len(actword)):\n",
    "        for j in range(min(len(actword[i]),len(predword[i]))):\n",
    "            if actword[i][j]==predword[i][j]:\n",
    "                val.append(1)\n",
    "            else:\n",
    "                val.append(0)\n",
    "    return sum(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_accuracy_wordlevel(predictedword__train_word,actualword_train_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_accuracy_wordlevel(predictedword__test_word,actualword_test_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "343"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_accuracy_charevel(predictedword__train_word,actualword_train_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_accuracy_charevel(predictedword__test_word,actualword_test_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "def character_counts(train_words):\n",
    "    total_characters=[]\n",
    "    for i in range(len(train_words)):\n",
    "        total_characters.append(len(train_words[i]))\n",
    "    return sum(total_characters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5702"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "character_counts(actualword_train_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "298"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "character_counts(actualword_test_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.060154331813398805"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "343/5702"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "#words without tokens\n",
    "actualword_test_word=actual_word(match_pred_act_test[0])\n",
    "predictedword__test_word=actual_word(match_pred_act_test[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}