MyShake_CNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import json\n",
    "import tsfresh\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.style.use('ggplot')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Directory structure\n",
    "\n",
    "MyShake_Trainig_Data  \n",
    "|-- EQ  \n",
    "----|-- shake_table (193 files)  \n",
    "----|-- simulated (993 files)  \n",
    "|-- Human (26344 files)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "table_files = glob.glob('./MyShake_Training_Data/EQ/shake_table/*')\n",
    "simulated_files = glob.glob('./MyShake_Training_Data/EQ/simulated/*')\n",
    "human_files = glob.glob('./MyShake_Training_Data/Human/*')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating DataFrames for one data file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def all_data(file):\n",
    "    'Get dictionary from JSON file'\n",
    "    with open(file, encoding='utf-8') as f:\n",
    "        data=[json.loads(line) for line in f]\n",
    "        data=data[0]\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_file = 'MyShake_Training_Data/Human/013306004148017_1415046600.json'\n",
    "test_data = all_data(test_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sampling_rate': 25.0,\n",
       " 'starttime': 1415046600,\n",
       " 'station': '013306004148017',\n",
       " 'stla': 12345,\n",
       " 'stlo': 12345,\n",
       " 'triggertime': 1415046662.432}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data['header']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_df_1 = pd.DataFrame(test_data['data'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "      <th>t</th>\n",
       "      <th>delta_t</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>7403.000000</td>\n",
       "      <td>7403.000000</td>\n",
       "      <td>7403.000000</td>\n",
       "      <td>7403.000000</td>\n",
       "      <td>7403.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-0.024189</td>\n",
       "      <td>0.031678</td>\n",
       "      <td>-1.001861</td>\n",
       "      <td>148.039859</td>\n",
       "      <td>148.039859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.003427</td>\n",
       "      <td>0.003656</td>\n",
       "      <td>0.003760</td>\n",
       "      <td>85.488173</td>\n",
       "      <td>85.488173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-0.057600</td>\n",
       "      <td>0.007281</td>\n",
       "      <td>-1.014853</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.027000</td>\n",
       "      <td>0.029400</td>\n",
       "      <td>-1.004829</td>\n",
       "      <td>74.019929</td>\n",
       "      <td>74.019929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-0.023780</td>\n",
       "      <td>0.031000</td>\n",
       "      <td>-1.002000</td>\n",
       "      <td>148.039859</td>\n",
       "      <td>148.039859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>-0.022610</td>\n",
       "      <td>0.034415</td>\n",
       "      <td>-0.998976</td>\n",
       "      <td>222.059788</td>\n",
       "      <td>222.059788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.011400</td>\n",
       "      <td>0.064488</td>\n",
       "      <td>-0.983454</td>\n",
       "      <td>296.079718</td>\n",
       "      <td>296.079718</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 x            y            z            t      delta_t\n",
       "count  7403.000000  7403.000000  7403.000000  7403.000000  7403.000000\n",
       "mean     -0.024189     0.031678    -1.001861   148.039859   148.039859\n",
       "std       0.003427     0.003656     0.003760    85.488173    85.488173\n",
       "min      -0.057600     0.007281    -1.014853     0.000000     0.000000\n",
       "25%      -0.027000     0.029400    -1.004829    74.019929    74.019929\n",
       "50%      -0.023780     0.031000    -1.002000   148.039859   148.039859\n",
       "75%      -0.022610     0.034415    -0.998976   222.059788   222.059788\n",
       "max       0.011400     0.064488    -0.983454   296.079718   296.079718"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df_1.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_time(data):\n",
    "    t0 = data['header']['starttime']\n",
    "    npoints = min(len(data['data']['x']), len(data['data']['y']), len(data['data']['z'])) \n",
    "    sampling_rate = data['header']['sampling_rate']\n",
    "    t1 = t0 + npoints / sampling_rate\n",
    "    # form the timestamp\n",
    "    t = np.arange(t0, t1, 1/sampling_rate)\n",
    "    return t0, t[:npoints]\n",
    "\n",
    "def get_df(file):\n",
    "    data = all_data(file)\n",
    "    df = pd.DataFrame(data['data'])\n",
    "    t0, t = get_time(data)\n",
    "    df['t'] = t\n",
    "    df['delta_t'] = t-t0\n",
    "    df['filename'] = file[len(dir_name):]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating DataFrame for entire directory\n",
    "\n",
    "## Human"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# length of file\n",
    "\n",
    "test_data = all_data(test_file)\n",
    "test_df_1 = pd.DataFrame(test_data['data'])\n",
    "\n",
    "t0 = test_data['header']['starttime']\n",
    "npoints = max(len(test_data['data']['x']), len(test_data['data']['y']), len(test_data['data']['z'])) \n",
    "sampling_rate = test_data['header']['sampling_rate']\n",
    "t1 = t0 + npoints / sampling_rate\n",
    "# form the timestamp\n",
    "t = np.arange(t0, t1, 1/sampling_rate)\n",
    "\n",
    "test_df_1['delta_t'] = t \n",
    "test_df_1['delta_t'] = t - t0\n",
    "\n",
    "test_df_1['filename'] = test_file[28:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def windows(data, window_size):\n",
    "    start = 0\n",
    "    while start < len(data):\n",
    "        yield start, start + window_size\n",
    "        start += (window_size / 2)\n",
    "\n",
    "def extract_features(parent_dir,sub_dirs,file_ext=\"*.wav\",bands = 60, frames = 41):\n",
    "    window_size = 512 * (frames - 1)\n",
    "    log_specgrams = []\n",
    "    labels = []\n",
    "    for l, sub_dir in enumerate(sub_dirs):\n",
    "        for fn in glob.glob(os.path.join(parent_dir, sub_dir, file_ext)):\n",
    "            sound_clip,s = librosa.load(fn)\n",
    "            label = fn.split('/')[2].split('-')[1]\n",
    "            for (start,end) in windows(sound_clip,window_size):\n",
    "                if(len(sound_clip[start:end]) == window_size):\n",
    "                    signal = sound_clip[start:end]\n",
    "                    melspec = librosa.feature.melspectrogram(signal, n_mels = bands)\n",
    "                    logspec = librosa.logamplitude(melspec)\n",
    "                    logspec = logspec.T.flatten()[:, np.newaxis].T\n",
    "                    log_specgrams.append(logspec)\n",
    "                    labels.append(label)\n",
    "            \n",
    "    log_specgrams = np.asarray(log_specgrams).reshape(len(log_specgrams),bands,frames,1)\n",
    "    features = np.concatenate((log_specgrams, np.zeros(np.shape(log_specgrams))), axis = 3)\n",
    "    for i in range(len(features)):\n",
    "        features[i, :, :, 1] = librosa.feature.delta(features[i, :, :, 0])\n",
    "    \n",
    "    return np.array(features), np.array(labels,dtype = np.int)\n",
    "\n",
    "def one_hot_encode(labels):\n",
    "    n_labels = len(labels)\n",
    "    n_unique_labels = len(np.unique(labels))\n",
    "    one_hot_encode = np.zeros((n_labels,n_unique_labels))\n",
    "    one_hot_encode[np.arange(n_labels), labels] = 1\n",
    "    return one_hot_encode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features,labels = extract_features(parent_dir,sub_dirs)\n",
    "labels = one_hot_encode(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weight_variable(shape):\n",
    "    initial = tf.truncated_normal(shape, stddev = 0.1)\n",
    "    return tf.Variable(initial)\n",
    "\n",
    "def bias_variable(shape):\n",
    "    initial = tf.constant(1.0, shape = shape)\n",
    "    return tf.Variable(initial)\n",
    "\n",
    "def conv2d(x, W):\n",
    "    return tf.nn.conv2d(x,W,strides=[1,2,2,1], padding='SAME')\n",
    "\n",
    "def apply_convolution(x,kernel_size,num_channels,depth):\n",
    "    weights = weight_variable([kernel_size, kernel_size, num_channels, depth])\n",
    "    biases = bias_variable([depth])\n",
    "    return tf.nn.relu(tf.add(conv2d(x, weights),biases))\n",
    "\n",
    "def apply_max_pool(x,kernel_size,stride_size):\n",
    "    return tf.nn.max_pool(x, ksize=[1, kernel_size, kernel_size, 1], \n",
    "                          strides=[1, stride_size, stride_size, 1], padding='SAME')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rnd_indices = np.random.rand(len(labels)) < 0.70\n",
    "\n",
    "train_x = features[rnd_indices]\n",
    "train_y = labels[rnd_indices]\n",
    "test_x = features[~rnd_indices]\n",
    "test_y = labels[~rnd_indices]\n",
    "\n",
    "frames = 41\n",
    "bands = 60\n",
    "\n",
    "feature_size = 2460 #60x41\n",
    "num_labels = 10\n",
    "num_channels = 2\n",
    "\n",
    "batch_size = 50\n",
    "kernel_size = 30\n",
    "depth = 20\n",
    "num_hidden = 200\n",
    "\n",
    "learning_rate = 0.01\n",
    "training_iterations = 2000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float32, shape=[None,bands,frames,num_channels])\n",
    "Y = tf.placeholder(tf.float32, shape=[None,num_labels])\n",
    "\n",
    "cov = apply_convolution(X,kernel_size,num_channels,depth)\n",
    "\n",
    "shape = cov.get_shape().as_list()\n",
    "cov_flat = tf.reshape(cov, [-1, shape[1] * shape[2] * shape[3]])\n",
    "\n",
    "f_weights = weight_variable([shape[1] * shape[2] * depth, num_hidden])\n",
    "f_biases = bias_variable([num_hidden])\n",
    "f = tf.nn.sigmoid(tf.add(tf.matmul(cov_flat, f_weights),f_biases))\n",
    "\n",
    "out_weights = weight_variable([num_hidden, num_labels])\n",
    "out_biases = bias_variable([num_labels])\n",
    "y_ = tf.nn.softmax(tf.matmul(f, out_weights) + out_biases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cross_entropy = -tf.reduce_sum(Y * tf.log(y_))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cross_entropy)\n",
    "correct_prediction = tf.equal(tf.argmax(y_,1), tf.argmax(Y,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cost_history = np.empty(shape=[1],dtype=float)\n",
    "with tf.Session() as session:\n",
    "    tf.initialize_all_variables().run()\n",
    "\n",
    "    for itr in range(training_iterations):    \n",
    "        offset = (itr * batch_size) % (train_y.shape[0] - batch_size)\n",
    "        batch_x = train_x[offset:(offset + batch_size), :, :, :]\n",
    "        batch_y = train_y[offset:(offset + batch_size), :]\n",
    "        \n",
    "        _, c = session.run([optimizer, cross_entropy],feed_dict={X: batch_x, Y : batch_y})\n",
    "        cost_history = np.append(cost_history,c)\n",
    "    \n",
    "    print('Test accuracy: ',round(session.run(accuracy, feed_dict={X: test_x, Y: test_y}) , 3))\n",
    "    fig = plt.figure(figsize=(15,10))\n",
    "    plt.plot(cost_history)\n",
    "    plt.axis([0,training_epochs,0,np.max(cost_history)])\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}