jithin pradeep Cognitive Research Scientist | AI and Mixed reality Enthusiast

LSTM Neural Network for Human Activity Recognition (HAR)

The data

Data used in this project is avaliable at Wireless Sensor Data Mining (WISDM) Lab. Data collected from accelerometer sensors in controlled, laboratory setting Most of the modern smartphone has a tri-axial accelerometer that measures acceleration in all three spatial dimensions. Additionally, accelerometers can detect device orientation.

Dataset contains 1,098,207 rows and 6 columns. There are 6 activities that we’ll try to recognize: Walking, Jogging, Upstairs, Downstairs, Sitting, Standing.

import pandas as pd
import numpy as np
import pickle
import matplotlib.pyplot as plt
from scipy import stats
import tensorflow as tf
import seaborn as sns
from pylab import rcParams
from sklearn import metrics
from sklearn.model_selection import train_test_split

%matplotlib inline

sns.set(style='whitegrid', palette='muted', font_scale=1.5)

rcParams['figure.figsize'] = 14, 8

RANDOM_SEED = 42
C:\Users\Jithin\Anaconda3\lib\site-packages\h5py\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters
#Loading the dataset into dataframe 
columns = ['user','activity','timestamp', 'x-axis', 'y-axis', 'z-axis']
df = pd.read_csv('data/WISDM_ar_v1.1_raw.txt', header = None, names = columns)
df = df.dropna()
# verfiying the data laod
df.head()
useractivitytimestampx-axisy-axisz-axis
033Jogging49105962326000-0.69463812.6805440.503953
133Jogging491060622710005.01228811.2640280.953424
233Jogging491061121670004.90332510.882658-0.081722
333Jogging49106222305000-0.61291618.4964313.023717
433Jogging49106332290000-1.18497012.1084897.205164
df.info() # always helpful way of undersatnd the datatype and count for individual feature
<class 'pandas.core.frame.DataFrame'>
Int64Index: 1098203 entries, 0 to 1098203
Data columns (total 6 columns):
user         1098203 non-null int64
activity     1098203 non-null object
timestamp    1098203 non-null int64
x-axis       1098203 non-null float64
y-axis       1098203 non-null float64
z-axis       1098203 non-null float64
dtypes: float64(3), int64(2), object(1)
memory usage: 58.7+ MB

Exploration

df['activity'].value_counts().plot(kind='bar', title='Training examples by activity type');

png

df['user'].value_counts().plot(kind='bar', title='Training examples by user');

png

#function to plot the activity along x,y, z axis 
def plot_activity(activity, df):
    data = df[df['activity'] == activity][['x-axis', 'y-axis', 'z-axis']][:200]
    axis = data.plot(subplots=True, figsize=(16, 12), 
                     title=activity)
    for ax in axis:
        ax.legend(loc='lower left', bbox_to_anchor=(1.0, 0.5))
plot_activity("Sitting", df)

png

plot_activity("Standing", df)

png

plot_activity("Walking", df)

png

plot_activity("Jogging", df)

png

Data preprocessing

LSTM model expects fixed-length sequences as training data. Hence I would be generating sequence that contains 200 training examples:

N_TIME_STEPS = 200
N_FEATURES = 3
step = 20
segments = []
labels = []
for i in range(0, len(df) - N_TIME_STEPS, step):
    xs = df['x-axis'].values[i: i + N_TIME_STEPS]
    ys = df['y-axis'].values[i: i + N_TIME_STEPS]
    zs = df['z-axis'].values[i: i + N_TIME_STEPS]
    label = stats.mode(df['activity'][i: i + N_TIME_STEPS])[0][0]
    segments.append([xs, ys, zs])
    labels.append(label)
C:\Anaconda3\lib\site-packages\scipy\stats\stats.py:253: RuntimeWarning: The input array could not be properly checked for nan values. nan values will be ignored.
  "values. nan values will be ignored.", RuntimeWarning)

Need to check more this warning, I dont feel good about them.

#segment shape
np.array(segments).shape

(54901, 3, 200)

#Transform the tensor shape into sequences of 200 rows, each containing x, y and z and apply one-hot encoding to labels.
reshaped_segments = np.asarray(segments, dtype= np.float32).reshape(-1, N_TIME_STEPS, N_FEATURES)
labels = np.asarray(pd.get_dummies(labels), dtype = np.float32)
reshaped_segments.shape

(54901, 200, 3)

labels[0]

array([0., 1., 0., 0., 0., 0.], dtype=float32)

#split the data into training and test (20%) set
X_train, X_test, y_train, y_test = train_test_split(
        reshaped_segments, labels, test_size=0.2, random_state=RANDOM_SEED)
len(X_train)

43920

len(X_test)

10981

Building the model

Model contains 2 fully-connected and 2 stacked LSTM layers with 64 hidden units.

N_CLASSES = 6
N_HIDDEN_UNITS = 64 # Number of hidden unit
def create_LSTM_model(inputs):
    W = {
        'hidden': tf.Variable(tf.random_normal([N_FEATURES, N_HIDDEN_UNITS])),
        'output': tf.Variable(tf.random_normal([N_HIDDEN_UNITS, N_CLASSES]))
    }
    biases = {
        'hidden': tf.Variable(tf.random_normal([N_HIDDEN_UNITS], mean=1.0)),
        'output': tf.Variable(tf.random_normal([N_CLASSES]))
    }
    
    X = tf.transpose(inputs, [1, 0, 2])
    X = tf.reshape(X, [-1, N_FEATURES])
    hidden = tf.nn.relu(tf.matmul(X, W['hidden']) + biases['hidden'])
    hidden = tf.split(hidden, N_TIME_STEPS, 0)

    # 2 stacked LSTM layers with 64 hidden units
    lstm_layers = [tf.contrib.rnn.BasicLSTMCell(N_HIDDEN_UNITS, forget_bias=1.0) for _ in range(2)]
    lstm_layers = tf.contrib.rnn.MultiRNNCell(lstm_layers)

    outputs, _ = tf.contrib.rnn.static_rnn(lstm_layers, hidden, dtype=tf.float32)

    # Get output for the last time step
    lstm_last_output = outputs[-1]

    return tf.matmul(lstm_last_output, W['output']) + biases['output']
tf.reset_default_graph()
#placeholder
X = tf.placeholder(tf.float32, [None, N_TIME_STEPS, N_FEATURES], name="input")
Y = tf.placeholder(tf.float32, [None, N_CLASSES])
pred_Y = create_LSTM_model(X)

pred_softmax = tf.nn.softmax(pred_Y, name="y_")
L2_LOSS = 0.0015
#L2 regularization
l2 = L2_LOSS * \
    sum(tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables())

loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits = pred_Y, labels = Y)) + l2
LEARNING_RATE = 0.0025
#Adam Optimizer
optimizer = tf.train.AdamOptimizer(learning_rate=LEARNING_RATE).minimize(loss)

correct_pred = tf.equal(tf.argmax(pred_softmax, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, dtype=tf.float32))

Training

N_EPOCHS = 50 #number of epcohs
BATCH_SIZE = 1024 #batch size
saver = tf.train.Saver()

history = dict(train_loss=[], 
                     train_acc=[], 
                     test_loss=[], 
                     test_acc=[])

sess=tf.InteractiveSession()
sess.run(tf.global_variables_initializer())

train_count = len(X_train)
print("Starting the traning ...")
for i in range(1, N_EPOCHS + 1):
    for start, end in zip(range(0, train_count, BATCH_SIZE),
                          range(BATCH_SIZE, train_count + 1,BATCH_SIZE)):
        sess.run(optimizer, feed_dict={X: X_train[start:end],
                                       Y: y_train[start:end]})

    _, acc_train, loss_train = sess.run([pred_softmax, accuracy, loss], feed_dict={
                                            X: X_train, Y: y_train})

    _, acc_test, loss_test = sess.run([pred_softmax, accuracy, loss], feed_dict={
                                            X: X_test, Y: y_test})

    history['train_loss'].append(loss_train)
    history['train_acc'].append(acc_train)
    history['test_loss'].append(loss_test)
    history['test_acc'].append(acc_test)

    if i != 1 and i % 2 != 0:
        continue

    print("epoch: {0} test accuracy: {1} loss: {2}".format(i,acc_test,loss_test))
    
predictions, acc_final, loss_final = sess.run([pred_softmax, accuracy, loss], feed_dict={X: X_test, Y: y_test})

print()
print("final results: accuracy: {0} loss: {1}".format(acc_final,loss_final))
Starting the traning ...
epoch: 1 test accuracy: 0.7552135586738586 loss: 1.3308054208755493
epoch: 2 test accuracy: 0.7981058359146118 loss: 1.2028048038482666
epoch: 4 test accuracy: 0.8325288891792297 loss: 1.0188990831375122
epoch: 6 test accuracy: 0.8737819790840149 loss: 0.875507652759552
epoch: 8 test accuracy: 0.908022940158844 loss: 0.7352402806282043
epoch: 10 test accuracy: 0.9130315780639648 loss: 0.684313178062439
epoch: 12 test accuracy: 0.9222292900085449 loss: 0.6207588315010071
epoch: 14 test accuracy: 0.9317002296447754 loss: 0.5680288672447205
epoch: 16 test accuracy: 0.9276933073997498 loss: 0.5508295893669128
epoch: 18 test accuracy: 0.9412621855735779 loss: 0.5007281303405762
epoch: 20 test accuracy: 0.9473636150360107 loss: 0.4725728929042816
epoch: 22 test accuracy: 0.952645480632782 loss: 0.4364386200904846
epoch: 24 test accuracy: 0.9512794613838196 loss: 0.4233739674091339
epoch: 26 test accuracy: 0.9576541185379028 loss: 0.404043585062027
epoch: 28 test accuracy: 0.9569255709648132 loss: 0.38592666387557983
epoch: 30 test accuracy: 0.9495491981506348 loss: 0.3906399607658386
epoch: 32 test accuracy: 0.9667607545852661 loss: 0.3368307948112488
epoch: 34 test accuracy: 0.9620253443717957 loss: 0.33798205852508545
epoch: 36 test accuracy: 0.9652126431465149 loss: 0.32209891080856323
epoch: 38 test accuracy: 0.9697659611701965 loss: 0.30286887288093567
epoch: 40 test accuracy: 0.9633002281188965 loss: 0.31487536430358887
epoch: 42 test accuracy: 0.9634823799133301 loss: 0.3000517189502716
epoch: 44 test accuracy: 0.9680356979370117 loss: 0.2813316881656647
epoch: 46 test accuracy: 0.9652126431465149 loss: 0.2874176502227783
epoch: 48 test accuracy: 0.9694017171859741 loss: 0.27466732263565063
epoch: 50 test accuracy: 0.9707676768302917 loss: 0.2629845440387726

final results: accuracy: 0.9707676768302917 loss: 0.2629845440387726

Model Accuracy : 0.9707676768302917 Loss: 0.2629845440387726 Nice!!!

#Lets store the model on disk
pickle.dump(predictions, open("predictions.p", "wb"))
pickle.dump(history, open("history.p", "wb"))
tf.train.write_graph(sess.graph_def, '.', './checkpoint/har.pbtxt')  
saver.save(sess, save_path = "./checkpoint/har.ckpt")
sess.close()
history = pickle.load(open("history.p", "rb"))
predictions = pickle.load(open("predictions.p", "rb"))

Evaluation

plt.figure(figsize=(12, 8))

plt.plot(np.array(history['train_loss']), "b--", label="Train loss")
plt.plot(np.array(history['train_acc']), "g--", label="Train accuracy")

plt.plot(np.array(history['test_loss']), "b-", label="Test loss")
plt.plot(np.array(history['test_acc']), "g-", label="Test accuracy")

plt.title("Training session's progress over iterations")
plt.legend(loc='upper right', shadow=True)
plt.ylabel('Training Progress (Loss or Accuracy values)')
plt.xlabel('Training Epoch')
plt.ylim(0)

plt.show()

png

LABELS = ['Downstairs', 'Jogging', 'Sitting', 'Standing', 'Upstairs', 'Walking']
Confusion Matrix
max_test = np.argmax(y_test, axis=1)
max_predictions = np.argmax(predictions, axis=1)
confusion_matrix = metrics.confusion_matrix(max_test, max_predictions)

plt.figure(figsize=(16, 14))
sns.heatmap(confusion_matrix, xticklabels=LABELS, yticklabels=LABELS, annot=True, fmt="d");
plt.title("Confusion matrix")
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.show();

png

References