Facial Keypoints Detection
Dataset¶
Data comes from a Kaggle competition and can be found here.
Objective¶
Our objective is to predict keypoint positions on face images. This can be used as a building block in several applications, such as:
- tracking faces in images and video
- analysing facial expressions
- detecting dysmorphic facial signs for medical diagnosis
- biometrics / face recognition
In [2]:
# Import relevant modules
import os
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.utils import shuffle
import tensorflow as tf
from tensorflow import keras
%matplotlib inline
In [6]:
print(tf.__version__)
print(keras.__version__)
In [7]:
def load_data(test=False):
"""Load the data from csv files. Set test to True to import test data."""
if test:
df = pd.read_csv("data/test.csv")
else:
df = pd.read_csv("data/training.csv")
# Drop rows with missing values
df = df.dropna()
# Image column is a list of pixels separated by a comma
# Convert the values to np arrays
df['Image'] = df['Image'].apply(lambda x: np.fromstring(x, sep=' '))
# Normalize pixel values
X = np.vstack(df['Image'].values) / 255.
X = X.astype(np.float32)
# Only TRAIN has target columns
if not test:
y = df.drop("Image", axis=1)
y = y.astype(np.float32)
else:
y = None
return X, y
In [8]:
def visualize(id):
plt.imshow(X_train[id].reshape(96,96), cmap="gray")
plt.scatter(y_train.loc[id, y_train.columns[::2]],\
y_train.loc[id, y_train.columns[1::2]],\
marker='x', s=10, c="red")
plt.axis("off")
In [9]:
def visualize_pred(model_predictions, id):
plt.imshow(X_test[id].reshape(96,96), cmap="gray")
pred = pd.DataFrame(model_predictions)
plt.scatter(pred.loc[id, pred.columns[::2]],\
pred.loc[id, pred.columns[1::2]],\
marker="x", s=10, c="red")
plt.axis("off")
In [10]:
# Plot the validation and train learning curves and loss
def learning_curves(model_history):
pd.DataFrame(model_history.history).plot(figsize=(8,5))
plt.grid(True)
In [11]:
# Load the data
X_train, y_train = load_data()
X_test, y_test = load_data(test=True)
print(f"X_train's shape: {X_train.shape}\ny_train's shape: {y_train.shape}\nX_test's shape: {X_test.shape}")
In [12]:
fig = plt.figure(figsize=(12, 12))
for i in range(9):
ax = fig.add_subplot(3, 3, i + 1)
visualize(i)
plt.savefig("facial_keypoints.png")
In [10]:
NUM_LABELS = 30
IMAGE_SIZE = 96
BATCH_SIZE = 64
DROPOUT_RATE = 0.2
In [11]:
# Define a first model
model_1 = keras.Sequential()
model_1.add(keras.layers.Dense(128, input_shape=(IMAGE_SIZE * IMAGE_SIZE,), activation="relu"))
model_1.add(keras.layers.Dropout(DROPOUT_RATE))
model_1.add(keras.layers.Dense(64, activation="relu"))
model_1.add(keras.layers.Dropout(DROPOUT_RATE))
model_1.add(keras.layers.Dense(NUM_LABELS))
# Summary of the model
model_1.summary()
# Compile the model
model_1.compile(optimizer="adam", loss="mse", metrics=[keras.metrics.RootMeanSquaredError(name='rmse')])
# Train the model
history_1 = model_1.fit(X_train, y_train, epochs = 100, batch_size = BATCH_SIZE,\
validation_split = 0.2, verbose=0)
print(f"Final loss of training data: {history_1.history['loss'][-1]}")
print(f"Final loss of validation data: {history_1.history['val_loss'][-1]}")
# Plot the learning curves
learning_curves(history_1)
In [12]:
# Create second model - CNN
X_train_2 = X_train.reshape(2140, IMAGE_SIZE, IMAGE_SIZE, 1)
model_2 = keras.Sequential()
model_2.add(keras.layers.Convolution2D(32, (3,3), padding='same', use_bias=False, activation="relu",\
input_shape=(IMAGE_SIZE, IMAGE_SIZE, 1)))
model_2.add(keras.layers.BatchNormalization())
model_2.add(keras.layers.Convolution2D(32, (3,3), padding='same', use_bias=False, activation="relu"))
model_2.add(keras.layers.BatchNormalization())
model_2.add(keras.layers.MaxPool2D(pool_size=(2, 2)))
model_2.add(keras.layers.Convolution2D(64, (3,3), padding='same', use_bias=False, activation="relu"))
model_2.add(keras.layers.BatchNormalization())
model_2.add(keras.layers.Convolution2D(64, (3,3), padding='same', use_bias=False, activation="relu"))
model_2.add(keras.layers.MaxPool2D(pool_size=(2, 2)))
model_2.add(keras.layers.Convolution2D(96, (3,3), padding='same', use_bias=False, activation="relu"))
model_2.add(keras.layers.BatchNormalization())
model_2.add(keras.layers.Convolution2D(96, (3,3), padding='same', use_bias=False, activation="relu"))
model_2.add(keras.layers.BatchNormalization())
model_2.add(keras.layers.MaxPool2D(pool_size=(2, 2)))
model_2.add(keras.layers.Convolution2D(128, (3,3),padding='same', use_bias=False, activation="relu"))
model_2.add(keras.layers.BatchNormalization())
model_2.add(keras.layers.Convolution2D(128, (3,3),padding='same', use_bias=False, activation="relu"))
model_2.add(keras.layers.BatchNormalization())
model_2.add(keras.layers.MaxPool2D(pool_size=(2, 2)))
model_2.add(keras.layers.Convolution2D(256, (3,3),padding='same',use_bias=False, activation="relu"))
model_2.add(keras.layers.BatchNormalization())
model_2.add(keras.layers.Convolution2D(256, (3,3),padding='same',use_bias=False, activation="relu"))
model_2.add(keras.layers.BatchNormalization())
model_2.add(keras.layers.MaxPool2D(pool_size=(2, 2)))
model_2.add(keras.layers.Convolution2D(512, (3,3), padding='same', use_bias=False, activation="relu"))
model_2.add(keras.layers.BatchNormalization())
model_2.add(keras.layers.Convolution2D(512, (3,3), padding='same', use_bias=False, activation="relu"))
model_2.add(keras.layers.BatchNormalization())
model_2.add(keras.layers.Flatten())
model_2.add(keras.layers.Dense(512, activation="relu"))
model_2.add(keras.layers.Dropout(DROPOUT_RATE))
model_2.add(keras.layers.Dense(NUM_LABELS))
# Summary of the model
model_2.summary()
# Compile the model
model_2.compile(optimizer="adam", loss="mse", metrics=[keras.metrics.RootMeanSquaredError(name='rmse')])
# Train the model
history_2 = model_2.fit(X_train_2, y_train, epochs = 25, batch_size = BATCH_SIZE,\
validation_split = 0.2, verbose=2)
print(f"Final loss of training data: {history_2.history['loss'][-1]}")
print(f"Final loss of validation data: {history_2.history['val_loss'][-1]}")
# Plot the learning curves
learning_curves(history_2)
In [16]:
# Predict from model 2 (CNN architecture)
X_test_2 = X_test.reshape(1783, IMAGE_SIZE, IMAGE_SIZE, 1)
model_2_predictions = model_2.predict(X_test_2)
model_2_predictions.shape
Out[16]:
In [17]:
# Visualize the predictions
fig = plt.figure(figsize=(12, 12))
for i in range(9):
ax = fig.add_subplot(3, 3, i + 1)
visualize_pred(model_2_predictions, i)
