#!/usr/bin/python3 # -*- coding: utf-8 -*- # MIT License # # Copyright (c) 2018 Justin Shenk # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # IMPORTANT: # # This code is derived from Iván de Paz Centeno's implementation of MTCNN # (https://github.com/ipazc/mtcnn/) and Octavia Arriaga's facial expression recognition repo # (https://github.com/oarriaga/face_classification). # import logging import os import pkg_resources import requests import sys from typing import Sequence, Tuple, Union import cv2 import numpy as np from tensorflow.keras.models import load_model from .utils import load_image logging.basicConfig(level=logging.INFO) log = logging.getLogger("fer") NumpyRects = Union[np.ndarray, Sequence[Tuple[int, int, int, int]]] __author__ = "Justin Shenk" PADDING = 40 SERVER_URL = "http://localhost:8501/v1/models/emotion_model:predict" class FER(object): """ Allows performing Facial Expression Recognition -> a) Detection of faces b) Detection of emotions """ def __init__( self, cascade_file: str = None, mtcnn=False, tfserving: bool = False, scale_factor: float = 1.1, min_face_size: int = 50, min_neighbors: int = 5, offsets: tuple = (10, 10), ): """ Initializes the face detector and Keras model for facial expression recognition. :param cascade_file: file URI with the Haar cascade for face classification :param mtcnn: use MTCNN network for face detection (not yet implemented) :param scale_factor: parameter specifying how much the image size is reduced at each image scale :param min_face_size: minimum size of the face to detect :param offsets: padding around face before classification """ self.__scale_factor = scale_factor self.__min_neighbors = min_neighbors self.__min_face_size = min_face_size self.__offsets = offsets self.tfserving = tfserving if cascade_file is None: cascade_file = cv2.data.haarcascades + "haarcascade_frontalface_default.xml" if mtcnn: try: from facenet_pytorch import MTCNN except ImportError: raise Exception( "MTCNN not installed, install it with pip install facenet-pytorch and from facenet_pytorch import MTCNN" ) self.__face_detector = "mtcnn" self._mtcnn = MTCNN(keep_all=True) else: self.__face_detector = cv2.CascadeClassifier(cascade_file) self._initialize_model() def _initialize_model(self): if self.tfserving: self.__emotion_target_size = (64, 64) # hardcoded for now else: # Local Keras model emotion_model = pkg_resources.resource_filename( "fer", "data/emotion_model.hdf5" ) log.debug("Emotion model: {}".format(emotion_model)) self.__emotion_classifier = load_model(emotion_model, compile=False) self.__emotion_classifier.make_predict_function() self.__emotion_target_size = self.__emotion_classifier.input_shape[1:3] return def _classify_emotions(self, gray_faces: np.ndarray) -> np.ndarray: # b x w x h """Run faces through online or offline classifier.""" if self.tfserving: gray_faces = np.expand_dims(gray_faces, -1) # to 4-dimensions instances = gray_faces.tolist() response = requests.post(SERVER_URL, json={"instances": instances}) response.raise_for_status() emotion_predictions = response.json()["predictions"] return emotion_predictions else: return self.__emotion_classifier(gray_faces) @staticmethod def pad(image): """Pad image.""" row, col = image.shape[:2] bottom = image[row - 2 : row, 0:col] mean = cv2.mean(bottom)[0] padded_image = cv2.copyMakeBorder( image, top=PADDING, bottom=PADDING, left=PADDING, right=PADDING, borderType=cv2.BORDER_CONSTANT, value=[mean, mean, mean], ) return padded_image @staticmethod def depad(image): row, col = image.shape[:2] return image[PADDING : row - PADDING, PADDING : col - PADDING] @staticmethod def tosquare(bbox): """Convert bounding box to square by elongating shorter side.""" x, y, w, h = bbox if h > w: diff = h - w x -= diff // 2 w += diff elif w > h: diff = w - h y -= diff // 2 h += diff if w != h: log.debug(f"{w} is not {h}") return (x, y, w, h) def find_faces(self, img: np.ndarray, bgr=True) -> list: """Image to list of faces bounding boxes(x,y,w,h)""" if isinstance(self.__face_detector, cv2.CascadeClassifier): if bgr: gray_image_array = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) else: # assume gray gray_image_array = img faces = self.__face_detector.detectMultiScale( gray_image_array, scaleFactor=self.__scale_factor, minNeighbors=self.__min_neighbors, flags=cv2.CASCADE_SCALE_IMAGE, minSize=(self.__min_face_size, self.__min_face_size), ) elif self.__face_detector == "mtcnn": boxes, probs = self._mtcnn.detect(img) faces = [] if type(boxes) == np.ndarray: for face in boxes: faces.append( [ int(face[0]), int(face[1]), int(face[2]) - int(face[0]), int(face[3]) - int(face[1]), ] ) return faces @staticmethod def __preprocess_input(x, v2=False): x = x.astype("float32") x = x / 255.0 if v2: x = x - 0.5 x = x * 2.0 return x def __apply_offsets(self, face_coordinates): """Offset face coordinates with padding before classification. x1, x2, y1, y2 = 0, 100, 0, 100 becomes -10, 110, -10, 110 """ x, y, width, height = face_coordinates x_off, y_off = self.__offsets x1 = x - x_off x2 = x + width + x_off y1 = y - y_off y2 = y + height + y_off return x1, x2, y1, y2 @staticmethod def _get_labels(): return { 0: "angry", 1: "disgust", 2: "fear", 3: "happy", 4: "sad", 5: "surprise", 6: "neutral", } def detect_emotions( self, img: np.ndarray, face_rectangles: NumpyRects = None ) -> list: """ Detects bounding boxes from the specified image with ranking of emotions. :param img: exact image path, numpy array (BGR or gray) or based64 encoded images could be passed. :return: list containing all the bounding boxes detected with their emotions. """ img = load_image(img) emotion_labels = self._get_labels() if face_rectangles is None: face_rectangles = self.find_faces(img, bgr=True) gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) gray_img = self.pad(gray_img) emotions = [] gray_faces = [] if face_rectangles is not None: for face_coordinates in face_rectangles: face_coordinates = self.tosquare(face_coordinates) # offset to expand bounding box # Note: x1 and y1 can be negative x1, x2, y1, y2 = self.__apply_offsets(face_coordinates) # account for padding in bounding box coordinates x1 += PADDING y1 += PADDING x2 += PADDING y2 += PADDING x1 = np.clip(x1, a_min=0, a_max=None) y1 = np.clip(y1, a_min=0, a_max=None) gray_face = gray_img[max(0, y1) : y2, max(0, x1) : x2] try: gray_face = cv2.resize(gray_face, self.__emotion_target_size) except Exception as e: log.warn("{} resize failed: {}".format(gray_face.shape, e)) continue # Local Keras model gray_face = self.__preprocess_input(gray_face, True) gray_faces.append(gray_face) # predict all faces if not len(gray_faces): return emotions # no valid faces # classify emotions emotion_predictions = self._classify_emotions(np.array(gray_faces)) # label scores for face_idx, face in enumerate(emotion_predictions): labelled_emotions = { emotion_labels[idx]: round(float(score), 2) for idx, score in enumerate(face) } emotions.append( dict(box=face_rectangles[face_idx], emotions=labelled_emotions) ) self.emotions = emotions return emotions def top_emotion( self, img: np.ndarray ) -> Tuple[Union[str, None], Union[float, None]]: """Convenience wrapper for `detect_emotions` returning only top emotion for first face in frame. :param img: image to process :return: top emotion and score (for first face in frame) or (None, None) """ emotions = self.detect_emotions(img=img) top_emotions = [ max(e["emotions"], key=lambda key: e["emotions"][key]) for e in emotions ] # Take first face if len(top_emotions): top_emotion = top_emotions[0] else: return (None, None) score = emotions[0]["emotions"][top_emotion] return top_emotion, score def parse_arguments(args): import argparse parser = argparse.ArgumentParser() parser.add_argument("--image", type=str, help="Image filepath") return parser.parse_args() def top_emotion(): args = parse_arguments(sys.argv) fer = FER() top_emotion, score = fer.top_emotion(args.image) print(top_emotion, score) def main(): top_emotion() if __name__ == "__main__": main()