Upload 8 files

model/app_utils.py

@@ -0,0 +1,174 @@
+"""
+File: app_utils.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: This module contains utility functions for facial expression recognition application.
+License: MIT License
+"""
+
+import torch
+import numpy as np
+import mediapipe as mp
+from PIL import Image
+import cv2
+from pytorch_grad_cam.utils.image import show_cam_on_image
+
+# Importing necessary components for the Gradio app
+from app.model import pth_model_static, pth_model_dynamic, cam, pth_processing
+from app.face_utils import get_box, display_info
+from app.config import DICT_EMO, config_data
+from app.plot import statistics_plot
+
+mp_face_mesh = mp.solutions.face_mesh
+
+
+def preprocess_image_and_predict(inp):
+    inp = np.array(inp)
+
+    if inp is None:
+        return None, None, None
+
+    try:
+        h, w = inp.shape[:2]
+    except Exception:
+        return None, None, None
+
+    with mp_face_mesh.FaceMesh(
+        max_num_faces=1,
+        refine_landmarks=False,
+        min_detection_confidence=0.5,
+        min_tracking_confidence=0.5,
+    ) as face_mesh:
+        results = face_mesh.process(inp)
+        if results.multi_face_landmarks:
+            for fl in results.multi_face_landmarks:
+                startX, startY, endX, endY = get_box(fl, w, h)
+                cur_face = inp[startY:endY, startX:endX]
+                cur_face_n = pth_processing(Image.fromarray(cur_face))
+                with torch.no_grad():
+                    prediction = (
+                        torch.nn.functional.softmax(pth_model_static(cur_face_n), dim=1)
+                        .detach()
+                        .numpy()[0]
+                    )
+                confidences = {DICT_EMO[i]: float(prediction[i]) for i in range(7)}
+                grayscale_cam = cam(input_tensor=cur_face_n)
+                grayscale_cam = grayscale_cam[0, :]
+                cur_face_hm = cv2.resize(cur_face,(224,224))
+                cur_face_hm = np.float32(cur_face_hm) / 255
+                heatmap = show_cam_on_image(cur_face_hm, grayscale_cam, use_rgb=True)
+
+            return cur_face, heatmap, confidences
+        
+        else:
+            return None, None, None
+
+def preprocess_video_and_predict(video):
+
+    cap = cv2.VideoCapture(video)
+    w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    fps = np.round(cap.get(cv2.CAP_PROP_FPS))
+
+    path_save_video_face = 'result_face.mp4'
+    vid_writer_face = cv2.VideoWriter(path_save_video_face, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+
+    path_save_video_hm = 'result_hm.mp4'
+    vid_writer_hm = cv2.VideoWriter(path_save_video_hm, cv2.VideoWriter_fourcc(*'mp4v'), fps, (224, 224))
+
+    lstm_features = []
+    count_frame = 1
+    count_face = 0
+    probs = []
+    frames = []
+    last_output = None
+    last_heatmap = None 
+    cur_face = None
+
+    with mp_face_mesh.FaceMesh(
+    max_num_faces=1,
+    refine_landmarks=False,
+    min_detection_confidence=0.5,
+    min_tracking_confidence=0.5) as face_mesh:
+
+        while cap.isOpened():
+            _, frame = cap.read()
+            if frame is None: break
+
+            frame_copy = frame.copy()
+            frame_copy.flags.writeable = False
+            frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_BGR2RGB)
+            results = face_mesh.process(frame_copy)
+            frame_copy.flags.writeable = True
+
+            if results.multi_face_landmarks:
+                for fl in results.multi_face_landmarks:
+                    startX, startY, endX, endY  = get_box(fl, w, h)
+                    cur_face = frame_copy[startY:endY, startX: endX]
+
+                    if count_face%config_data.FRAME_DOWNSAMPLING == 0:
+                        cur_face_copy = pth_processing(Image.fromarray(cur_face))
+                        with torch.no_grad():
+                            features = torch.nn.functional.relu(pth_model_static.extract_features(cur_face_copy)).detach().numpy()
+
+                        grayscale_cam = cam(input_tensor=cur_face_copy)
+                        grayscale_cam = grayscale_cam[0, :]
+                        cur_face_hm = cv2.resize(cur_face,(224,224), interpolation = cv2.INTER_AREA)
+                        cur_face_hm = np.float32(cur_face_hm) / 255
+                        heatmap = show_cam_on_image(cur_face_hm, grayscale_cam, use_rgb=False)
+                        last_heatmap = heatmap
+        
+                        if len(lstm_features) == 0:
+                            lstm_features = [features]*10
+                        else:
+                            lstm_features = lstm_features[1:] + [features]
+
+                        lstm_f = torch.from_numpy(np.vstack(lstm_features))
+                        lstm_f = torch.unsqueeze(lstm_f, 0)
+                        with torch.no_grad():
+                            output = pth_model_dynamic(lstm_f).detach().numpy()
+                        last_output = output
+
+                        if count_face == 0:
+                            count_face += 1
+
+                    else:
+                        if last_output is not None:
+                            output = last_output
+                            heatmap = last_heatmap
+
+                        elif last_output is None:
+                            output = np.empty((1, 7))
+                            output[:] = np.nan
+                            
+                    probs.append(output[0])
+                    frames.append(count_frame)
+            else:
+                if last_output is not None:
+                    lstm_features = []
+                    empty = np.empty((7))
+                    empty[:] = np.nan
+                    probs.append(empty)
+                    frames.append(count_frame)                        
+
+            if cur_face is not None:
+                heatmap_f = display_info(heatmap, 'Frame: {}'.format(count_frame), box_scale=.3)
+
+                cur_face = cv2.cvtColor(cur_face, cv2.COLOR_RGB2BGR)
+                cur_face = cv2.resize(cur_face, (224,224), interpolation = cv2.INTER_AREA)
+                cur_face = display_info(cur_face, 'Frame: {}'.format(count_frame), box_scale=.3)
+                vid_writer_face.write(cur_face)
+                vid_writer_hm.write(heatmap_f)
+
+            count_frame += 1
+            if count_face != 0:
+                count_face += 1
+
+        vid_writer_face.release()
+        vid_writer_hm.release()
+
+        stat = statistics_plot(frames, probs)
+
+        if not stat:
+            return None, None, None, None
+        
+    return video, path_save_video_face, path_save_video_hm, stat

model/authors.py

@@ -0,0 +1,34 @@
+"""
+File: authors.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: About the authors.
+License: MIT License
+"""
+
+
+AUTHORS = """
+    Authors: [Elena Ryumina](https://github.com/ElenaRyumina), [Dmitry Ryumin](https://github.com/DmitryRyumin), [Denis Dresvyanskiy](https://www.uni-ulm.de/en/nt/staff/research-assistants/dresvyanskiy/), [Maxim Markitantov](https://hci.nw.ru/en/employees/10) and [Alexey Karpov](https://hci.nw.ru/en/employees/1)
+
+    Authorship contribution:
+
+    App developers: ``Elena Ryumina`` and ``Dmitry Ryumin``
+
+    Methodology developers: ``Elena Ryumina``, ``Denis Dresvyanskiy`` and ``Alexey Karpov``
+
+    Model developer: ``Elena Ryumina``
+
+    TensorFlow to PyTorch model converters: ``Maxim Markitantov`` and ``Elena Ryumina``
+
+    Citation
+
+    If you are using EMO-AffectNetModel in your research, please consider to cite research [paper](https://www.sciencedirect.com/science/article/pii/S0925231222012656). Here is an example of BibTeX entry:
+
+    <div class="highlight highlight-text-bibtex notranslate position-relative overflow-auto" dir="auto"><pre><span class="pl-k">@article</span>{<span class="pl-en">RYUMINA2022</span>,
+        <span class="pl-s">title</span>        = <span class="pl-s"><span class="pl-pds">{</span>In Search of a Robust Facial Expressions Recognition Model: A Large-Scale Visual Cross-Corpus Study<span class="pl-pds">}</span></span>,
+        <span class="pl-s">author</span>       = <span class="pl-s"><span class="pl-pds">{</span>Elena Ryumina and Denis Dresvyanskiy and Alexey Karpov<span class="pl-pds">}</span></span>,
+        <span class="pl-s">journal</span>      = <span class="pl-s"><span class="pl-pds">{</span>Neurocomputing<span class="pl-pds">}</span></span>,
+        <span class="pl-s">year</span>         = <span class="pl-s"><span class="pl-pds">{</span>2022<span class="pl-pds">}</span></span>,
+        <span class="pl-s">doi</span>          = <span class="pl-s"><span class="pl-pds">{</span>10.1016/j.neucom.2022.10.013<span class="pl-pds">}</span></span>,
+        <span class="pl-s">url</span>          = <span class="pl-s"><span class="pl-pds">{</span>https://www.sciencedirect.com/science/article/pii/S0925231222012656<span class="pl-pds">}</span></span>,
+    }</div>
+"""

model/config.py

@@ -0,0 +1,49 @@
+"""
+File: config.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: Configuration file.
+License: MIT License
+"""
+
+import toml
+from typing import Dict
+from types import SimpleNamespace
+
+
+def flatten_dict(prefix: str, d: Dict) -> Dict:
+    result = {}
+
+    for k, v in d.items():
+        if isinstance(v, dict):
+            result.update(flatten_dict(f"{prefix}{k}_", v))
+        else:
+            result[f"{prefix}{k}"] = v
+
+    return result
+
+
+config = toml.load("config.toml")
+
+config_data = flatten_dict("", config)
+
+config_data = SimpleNamespace(**config_data)
+
+DICT_EMO = {
+    0: "Neutral",
+    1: "Happiness",
+    2: "Sadness",
+    3: "Surprise",
+    4: "Fear",
+    5: "Disgust",
+    6: "Anger",
+}
+
+COLORS = {
+    0: 'blue', 
+    1: 'orange', 
+    2: 'green', 
+    3: 'red', 
+    4: 'purple', 
+    5: 'brown', 
+    6: 'pink'
+}

model/description.py

@@ -0,0 +1,27 @@
+"""
+File: description.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: Project description for the Gradio app.
+License: MIT License
+"""
+
+# Importing necessary components for the Gradio app
+from app.config import config_data
+
+DESCRIPTION_STATIC = f"""\
+# Static Facial Expression Recognition
+<div class="app-flex-container">
+    <img src="https://img.shields.io/badge/version-v{config_data.APP_VERSION}-rc0" alt="Version">
+    <a href="https://visitorbadge.io/status?path=https%3A%2F%2Fhuggingface.co%2Fspaces%2FElenaRyumina%2FFacial_Expression_Recognition"><img src="https://api.visitorbadge.io/api/combined?path=https%3A%2F%2Fhuggingface.co%2Fspaces%2FElenaRyumina%2FFacial_Expression_Recognition&countColor=%23263759&style=flat" /></a>
+    <a href="https://paperswithcode.com/paper/in-search-of-a-robust-facial-expressions"><img src="https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/in-search-of-a-robust-facial-expressions/facial-expression-recognition-on-affectnet" /></a>
+    </div>
+"""
+
+DESCRIPTION_DYNAMIC = f"""\
+# Dynamic Facial Expression Recognition
+<div class="app-flex-container">
+    <img src="https://img.shields.io/badge/version-v{config_data.APP_VERSION}-rc0" alt="Version">
+    <a href="https://visitorbadge.io/status?path=https%3A%2F%2Fhuggingface.co%2Fspaces%2FElenaRyumina%2FFacial_Expression_Recognition"><img src="https://api.visitorbadge.io/api/combined?path=https%3A%2F%2Fhuggingface.co%2Fspaces%2FElenaRyumina%2FFacial_Expression_Recognition&countColor=%23263759&style=flat" /></a>
+    <a href="https://paperswithcode.com/paper/in-search-of-a-robust-facial-expressions"><img src="https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/in-search-of-a-robust-facial-expressions/facial-expression-recognition-on-affectnet" /></a>
+    </div>
+"""

model/face_utils.py

@@ -0,0 +1,68 @@
+"""
+File: face_utils.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: This module contains utility functions related to facial landmarks and image processing.
+License: MIT License
+"""
+
+import numpy as np
+import math
+import cv2
+
+
+def norm_coordinates(normalized_x, normalized_y, image_width, image_height):
+    x_px = min(math.floor(normalized_x * image_width), image_width - 1)
+    y_px = min(math.floor(normalized_y * image_height), image_height - 1)
+    return x_px, y_px
+
+
+def get_box(fl, w, h):
+    idx_to_coors = {}
+    for idx, landmark in enumerate(fl.landmark):
+        landmark_px = norm_coordinates(landmark.x, landmark.y, w, h)
+        if landmark_px:
+            idx_to_coors[idx] = landmark_px
+
+    x_min = np.min(np.asarray(list(idx_to_coors.values()))[:, 0])
+    y_min = np.min(np.asarray(list(idx_to_coors.values()))[:, 1])
+    endX = np.max(np.asarray(list(idx_to_coors.values()))[:, 0])
+    endY = np.max(np.asarray(list(idx_to_coors.values()))[:, 1])
+
+    (startX, startY) = (max(0, x_min), max(0, y_min))
+    (endX, endY) = (min(w - 1, endX), min(h - 1, endY))
+
+    return startX, startY, endX, endY
+
+def display_info(img, text, margin=1.0, box_scale=1.0):
+    img_copy = img.copy()
+    img_h, img_w, _ = img_copy.shape
+    line_width = int(min(img_h, img_w) * 0.001)
+    thickness = max(int(line_width / 3), 1)
+
+    font_face = cv2.FONT_HERSHEY_SIMPLEX
+    font_color = (0, 0, 0)
+    font_scale = thickness / 1.5
+
+    t_w, t_h = cv2.getTextSize(text, font_face, font_scale, None)[0]
+
+    margin_n = int(t_h * margin)
+    sub_img = img_copy[0 + margin_n: 0 + margin_n + t_h + int(2 * t_h * box_scale),
+              img_w - t_w - margin_n - int(2 * t_h * box_scale): img_w - margin_n]
+
+    white_rect = np.ones(sub_img.shape, dtype=np.uint8) * 255
+
+    img_copy[0 + margin_n: 0 + margin_n + t_h + int(2 * t_h * box_scale),
+    img_w - t_w - margin_n - int(2 * t_h * box_scale):img_w - margin_n] = cv2.addWeighted(sub_img, 0.5, white_rect, .5, 1.0)
+
+    cv2.putText(img=img_copy,
+                text=text,
+                org=(img_w - t_w - margin_n - int(2 * t_h * box_scale) // 2,
+                     0 + margin_n + t_h + int(2 * t_h * box_scale) // 2),
+                fontFace=font_face,
+                fontScale=font_scale,
+                color=font_color,
+                thickness=thickness,
+                lineType=cv2.LINE_AA,
+                bottomLeftOrigin=False)
+
+    return img_copy

model/model.py

@@ -0,0 +1,64 @@
+"""
+File: model.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: This module provides functions for loading and processing a pre-trained deep learning model
+             for facial expression recognition.
+License: MIT License
+"""
+
+import torch
+import requests
+from PIL import Image
+from torchvision import transforms
+from pytorch_grad_cam import GradCAM
+
+# Importing necessary components for the Gradio app
+from app.config import config_data
+from app.model_architectures import ResNet50, LSTMPyTorch
+
+
+def load_model(model_url, model_path):
+    try:
+        with requests.get(model_url, stream=True) as response:
+            with open(model_path, "wb") as file:
+                for chunk in response.iter_content(chunk_size=8192):
+                    file.write(chunk)
+        return model_path
+    except Exception as e:
+        print(f"Error loading model: {e}")
+        return None
+
+path_static = load_model(config_data.model_static_url, config_data.model_static_path)   
+pth_model_static = ResNet50(7, channels=3)
+pth_model_static.load_state_dict(torch.load(path_static))
+pth_model_static.eval()
+
+path_dynamic = load_model(config_data.model_dynamic_url, config_data.model_dynamic_path) 
+pth_model_dynamic = LSTMPyTorch()
+pth_model_dynamic.load_state_dict(torch.load(path_dynamic))
+pth_model_dynamic.eval()
+
+target_layers = [pth_model_static.layer4]
+cam = GradCAM(model=pth_model_static, target_layers=target_layers)
+
+def pth_processing(fp):
+    class PreprocessInput(torch.nn.Module):
+        def init(self):
+            super(PreprocessInput, self).init()
+
+        def forward(self, x):
+            x = x.to(torch.float32)
+            x = torch.flip(x, dims=(0,))
+            x[0, :, :] -= 91.4953
+            x[1, :, :] -= 103.8827
+            x[2, :, :] -= 131.0912
+            return x
+
+    def get_img_torch(img, target_size=(224, 224)):
+        transform = transforms.Compose([transforms.PILToTensor(), PreprocessInput()])
+        img = img.resize(target_size, Image.Resampling.NEAREST)
+        img = transform(img)
+        img = torch.unsqueeze(img, 0)
+        return img
+
+    return get_img_torch(fp)

model/model_architectures.py

@@ -0,0 +1,150 @@
+"""
+File: model.py
+Author: Elena Ryumina and Dmitry Ryumin
+Description: This module provides model architectures.
+License: MIT License
+"""
+
+import torch
+import torch.nn as  nn
+import torch.nn.functional as F
+import math
+
+class Bottleneck(nn.Module):
+    expansion = 4
+    def __init__(self, in_channels, out_channels, i_downsample=None, stride=1):
+        super(Bottleneck, self).__init__()
+        
+        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, padding=0, bias=False)
+        self.batch_norm1 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.99)
+        
+        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, padding='same', bias=False)
+        self.batch_norm2 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.99)
+        
+        self.conv3 = nn.Conv2d(out_channels, out_channels*self.expansion, kernel_size=1, stride=1, padding=0, bias=False)
+        self.batch_norm3 = nn.BatchNorm2d(out_channels*self.expansion, eps=0.001, momentum=0.99)
+        
+        self.i_downsample = i_downsample
+        self.stride = stride
+        self.relu = nn.ReLU()
+        
+    def forward(self, x):
+        identity = x.clone()
+        x = self.relu(self.batch_norm1(self.conv1(x)))
+        
+        x = self.relu(self.batch_norm2(self.conv2(x)))
+        
+        x = self.conv3(x)
+        x = self.batch_norm3(x)
+        
+        #downsample if needed
+        if self.i_downsample is not None:
+            identity = self.i_downsample(identity)
+        #add identity
+        x+=identity
+        x=self.relu(x)
+        
+        return x
+
+class Conv2dSame(torch.nn.Conv2d):
+
+    def calc_same_pad(self, i: int, k: int, s: int, d: int) -> int:
+        return max((math.ceil(i / s) - 1) * s + (k - 1) * d + 1 - i, 0)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        ih, iw = x.size()[-2:]
+
+        pad_h = self.calc_same_pad(i=ih, k=self.kernel_size[0], s=self.stride[0], d=self.dilation[0])
+        pad_w = self.calc_same_pad(i=iw, k=self.kernel_size[1], s=self.stride[1], d=self.dilation[1])
+
+        if pad_h > 0 or pad_w > 0:
+            x = F.pad(
+                x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2]
+            )
+        return F.conv2d(
+            x,
+            self.weight,
+            self.bias,
+            self.stride,
+            self.padding,
+            self.dilation,
+            self.groups,
+        )
+
+class ResNet(nn.Module):
+    def __init__(self, ResBlock, layer_list, num_classes, num_channels=3):
+        super(ResNet, self).__init__()
+        self.in_channels = 64
+
+        self.conv_layer_s2_same = Conv2dSame(num_channels, 64, 7, stride=2, groups=1, bias=False)
+        self.batch_norm1 = nn.BatchNorm2d(64, eps=0.001, momentum=0.99)
+        self.relu = nn.ReLU()
+        self.max_pool = nn.MaxPool2d(kernel_size = 3, stride=2)
+        
+        self.layer1 = self._make_layer(ResBlock, layer_list[0], planes=64, stride=1)
+        self.layer2 = self._make_layer(ResBlock, layer_list[1], planes=128, stride=2)
+        self.layer3 = self._make_layer(ResBlock, layer_list[2], planes=256, stride=2)
+        self.layer4 = self._make_layer(ResBlock, layer_list[3], planes=512, stride=2)
+        
+        self.avgpool = nn.AdaptiveAvgPool2d((1,1))
+        self.fc1 = nn.Linear(512*ResBlock.expansion, 512)
+        self.relu1 = nn.ReLU()
+        self.fc2 = nn.Linear(512, num_classes)
+
+    def extract_features(self, x):
+        x = self.relu(self.batch_norm1(self.conv_layer_s2_same(x)))
+        x = self.max_pool(x)
+        # print(x.shape)
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+        
+        x = self.avgpool(x)
+        x = x.reshape(x.shape[0], -1)
+        x = self.fc1(x)
+        return x
+        
+    def forward(self, x):
+        x = self.extract_features(x)
+        x = self.relu1(x)
+        x = self.fc2(x)
+        return x
+        
+    def _make_layer(self, ResBlock, blocks, planes, stride=1):
+        ii_downsample = None
+        layers = []
+        
+        if stride != 1 or self.in_channels != planes*ResBlock.expansion:
+            ii_downsample = nn.Sequential(
+                nn.Conv2d(self.in_channels, planes*ResBlock.expansion, kernel_size=1, stride=stride, bias=False, padding=0),
+                nn.BatchNorm2d(planes*ResBlock.expansion, eps=0.001, momentum=0.99)
+            )
+            
+        layers.append(ResBlock(self.in_channels, planes, i_downsample=ii_downsample, stride=stride))
+        self.in_channels = planes*ResBlock.expansion
+        
+        for i in range(blocks-1):
+            layers.append(ResBlock(self.in_channels, planes))
+            
+        return nn.Sequential(*layers)
+        
+def ResNet50(num_classes, channels=3):
+    return ResNet(Bottleneck, [3,4,6,3], num_classes, channels)
+
+
+class LSTMPyTorch(nn.Module):
+    def __init__(self):
+        super(LSTMPyTorch, self).__init__()
+        
+        self.lstm1 = nn.LSTM(input_size=512, hidden_size=512, batch_first=True, bidirectional=False)
+        self.lstm2 = nn.LSTM(input_size=512, hidden_size=256, batch_first=True, bidirectional=False)
+        self.fc = nn.Linear(256, 7)
+        self.softmax = nn.Softmax(dim=1)
+
+    def forward(self, x):
+        x, _ = self.lstm1(x)
+        x, _ = self.lstm2(x)        
+        x = self.fc(x[:, -1, :])
+        x = self.softmax(x)
+        return x