add docstring

Dockerfile

@@ -5,24 +5,28 @@ RUN apt-get update && \
     apt-get install ffmpeg libsm6 libxext6 -y && \
     apt-get clean
 
-# Install the dependancies
+# ---- Install the dependancies
+#
 COPY requirements.txt /
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Will execute nltk.download('wordnet')
-#COPY post_install.py .
-#RUN python post_install.py
-
+# ---- RUN python
+# ---- Will execute nltk.download('wordnet')
+#
 RUN [ "python", "-c", "import nltk; nltk.download('wordnet', download_dir='/usr/local/nltk_data')" ]
 
-# Copy the code files
+# ---- Copy the code files
+#
 COPY src /
 
-# Listen to port 7860
+# ---- Listen to port 5000
+#
 EXPOSE 5000
 
-# Define the working dir in the contener
+# ---- Define the working dir in the contener
+#
 WORKDIR /
 
-# Commande to start the app
+# ---- Commande to start the app
+# ----
 CMD ["gunicorn", "--bind", "0.0.0.0:5000", "main:app"]

src/display_gloss.py

@@ -2,36 +2,44 @@ import cv2
 import json
 import numpy as np
 import pandas as pd
-import os
 import time
 
+
 def draw_hands_connections(frame, hand_landmarks):
     '''
-    Draw white lines between relevant points of hands landmarks
-    
+    Draw white lines on the given frame between relevant hand keypoints.
+
     Parameters
     ----------
-    frame: numpy array, corresponding to the frame on which we want to draw
-    hand_landmarks: dictionnary, collecting the hands landmarks
+    frame: numpy array
+        The frame on which we want to draw.
+    hand_landmarks: dict
+        Dictionary mapping keypoint IDs (integers) to hand landmarks 
+        (lists of two floats corresponding to the coordinates) for both hands.
 
-    Return
-    ------
-    frame: numpy array, with the newly drawing of the hands
+    Returns
+    -------
+    frame: numpy array
+        The frame with the newly drawn hand connections.
     '''
-    # define hand_connections between keypoints
+
+    # ---- Define hand_connections between keypoints to draw
+    #
     hand_connections = [[0, 1], [1, 2], [2, 3], [3, 4],
                         [5, 6], [6, 7], [7, 8],
                         [9, 10], [10, 11], [11, 12],
                         [13, 14], [14, 15], [15, 16],
                         [17, 18], [18, 19], [19, 20]] #[5, 2], [0, 17]]
     
-    # loop to draw left hand connection
+    # ---- loop to draw left hand connections
+    #
     for connection in hand_connections:
         landmark_start = hand_landmarks['left_hand'].get(str(connection[0]))
         landmark_end = hand_landmarks['left_hand'].get(str(connection[1]))
         cv2.line(frame, landmark_start, landmark_end, (255, 255, 255), 2)
     
-    # loop to to draw right hand connection
+    # ---- loop to to draw right hand connections
+    #
     for connection in hand_connections:
         landmark_start = hand_landmarks['right_hand'].get(str(connection[0]))
         landmark_end = hand_landmarks['right_hand'].get(str(connection[1]))
@@ -41,20 +49,28 @@ def draw_hands_connections(frame, hand_landmarks):
 
 def draw_pose_connections(frame, pose_landmarks):
     '''
-    Draw white lines between relevant points of pose landmarks
-    
+    Draw white lines on the given frame between relevant posture keypoints.
+
     Parameters
     ----------
-    frame: numpy array, corresponding to the frame on which we want to draw
-    hand_landmarks: dictionnary, collecting the pose landmarks
-    
-    Return
-    ------
-    frame: numpy array, with the newly drawing of the pose
+    frame: numpy array
+        The frame on which we want to draw.
+    pose_landmarks: dict
+        Dictionary mapping keypoint IDs (integers) to posture landmarks 
+        (lists of two floats corresponding to the coordinates).
+
+    Returns
+    -------
+    frame: numpy array
+        The frame with the newly drawn posture connections.
     '''
-    # define pose connections
+
+    # ---- define posture connections between keypoints to draw
+    #
     pose_connections = [[11, 12], [11, 13], [12, 14], [13, 15], [14, 16]]
 
+    # ---- loop to to draw posture connections
+    #
     for connection in pose_connections:
         landmark_start = pose_landmarks.get(str(connection[0]))
         landmark_end = pose_landmarks.get(str(connection[1]))
@@ -64,31 +80,38 @@ def draw_pose_connections(frame, pose_landmarks):
 
 def draw_face_connections(frame, face_landmarks):
     '''
-    Draw white lines between relevant points of face landmarks
-    
+    Draw white lines on the given frame between relevant face keypoints.
+
     Parameters
     ----------
-    frame: numpy array, corresponding to the frame on which we want to draw
-    hand_landmarks: dictionnary, collecting the face landmarks
-    
-    Return
-    ------
-    frame: numpy array, with the newly drawing of the face
+    frame: numpy array
+        The frame on which we want to draw.
+    face_landmarks: dict
+        Dictionary mapping keypoint IDs (integers) to face landmarks 
+        (lists of two floats corresponding to the coordinates).
+
+    Returns
+    -------
+    frame: numpy array
+        The frame with the newly drawn face connections.
     '''
-    # define pose connections
+    # ---- define pose connections
+    #
     connections_dict = {'lipsUpperInner_connections' : [78, 191, 80, 81, 82, 13, 312, 311, 310, 415, 308],\
-    'lipsLowerInner_connections' : [78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308],\
-    'rightEyeUpper0_connections': [246, 161, 160, 159, 158, 157, 173],\
-    'rightEyeLower0' : [33, 7, 163, 144, 145, 153, 154, 155, 133],\
-    'rightEyebrowLower' : [35, 124, 46, 53, 52, 65],\
-    'leftEyeUpper0' : [466, 388, 387, 386, 385, 384, 398],\
-    'leftEyeLower0' : [263, 249, 390, 373, 374, 380, 381, 382, 362],\
-    'leftEyebrowLower' : [265, 353, 276, 283, 282, 295],\
-    'noseTip_midwayBetweenEye' :  [1, 168],\
-    'noseTip_noseRightCorner' : [1, 98],\
-    'noseTip_LeftCorner' : [1, 327]\
-    }
+                    'lipsLowerInner_connections' : [78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308],\
+                    'rightEyeUpper0_connections': [246, 161, 160, 159, 158, 157, 173],\
+                    'rightEyeLower0' : [33, 7, 163, 144, 145, 153, 154, 155, 133],\
+                    'rightEyebrowLower' : [35, 124, 46, 53, 52, 65],\
+                    'leftEyeUpper0' : [466, 388, 387, 386, 385, 384, 398],\
+                    'leftEyeLower0' : [263, 249, 390, 373, 374, 380, 381, 382, 362],\
+                    'leftEyebrowLower' : [265, 353, 276, 283, 282, 295],\
+                    'noseTip_midwayBetweenEye' :  [1, 168],\
+                    'noseTip_noseRightCorner' : [1, 98],\
+                    'noseTip_LeftCorner' : [1, 327]\
+                    }
 
+    # ---- loop to to draw face connections
+    #
     for keypoints_list in connections_dict.values():
         for index in range(len(keypoints_list)):
             if index + 1 < len(keypoints_list):
@@ -98,20 +121,78 @@ def draw_face_connections(frame, face_landmarks):
     return frame
 
 def resize_landmarks(landmarks, resize_rate_width, resize_rate_height):
+    '''
+    Resize landmark coordinates by applying specific scaling factors 
+    to both the width and height of the frame.
+
+    Parameters
+    ----------
+    landmarks: dict
+        Dictionary mapping keypoint IDs (integers) to landmarks
+        (lists of two floats corresponding to the coordinates).
+    resize_rate_width: float
+        Scaling factor applied to the x-coordinate (width).
+    resize_rate_height: float
+        Scaling factor applied to the y-coordinate (height).
+
+    Returns
+    -------
+    landmarks: dict
+        Dictionary mapping keypoint IDs (integers) to the newly resized landmarks
+        (lists of two integers corresponding to the coordinates).
+    '''
+
     for keypoint in landmarks.keys():
         landmark_x, landmark_y = landmarks[keypoint]
         landmarks[keypoint] = [int(resize_rate_width * landmark_x), int(resize_rate_height*landmark_y)]
+
     return landmarks
 
 def generate_video(gloss_list, dataset, vocabulary_list):
-    # size of video of signer 11
-    # FIXED_WIDTH, FIXED_HEIGHT, = 288, 192,
+    '''
+    Generate a video stream from a list of glosses.
+
+    Parameters
+    ----------
+    gloss_list: list of str
+        List of glosses from which the signing video will be generated.
+    dataset: pandas.DataFrame
+        Dataset containing information about each gloss, including paths to landmark data.
+    vocabulary_list: list of str
+        List of tokens that have associated landmarks collected.
+
+    Yields
+    ------
+    frame: bytes
+        JPEG-encoded frame for streaming.
+    '''
+    # ---- Fix size of the frame to the most common size of video we have in the dataset
+    # (corresponding to signer ID 11 who has the maximum number of videos).
+    #
     FIXED_WIDTH,  FIXED_HEIGHT = 576, 384
-    fps = 25
 
+    # ---- Fix the Frames Per Second (FPS) to match the videos collected in the dataset.
+    #
+    FPS = 25
+
+    # ---- Define carachteristics for text display.
+    #
+    font = cv2.FONT_HERSHEY_SIMPLEX
+    font_scale = 1
+    font_color = (0, 255, 0)
+    thickness = 2
+    line_type = cv2.LINE_AA
+
+    # ---- Loop over each gloss
+    #
     for gloss in gloss_list:
+        # ---- Skip if gloss not in the vocabulary_list.
+        #
         if not check_gloss_in_vocabulary(gloss, vocabulary_list):
             continue
+
+        # ---- Get landmarks of all the frame in the dataset corresponding to the appropriate gloss.
+        #
         video_id = select_video_id_from_gloss(gloss, dataset)
         video_landmarks_path = dataset.loc[dataset['video_id'] == video_id, 'video_landmarks_path'].values[0]
         with open(video_landmarks_path, 'r') as f:
@@ -119,76 +200,120 @@ def generate_video(gloss_list, dataset, vocabulary_list):
         width = video_landmarks[-1].get('width')
         height = video_landmarks[-1].get('height')
 
-        # calculate resize rate
+        # ---- Calculate resize rate for future landmark rescaling.
+        #
         resize_rate_width, resize_rate_height  = FIXED_WIDTH / width, FIXED_HEIGHT/height
 
-        text = gloss
-        font = cv2.FONT_HERSHEY_SIMPLEX
-        font_scale = 1
-        font_color = (0, 255, 0)
-        thickness = 2
-        line_type = cv2.LINE_AA
-
+        # ---- Loop over each frame
+        #
         for frame_landmarks in video_landmarks[:-1]:
+            # ---- Initialize blank image and get all landmarks of the given frame.
+            #
             blank_image = np.zeros((FIXED_HEIGHT, FIXED_WIDTH, 3), dtype=np.uint8)
             frame_hands_landmarks = frame_landmarks['hands_landmarks']
             frame_pose_landmarks = frame_landmarks['pose_landmarks']
             frame_face_landmarks = frame_landmarks['face_landmarks']
 
-            #left_hand_landmarks_xy = [(x, y) for x, y in frame_hands_landmarks['left_hand'].values()]
-            #right_hand_landmarks_xy = [(x, y) for x, y in frame_hands_landmarks['right_hand'].values()]
-
-            #for x, y in left_hand_landmarks_xy:
-            #    cv2.circle(blank_image, (x, y), 1, (255, 255, 255), -1)
-            #for x, y in right_hand_landmarks_xy:
-            # cv2.circle(blank_image, (x, y), 1, (255, 255, 255), -1)
-
-            # pose_landmarks_xy = [(x, y) for x, y in frame_pose_landmarks.values()]
-            # for x, y in pose_landmarks_xy:
-            #     cv2.circle(blank_image, (x, y), 1, (255, 255, 255), -1)
-
-            # face_landmarks_xy = [(x, y) for x, y in frame_face_landmarks.values()]
-            # for x, y in face_landmarks_xy:
-            #     cv2.circle(blank_image, (x, y), 1, (255, 255, 255), -1)
+            # ---- Resize landmarks.
+            #
             frame_hands_landmarks_rs = {
                             'left_hand': resize_landmarks(frame_hands_landmarks['left_hand'], resize_rate_width, resize_rate_height),
                             'right_hand': resize_landmarks(frame_hands_landmarks['right_hand'], resize_rate_width, resize_rate_height)
                                         }
             frame_pose_landmarks_rs = resize_landmarks(frame_pose_landmarks, resize_rate_width, resize_rate_height)
             frame_face_landmarks_rs = resize_landmarks(frame_face_landmarks, resize_rate_width, resize_rate_height)
+            
+            # ---- Draw relevant connections between keypoints on the frame.
+            #
             draw_hands_connections(blank_image, frame_hands_landmarks_rs)
             draw_pose_connections(blank_image, frame_pose_landmarks_rs)
             draw_face_connections(blank_image, frame_face_landmarks_rs)
 
-            text_size, _ = cv2.getTextSize(text, font, font_scale, thickness)
+            # ---- Display text corresponding to the gloss on the frame.
+            #
+            text_size, _ = cv2.getTextSize(gloss, font, font_scale, thickness)
             text_x = (FIXED_WIDTH - text_size[0]) // 2
             text_y = FIXED_HEIGHT - 10
-            cv2.putText(blank_image, text, (text_x, text_y), font, font_scale, font_color, thickness, line_type)
+            cv2.putText(blank_image, gloss, (text_x, text_y), font, font_scale, font_color, thickness, line_type)
             
-            # Convertir l'image en JPEG encodé
+             # ---- JPEG-encode the frame for streaming.
+            #
             _, buffer = cv2.imencode('.jpg', blank_image)
             frame = buffer.tobytes()
 
             yield (b'--frame\r\n'
                    b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
 
-            time.sleep(1 / fps)
+            time.sleep(1 / FPS)
+
 
+def load_data(dataset_path='enhanced_dataset'):
+    '''
+    Load the dataset that contains all information about glosses.
+
+    Parameters
+    ----------
+    dataset_path: str
+        Local path to the dataset.
+
+    Returns
+    -------
+    data_df: pandas.DataFrame
+        DataFrame containing the dataset with information about each gloss.
+    vocabulary_list: list of str
+        List of glosses (tokens) that have associated landmarks collected.
+    '''
 
-def load_data(dataset_path='local_dataset'):
     filepath = dataset_path
     data_df = pd.read_csv(filepath, dtype={'video_id': str})
     vocabulary_list = data_df['gloss'].tolist()
+
     return data_df, vocabulary_list
 
 
 def check_gloss_in_vocabulary(gloss, vocabulary_list):
+    '''
+    Check if the given gloss is in the vocabulary list.
+
+    Parameters
+    ----------
+    gloss: str
+        The gloss to check.
+    vocabulary_list: list of str
+        List of glosses (tokens) that have associated landmarks collected.
+
+    Returns
+    -------
+    bool
+        True if the gloss is in the vocabulary list, False otherwise.
+    '''
+
     return gloss in vocabulary_list
 
+
 def select_video_id_from_gloss(gloss, dataset):
+    '''
+    Selects a video ID corresponding to the given gloss from the dataset.
+
+    Parameters
+    ----------
+    gloss : str
+        The gloss for which to retrieve the video ID.
+    dataset : pandas.DataFrame
+        A DataFrame containing information about each gloss, including 'signer_id', 'gloss', and 'video_id'.
+
+    Returns
+    -------
+    int
+        The video ID corresponding to the given gloss. If the gloss is found for 'signer_id' 11, the video ID for that signer is returned; otherwise, the video ID for the gloss from the entire dataset is returned.
+    '''
+    # ---- Choose preferentialy ID 11 because this signer with this ID signed the more video
+    #
     filtered_data_id_11 = dataset.loc[dataset['signer_id'] == 11]
+
     if gloss in filtered_data_id_11['gloss'].tolist():
         video_id = filtered_data_id_11.loc[filtered_data_id_11['gloss'] == gloss, 'video_id'].values
     else:
         video_id = dataset.loc[dataset['gloss'] == gloss, 'video_id'].values
+
     return video_id[0]

src/main.py

@@ -1,40 +1,60 @@
 import display_gloss as dg
-import numpy as np
 import synonyms_preprocess as sp
 from NLP_Spacy_base_translator import NlpSpacyBaseTranslator 
-from flask import Flask,  render_template, Response, request
-
+from flask import Flask, render_template, Response, request
 
+# ---- Initialise Flask App
+#
 app = Flask(__name__)
 
-
+# ---- Render the homepage template
+#
 @app.route('/')
 def index():
+
     return render_template('index.html')
 
+# ---- Translate english input sentence into gloss sentence
+#
 @app.route('/translate/', methods=['POST'])
 def result():
+
+    # ---- Load NLP models and data
+    #
     nlp, dict_docs_spacy = sp.load_spacy_values()
     _, list_2000_tokens = dg.load_data()
 
     if request.method == 'POST':
+
+        # ---- Get the raw sentence and translate it to gloss
+        #
         sentence = request.form['inputSentence']
         eng_to_asl_translator = NlpSpacyBaseTranslator(sentence=sentence)
         generated_gloss = eng_to_asl_translator.translate_to_gloss()
         gloss_list_lower = [gloss.lower() for gloss in generated_gloss.split() if gloss.isalnum() ]
         gloss_sentence_before_synonym = " ".join(gloss_list_lower)
+
+        # ---- Substitute gloss tokens with synonyms if not in the common token list
+        #
         gloss_list = [sp.find_synonyms(gloss, nlp, dict_docs_spacy, list_2000_tokens) for gloss in gloss_list_lower]
         gloss_sentence_after_synonym  = " ".join(gloss_list)
+
+        # ---- Render the result template with both versions of the gloss sentence
+        #
         return render_template('translate.html',\
                                 sentence=sentence,\
                                 gloss_sentence_before_synonym=gloss_sentence_before_synonym,\
                                 gloss_sentence_after_synonym=gloss_sentence_after_synonym)
 
+# ---- Generate video streaming from gloss_sentence
+#
 @app.route('/video_feed')
 def video_feed():
+
     dataset, list_2000_tokens = dg.load_data()
     sentence = request.args.get('gloss_sentence_to_display', '')
     gloss_list = sentence.split()
+
     return Response(dg.generate_video(gloss_list, dataset, list_2000_tokens), mimetype='multipart/x-mixed-replace; boundary=frame')
 
 if __name__ == "__main__":

src/synonyms_preprocess.py

@@ -4,10 +4,34 @@ from nltk.corpus import wordnet
 
 
 def load_spacy_values(filepath_model_spacy='model_spacy_synonyms', filepath_docs_spacy = 'dict_spacy_object.pkl'):
+    '''
+    Loads a spaCy model and a dictionary of spaCy Doc objects from a pickle file.
 
-    nlp = spacy.load(filepath_model_spacy)
+    Parameters
+    ----------
+    filepath_model_spacy : str
+        The local path to the spaCy model used for synonym detection.
+
+    filepath_docs_spacy : str
+        The local path to the pickle file containing a dictionary where the keys are tokens 
+        and the values are the corresponding spaCy Doc objects serialized as bytes.
+
+    Returns
+    -------
+    nlp : spacy.language.Language
+        The loaded spaCy language model.
 
+    dict_docs_spacy : dict
+        A dictionary where the keys are tokens (str) and the values are spaCy Doc objects, 
+        reconstructed from the serialized bytes.
+    '''
     
+    # ---- Load the spaCy NLP model
+    #
+    nlp = spacy.load(filepath_model_spacy)
+    
+    # ---- Load pickle file and reconstruct the dictionary with tokens as keys and spaCy Doc objects as values
+    #
     with open(filepath_docs_spacy, 'rb') as file:
         dict_docs_spacy_bytes = pickle.load(file)
     
@@ -15,33 +39,86 @@ def load_spacy_values(filepath_model_spacy='model_spacy_synonyms', filepath_docs
     
     return nlp, dict_docs_spacy
 
+
 def find_antonyms(word):
+    '''
+    Generate a set of all the antonyms of a given word
+
+    Parameters
+    ----------
+    word : str
+        The word that we want to find the antonyms
+
+    Returns
+    -------
+    antonyms : set of str
+        A set of all the antonym detected using nltk and WordNet
+    '''
+    
     antonyms = set()
+
+    # ---- Load all the set of synonyms of the word recorded from wordnet
+    #
     syn_set = wordnet.synsets(word)
+
+    # ---- Loop over each set of synonyms
+    #
     for syn in syn_set:
+        # ---- Loop over each synonym
+        #
         for lemma in syn.lemmas():
+            # ---- Add antonyms of the synonyms to the antonyms set
+            #
             if lemma.antonyms():
                 antonyms.add(lemma.antonyms()[0].name())
+
     return antonyms
 
-def find_synonyms(word, model, dict_embedding, dict_2000_tokens): #cluster_to_words, dbscan_model):
-    """
-    This function finds the most similar word in the same cluster, and excludes antonyms
-    """
 
-    if word in dict_2000_tokens:
+def find_synonyms(word, model, dict_embedding, list_2000_tokens):
+    '''
+    Finds the most similar token to a given word.
+
+    Parameters
+    ----------
+    word : str
+        The word that we want to find the most similar word
+
+    model : spacy.language.Language
+        spaCy language model to use for the detection of the synonym
+    
+    dict_embedding: dict
+        A dictionary where the keys are tokens (str) and the values are spaCy Doc objects
+
+    list_2000_tokens : list of str
+        A list of 2000 tokens against which the gloss will be checked.
+    
+    Returns
+    -------
+    most_similar_token : str
+        The most similar token to the given word 
+    '''
+
+    # ---- Skip synonym detection if the word is already in the list_2000_token
+    #
+    if word in list_2000_tokens:
         return word
     else:
+        # ---- Remove antonyms of the given word of the list_2000_tokens (a word and an antonym might be similar in embedding representation)
+        #
         antonyms = find_antonyms(word)
-        dict_2000_tokens_less_antonyms = [token for token in dict_2000_tokens if token not in antonyms]
+        list_2000_tokens_less_antonyms = [token for token in list_2000_tokens if token not in antonyms]
 
+        # ---- Generate a list of tuple (token, similarities values between the embedding of the given word and the embedding of each token of the list_2000_tokens)
+        #
         word_embedding = model(word)
-
         similarities=[]
     
-        for token in dict_2000_tokens_less_antonyms:
+        for token in list_2000_tokens_less_antonyms:
             similarities.append((token, dict_embedding.get(token).similarity(word_embedding)))
-    
+
+        # ---- Extract the most similar token of the list
+        #
         most_similar_token = sorted(similarities, key=lambda item: -item[1])[0][0]
 
         return most_similar_token