landmarks animation

app.py

@@ -12,7 +12,7 @@ The text is preprocessed, tokenized and rearranged and then each token is mapped
 
 > *NOTE*: This model only supports a fixed vocabulary. See the [`*-dictionary-mapping.json`](https://github.com/sign-language-translator/sign-language-datasets/tree/main/parallel_texts) files for supported words.
 > This version needs to re-encode the generated video so that will take some extra time after translation.
-> Since this is a rule-based model, you will have to add *context* to ambiguous words (e.g. glass(material) vs glass(container)).
+> Since this is a rule-based model, you will have to add **context** to ambiguous words (e.g. glass(material) vs glass(container)).
 """.strip()
 
 TITLE = "Concatenative Synthesis: Rule Based Text to Sign Language Translator"
@@ -79,19 +79,26 @@ def text_to_video(
     text: str,
     text_language: str,
     sign_language: str,
+    sign_format: str = "video",
     output_path: str = "output.mp4",
     codec="h264",  # ToDo: install h264 codec for opencv
 ):
     translation_model.text_language = text_language
     translation_model.sign_language = sign_language
+    translation_model.sign_format = sign_format
+    if sign_format == "landmarks":
+        translation_model.sign_embedding_model = "mediapipe-world"
 
-    video = translation_model.translate(text)
-    video.save(output_path, overwrite=True, codec=codec)
+    sign = translation_model.translate(text)
+    if isinstance(sign, slt.Landmarks):
+        sign.save_animation(output_path, overwrite=True)
+    else:
+        sign.save(output_path, overwrite=True, codec=codec)
 
     # ToDo: video.watermark("Sign Language Translator\nAI Generated Video")
 
 
-def translate(text: str, text_lang: str, sign_lang: str):
+def translate(text: str, text_lang: str, sign_lang: str, sign_format: str):
     log = [
         text,
         text_lang,
@@ -101,7 +108,14 @@ def translate(text: str, text_lang: str, sign_lang: str):
     ]
     try:
         path = "output.mp4"
-        text_to_video(text, text_lang, sign_lang, output_path=path, codec="mp4v")
+        text_to_video(
+            text,
+            text_lang,
+            sign_lang,
+            sign_format=sign_format,
+            output_path=path,
+            codec="mp4v",
+        )
         request_logger.flag(log)
         return path
 
@@ -114,11 +128,11 @@ def translate(text: str, text_lang: str, sign_lang: str):
 with gradio.Blocks(title=TITLE, head=CUSTOM_JS, css=CUSTOM_CSS) as gradio_app:
     gradio.Markdown(f"# {TITLE}")
     gradio.Markdown(DESCRIPTION)
-    with gradio.Row():
-        with gradio.Column():
+    with gradio.Row():  # Inputs and Outputs
+        with gradio.Column():  # Inputs
             gradio.Markdown("## Input Text")
             with gradio.Row():
-                with gradio.Column():
+                with gradio.Column():  # Source TextArea
                     gradio.Markdown("Write here (in selected language):")
                     source_textbox = gradio.Textbox(
                         lines=1,
@@ -127,7 +141,7 @@ with gradio.Blocks(title=TITLE, head=CUSTOM_JS, css=CUSTOM_CSS) as gradio_app:
                         show_copy_button=True,
                         elem_id="source-textbox",
                     )
-                with gradio.Column():
+                with gradio.Column():  # Language Model
                     gradio.Markdown("Generate sample text instead:")
                     with gradio.Row():
                         language_model_dropdown = gradio.Dropdown(
@@ -168,9 +182,17 @@ with gradio.Blocks(title=TITLE, head=CUSTOM_JS, css=CUSTOM_CSS) as gradio_app:
                     value=slt.SignLanguageCodes.PAKISTAN_SIGN_LANGUAGE.value,
                     label="Sign Language",
                 )
+                output_format_dropdown = gradio.Dropdown(
+                    choices=[
+                        slt.SignFormatCodes.VIDEO.value,
+                        slt.SignFormatCodes.LANDMARKS.value,
+                    ],
+                    value=slt.SignFormatCodes.VIDEO.value,
+                    label="Output Format",
+                )
                 # todo: sign format: video/landmarks (tabs?)
 
-        with gradio.Column():
+        with gradio.Column():  # Outputs
             gradio.Markdown("## Output Sign Language")
             output_video = gradio.Video(
                 format="mp4",
@@ -180,11 +202,16 @@ with gradio.Blocks(title=TITLE, head=CUSTOM_JS, css=CUSTOM_CSS) as gradio_app:
                 include_audio=False,
             )
 
-    with gradio.Row():
+    with gradio.Row():  # Translate Button
         translate_button = gradio.Button("Translate", variant="primary")
         translate_button.click(
             translate,
-            inputs=[source_textbox, text_lang_dropdown, sign_lang_dropdown],
+            inputs=[
+                source_textbox,
+                text_lang_dropdown,
+                sign_lang_dropdown,
+                output_format_dropdown,
+            ],
             outputs=[output_video],
             api_name="translate",
         )

requirements.txt

@@ -1,2 +1,2 @@
-sign-language-translator
+sign-language-translator==0.8.*
 opencv-python