import os
import re
from datetime import datetime
from typing import Dict
import gradio
import sign_language_translator as slt
DESCRIPTION = """Enter your text and select languages from the dropdowns, then click Submit to generate a video. [`Library Repository`](https://github.com/sign-language-translator/sign-language-translator)
The text is preprocessed, tokenized and rearranged and then each token is mapped to a prerecorded video which are concatenated and returned. [`Model Code`](https://github.com/sign-language-translator/sign-language-translator/blob/main/sign_language_translator/models/text_to_sign/concatenative_synthesis.py)
> **NOTE**
> - This model only supports a fixed vocabulary. See the [`*-dictionary-mapping.json`](https://github.com/sign-language-translator/sign-language-datasets/tree/main/parallel_texts) files for supported words.
> - This version needs to re-encode the generated video so that will take some extra time after translation.
> - Since this is a rule-based model, you will have to add **context** to ambiguous words (e.g. glass(material) vs glass(container)).
> - Some signs correspond to words very specific in a particular language so their mapping in other languages will not make sense (e.g. in pakistan-sign-language, signs were recorded in reference to common Urdu words, hence English words "for" & "to" etc do not map well to their original Urdu words "کے لئے" and "کو" etc).
""".strip()
TITLE = "Concatenative Synthesis: Rule Based Text to Sign Language Translator"
CUSTOM_JS = """<script>
const rtlLanguages = ["urdu", "arabic"];
const keyMap = {
"urdu": {
"1": "۱",
"2": "۲",
"3": "۳",
"4": "۴",
"5": "۵",
"6": "٦",
"7": "۷",
"8": "۸",
"9": "۹",
"0": "۰",
"q": "ق",
"w": "و",
"e": "ع",
"r": "ر",
"t": "ت",
"y": "ے",
"u": "ء",
"i": "ی",
"o": "ہ",
"p": "پ",
"a": "ا",
"s": "س",
"d": "د",
"f": "ف",
"g": "گ",
"h": "ح",
"j": "ج",
"k": "ک",
"l": "ل",
"z": "ز",
"x": "ش",
"c": "چ",
"v": "ط",
"b": "ب",
"n": "ن",
"m": "م",
"R": "ڑ",
"T": "ٹ",
"Y": "َ",
"U": "ئ",
"I": "ِ",
"P": "ُ",
"A": "آ",
"S": "ص",
"D": "ڈ",
"F": "أ",
"G": "غ",
"H": "ھ",
"J": "ض",
"K": "خ",
"Z": "ذ",
"X": "ژ",
"C": "ث",
"V": "ظ",
"N": "ں",
",": "،",
".": "۔",
"?": "؟",
";": "؛",
},
"hindi": {
"1": "१",
"2": "२",
"3": "३",
"4": "४",
"5": "५",
"6": "६",
"7": "७",
"8": "८",
"9": "९",
"0": "०",
"=": "ृ",
"!": "ऍ",
"@": "ॅ",
"#": "्र",
"$": "र्",
"%": "ज्ञ",
"^": "त्र",
"&": "क्ष",
"*": "श्र",
"_": "ः",
"+": "ऋ",
"q": "ौ",
"w": "ै",
"e": "ा",
"r": "ी",
"t": "ू",
"y": "ब",
"u": "ह",
"i": "ग",
"o": "द",
"p": "ज",
"[": "ड",
"]": "़",
'\\\\': "ॉ",
"Q": "औ",
"W": "ऐ",
"E": "आ",
"R": "ई",
"T": "ऊ",
"Y": "भ",
"U": "ङ",
"I": "घ",
"O": "ध",
"P": "झ",
"{": "ढ",
"}": "ञ",
"|": "ऑ",
"a": "ो",
"s": "े",
"d": "्",
"f": "ि",
"g": "ु",
"h": "प",
"j": "र",
"k": "क",
"l": "त",
";": "च",
"'": "ट",
"A": "ओ",
"S": "ए",
"D": "अ",
"F": "इ",
"G": "उ",
"H": "फ",
"J": "ऱ",
"K": "ख",
"L": "थ",
":": "छ",
'"': "ठ",
"z": "ॆ",
"x": "ं",
"c": "म",
"v": "न",
"b": "व",
"n": "ल",
"m": "स",
".": "।",
"/": "य",
"Z": "ऎ",
"X": "ँ",
"C": "ण",
"V": "ऩ",
"B": "ऴ",
"N": "ळ",
"M": "श",
"<": "ष",
">": "य़",
// "?":"य़",
}
};
function updateTextareaDir(language) {
const sourceTextarea = document.getElementById("source-textbox").querySelector("textarea");
if (rtlLanguages.includes(language)) {
sourceTextarea.setAttribute("dir", "rtl");
} else {
sourceTextarea.setAttribute("dir", "ltr");
}
function keypressHandler(event) {
const key = event.key;
if (keyMap[language].hasOwnProperty(key)) {
event.preventDefault();
const mappedValue = keyMap[language][key];
const start = sourceTextarea.selectionStart;
const end = sourceTextarea.selectionEnd;
sourceTextarea.value = sourceTextarea.value.slice(0, start) + mappedValue + sourceTextarea.value.slice(end);
sourceTextarea.selectionStart = sourceTextarea.selectionEnd = start + mappedValue.length;
}
}
sourceTextarea.removeEventListener("keypress", sourceTextarea.keypressHandler);
sourceTextarea.addEventListener("keypress", keypressHandler);
// Save the handler function to the textarea element for future removal
sourceTextarea.keypressHandler = keypressHandler;
}
</script>
"""
# todo: add dropdown keyboard custom component with key mapping
# todo: output full height
CUSTOM_CSS = """
.reverse-row {
flex-direction: row-reverse;
}
#auto-complete-button {
border-color: var(--button-primary-border-color-hover);
}
"""
HF_TOKEN = os.getenv("HF_TOKEN")
request_logger = (
gradio.HuggingFaceDatasetSaver(
HF_TOKEN,
"sltAI/crowdsourced-text-to-sign-language-rule-based-translation-corpus",
)
if HF_TOKEN
else gradio.CSVLogger()
)
translation_model = slt.models.ConcatenativeSynthesis("ur", "pk-sl", "video")
language_models: Dict[str, slt.models.BeamSampling] = {}
full_to_short = {
"english": "en",
"urdu": "ur",
"hindi": "hi",
}
short_to_full = {s: f for f, s in full_to_short.items()}
def auto_complete_text(model_code: str, text: str):
if model_code not in language_models:
lm = slt.get_model(model_code)
language_models[model_code] = slt.models.BeamSampling(
lm, # type: ignore
start_of_sequence_token=getattr(lm, "start_of_sequence_token", "<"), # type: ignore
end_of_sequence_token=getattr(lm, "end_of_sequence_token", ">"), # type: ignore
)
# todo: better tokenize/detokenize
tokens = [w for w in re.split(r"\b", text) if w]
lm = language_models[model_code]
lm.max_length = len(tokens) + 10
completion, _ = lm.complete(tokens or None)
if completion[0] == lm.start_of_sequence_token: # type: ignore
completion = completion[1:] # type: ignore
if completion[-1] == lm.end_of_sequence_token: # type: ignore
completion = completion[:-1] # type: ignore
new_text = "".join(completion)
return new_text
def text_to_video(
text: str,
text_language: str,
sign_language: str,
sign_format: str = "video",
output_path: str = "output.mp4",
codec="h264", # ToDo: install h264 codec for opencv
):
translation_model.text_language = text_language
translation_model.sign_language = sign_language
translation_model.sign_format = sign_format
if sign_format == "landmarks":
translation_model.sign_embedding_model = "mediapipe-world"
sign = translation_model.translate(text)
if isinstance(sign, slt.Landmarks):
# large hands on sides
# sign.data[:, 33:] *= 2
# sign.data[:, 33:54, 0] += 0.25
# sign.data[:, 54:, 0] -= 0.25
# hands moved to pose wrists
sign.data[:, 33:54, :3] += -sign.data[:, 33:34, :3] + sign.data[:, 15:16, :3]
sign.data[:, 54: , :3] += -sign.data[:, 54:55, :3] + sign.data[:, 16:17, :3]
sign.save_animation(output_path, overwrite=True)
else:
sign.save(output_path, overwrite=True, codec=codec)
# ToDo: video.watermark("Sign Language Translator\nAI Generated Video")
def translate(text: str, text_lang: str, sign_lang: str, sign_format: str):
text_lang = full_to_short.get(text_lang, text_lang)
log = [
text,
text_lang,
sign_lang,
None,
datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f"),
]
try:
if text_lang == "en":
text = text[:1].lower() + text[1:]
path = "output.mp4"
text_to_video(
text,
text_lang,
sign_lang,
sign_format=sign_format,
output_path=path,
codec="mp4v",
)
request_logger.flag(log)
return path
except Exception as exc:
log[3] = str(exc)
request_logger.flag(log)
raise gradio.Error(f"Error during translation: {exc}")
with gradio.Blocks(title=TITLE, head=CUSTOM_JS, css=CUSTOM_CSS) as gradio_app:
gradio.Markdown(f"# {TITLE}")
gradio.Markdown(DESCRIPTION)
with gradio.Row(elem_classes=["reverse-row"]): # Inputs and Outputs
with gradio.Column(): # Outputs
gradio.Markdown("## Output Sign Language")
output_video = gradio.Video(
format="mp4",
label="Synthesized Sign Language Video",
autoplay=True,
show_download_button=True,
include_audio=False,
)
with gradio.Column(): # Inputs
gradio.Markdown("## Select Languages")
with gradio.Row():
text_lang_dropdown = gradio.Dropdown(
choices=[
short_to_full.get(code.value, code.value)
for code in slt.TextLanguageCodes
],
value=short_to_full.get(
slt.TextLanguageCodes.URDU.value,
slt.TextLanguageCodes.URDU.value,
),
label="Text Language",
elem_id="text-lang-dropdown",
)
text_lang_dropdown.change(
None, inputs=text_lang_dropdown, js="updateTextareaDir"
)
sign_lang_dropdown = gradio.Dropdown(
choices=[code.value for code in slt.SignLanguageCodes],
value=slt.SignLanguageCodes.PAKISTAN_SIGN_LANGUAGE.value,
label="Sign Language",
)
output_format_dropdown = gradio.Dropdown(
choices=[
slt.SignFormatCodes.VIDEO.value,
slt.SignFormatCodes.LANDMARKS.value,
],
value=slt.SignFormatCodes.VIDEO.value,
label="Output Format",
)
# todo: sign format: video/landmarks (tabs?)
gradio.Markdown("## Input Text")
with gradio.Row(): # Source TextArea
source_textbox = gradio.Textbox(
lines=4,
placeholder="Enter Text Here...",
label="Spoken Language Sentence",
show_copy_button=True,
elem_id="source-textbox",
)
with gradio.Row(): # clear/auto-complete/Language Model
language_model_dropdown = gradio.Dropdown(
choices=[
slt.ModelCodes.MIXER_LM_NGRAM_URDU.value,
slt.ModelCodes.TRANSFORMER_LM_UR_SUPPORTED.value,
],
value=slt.ModelCodes.MIXER_LM_NGRAM_URDU.value,
label="Select language model to Generate sample text",
)
auto_complete_button = gradio.Button(
"Auto-Complete", elem_id="auto-complete-button"
)
auto_complete_button.click(
auto_complete_text,
inputs=[language_model_dropdown, source_textbox],
outputs=[source_textbox],
api_name=False,
)
clear_button = gradio.ClearButton(source_textbox, api_name=False)
with gradio.Row(): # Translate Button
translate_button = gradio.Button("Translate", variant="primary")
translate_button.click(
translate,
inputs=[
source_textbox,
text_lang_dropdown,
sign_lang_dropdown,
output_format_dropdown,
],
outputs=[output_video],
api_name="translate",
)
gradio.Examples(
[
["We are here to use this.", "english", "pakistan-sign-language", "video"],
["i(me) admire art.", "english", "pakistan-sign-language", "landmarks"],
["یہ بہت اچھا ہے۔", "urdu", "pakistan-sign-language", "video"],
["وہ کام آسان تھا۔", "urdu", "pakistan-sign-language", "landmarks"],
["कैसे हैं आप?", "hindi", "pakistan-sign-language", "video"],
["पाँच घंटे।", "hindi", "pakistan-sign-language", "landmarks"],
],
inputs=[
source_textbox,
text_lang_dropdown,
sign_lang_dropdown,
output_format_dropdown,
],
outputs=output_video,
)
request_logger.setup(
[
source_textbox,
text_lang_dropdown,
sign_lang_dropdown,
gradio.Markdown(label="Exception"),
gradio.Markdown(label="Timestamp"),
],
"flagged",
)
gradio_app.load(None, inputs=[text_lang_dropdown], js="updateTextareaDir")
if __name__ == "__main__":
gradio_app.launch()