Refactor tokenize_text function to include better tokenization for Arabic Text

app.py

@@ -133,28 +133,24 @@ def decode_bpe_tokens(tokens):
         fixed_tokens.append(fixed_token)
     return fixed_tokens
 
-def decode_arabic_tokens(tokens):
-    decoded_tokens = []
-    for token in tokens:
-        decoded_token = token.encode('latin-1', 'backslashreplace').decode('unicode-escape')
-        decoded_tokens.append(decoded_token)
-    return decoded_tokens
-
-def tokenize_text(text, chosen_model):
+def tokenize_text(text, chosen_model, better_tokenization=False):
     tokenizer = AutoTokenizer.from_pretrained(chosen_model)
     tokenized_text = decode_bpe_tokens(tokenizer.tokenize(text))
     random_colors = generate_distinct_colors(len(tokenized_text))
 
-    final_tokenized_text = []
-    for token in tokenized_text:
-        correct_tokenized_text = ""
-        for char in text:
-            correct_tokenized_text += char 
-            current_token = decode_bpe_tokens(tokenizer.tokenize(correct_tokenized_text))
-            if current_token[0] == token:
-                final_tokenized_text.append(correct_tokenized_text)
-                text = text[len(correct_tokenized_text):]
-                break
+    if better_tokenization:
+        final_tokenized_text = []
+        for token in tokenized_text:
+            correct_tokenized_text = ""
+            for char in text:
+                correct_tokenized_text += char 
+                current_token = decode_bpe_tokens(tokenizer.tokenize(correct_tokenized_text))
+                if current_token[0] == token:
+                    final_tokenized_text.append(correct_tokenized_text)
+                    text = text[len(correct_tokenized_text):]
+                    break
+    else:
+        final_tokenized_text = tokenized_text
     print(final_tokenized_text)
 
     output = []
@@ -199,11 +195,13 @@ with gr.Blocks() as demo:
             choices=df["📛 Models"].tolist(),
             value=df["📛 Models"].tolist()[0],
         )
-        submit_text_btn = gr.Button(value="Submit", variant="primary")
+        with gr.Row():
+            submit_text_btn = gr.Button(value="Submit", variant="primary", scale=3)
+            checkbox = gr.Checkbox(label="Better tokenization for Arabic Text", value=False, scale=1)
         tokenized_textbox = gr.HighlightedText(label="Tokenized text")
 
     submit_new_model_btn.click(submit, model_name, outputs=[dataframe, barplot, dropdown])
-    submit_text_btn.click(tokenize_text, inputs=[text, dropdown], outputs=[tokenized_textbox])
+    submit_text_btn.click(tokenize_text, inputs=[text, dropdown, checkbox], outputs=[tokenized_textbox])
 
 
 demo.launch()