Update app.py

app.py

@@ -2,11 +2,20 @@ from gradio import Interface
 import gradio as gr
 import aranizer
 from aranizer import aranizer_bpe50k, aranizer_bpe64k, aranizer_bpe86k, aranizer_sp32k, aranizer_sp50k, aranizer_sp64k, aranizer_sp86k
+from transformers import AutoTokenizer
+
+# Load additional tokenizers from transformers
+gpt_13b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-13B")
+gpt_7b_tokenizer = AutoTokenizer.from_pretrained("FreedomIntelligence/AceGPT-7B")
+jais_13b_tokenizer = AutoTokenizer.from_pretrained("inception-mbzuai/jais-13b")
 
 # List of available tokenizers and a dictionary to load them
 tokenizer_options = [
     "aranizer_bpe50k", "aranizer_bpe64k", "aranizer_bpe86k",
-    "aranizer_sp32k", "aranizer_sp50k", "aranizer_sp64k", "aranizer_sp86k"
+    "aranizer_sp32k", "aranizer_sp50k", "aranizer_sp64k", "aranizer_sp86k",
+    "FreedomIntelligence/AceGPT-13B",  # Previously added GPT tokenizer
+    "FreedomIntelligence/AceGPT-7B",   # Another previously added GPT tokenizer
+    "inception-mbzuai/jais-13b"        # Adding the new tokenizer to the options
 ]
 
 tokenizers = {
@@ -17,33 +26,49 @@ tokenizers = {
     "aranizer_sp50k": aranizer_sp50k.get_tokenizer,
     "aranizer_sp64k": aranizer_sp64k.get_tokenizer,
     "aranizer_sp86k": aranizer_sp86k.get_tokenizer,
+    "FreedomIntelligence/AceGPT-13B": lambda: gpt_13b_tokenizer,
+    "FreedomIntelligence/AceGPT-7B": lambda: gpt_7b_tokenizer,
+    "inception-mbzuai/jais-13b": lambda: jais_13b_tokenizer  # Adding the new Jais tokenizer
 }
 
 def compare_tokenizers(tokenizer_name, text):
-    # Load the selected tokenizer
-    tokenizer = tokenizers[tokenizer_name]()
-    tokens = tokenizer.tokenize(text)
-    encoded_output = tokenizer.encode(text, add_special_tokens=True)
-    decoded_text = tokenizer.decode(encoded_output)
+    # Handle the transformer tokenizers separately due to API differences
+    if tokenizer_name in ["FreedomIntelligence/AceGPT-13B", "FreedomIntelligence/AceGPT-7B", "inception-mbzuai/jais-13b"]:
+        tokenizer = tokenizers[tokenizer_name]()
+        tokens = tokenizer.tokenize(text)
+        encoded_output = tokenizer.encode(text, add_special_tokens=True, return_tensors="pt")
+        decoded_text = tokenizer.decode(encoded_output[0], skip_special_tokens=True)
+    else:
+        # AraNizer tokenizers
+        tokenizer = tokenizers[tokenizer_name]()
+        tokens = tokenizer.tokenize(text)
+        encoded_output = tokenizer.encode(text, add_special_tokens=True)
+        decoded_text = tokenizer.decode(encoded_output)
     
     # Prepare the results to be displayed
     results = [(tokenizer_name, tokens, encoded_output, decoded_text)]
     return results
+
+# Define the Gradio interface components with a dropdown for model selection
 inputs_component = [
     gr.Dropdown(choices=tokenizer_options, label="Select Tokenizer"),
-    gr.Textbox(lines=2, placeholder="Enter Arabic text here...", label="Input Text")
+    gr.Textbox(lines=2, placeholder="Enter text here...", label="Input Text")
 ]
 
-# Define the outputs component normally without the 'css' parameter
-outputs_component = gr.Dataframe(headers=["Tokenizer", "Tokens", "Encoded Output", "Decoded Text"], label="Results")
-
-
+outputs_component = gr.Dataframe(
+    headers=["Tokenizer", "Tokens", "Encoded Output", "Decoded Text"], 
+    label="Results",
+    type="pandas"
+)
 
-# Setting up the interface with the custom theme
-iface = Interface(fn=compare_tokenizers,
-                  inputs=inputs_component,
-                  outputs=outputs_component,
-                  title="AraNizer Tokenizer Comparison")
+# Setting up the interface
+iface = Interface(
+    fn=compare_tokenizers, 
+    inputs=inputs_component, 
+    outputs=outputs_component, 
+    title="Tokenizer Comparison",
+    live=True
+)
 
 # Launching the Gradio app
-iface.launch()
+iface.launch()