Implement Gradio interface for Turkish Tokenizer, replacing Streamlit; update requirements to include Gradio.

app.py

@@ -1,255 +1,174 @@
-import atexit
-import base64
-import colorsys
-import importlib.util
-import shutil
-import sys
-from datetime import datetime
-from pathlib import Path
-
-import requests
-import streamlit as st
-
-# Set page config - MUST BE FIRST STREAMLIT COMMAND
-st.set_page_config(
-    page_title="Turkish Tiktokenizer",
-    page_icon="🇹🇷",
-    layout="wide"
-)
-
-# Initialize session state
-if 'text' not in st.session_state:
-    st.session_state.text = "Akademisyenler ve aileleri birlikte çalışıyorlar."
-if 'token_results' not in st.session_state:
-    st.session_state.token_results = None
-
-# Constants
-GITHUB_REPO = "malibayram/tokenizer"
-GITHUB_BRANCH = "main"
-
-# Special tokens and their IDs
-SPECIAL_TOKENS = {
-    "<uppercase>": 0,    # Uppercase letter marker
-    "<space>": 1,       # Space character
-    "<newline>": 2,     # Newline character
-    "<tab>": 3,         # Tab character
-    "<unknown>": 4      # Unknown token
+import gradio as gr
+
+# Assuming tr_tokenizer.py contains both TRTokenizer and TokenType
+# and that it correctly imports TRDecoder from tr_decoder.py.
+# Make sure tr_tokenizer.py, tr_decoder.py, and your .json files
+# are in the same directory as this app.py file.
+from tr_tokenizer import TokenType, TRTokenizer
+
+# --- Gradio App ---
+
+# Instantiate the tokenizer
+# This will now load directly from your existing .json files
+# as defined in your TRTokenizer class.
+tokenizer = TRTokenizer()
+
+# Define colors for each token type (dark theme)
+dark_color_map = {
+    TokenType.ROOT.name: "#FF6B6B",      # Darker Red
+    TokenType.SUFFIX.name: "#4ECDC4",    # Teal
+    TokenType.BPE.name: "#FFE66D",       # Darker Yellow
 }
 
-# Special token display symbols
-SPECIAL_TOKEN_SYMBOLS = {
-    "<uppercase>": "[uppercase]",    # Up arrow for uppercase
-    "<space>": "[space]",        # Space symbol
-    "<newline>": "[newline]",      # Return symbol
-    "<tab>": "[tab]",          # Tab symbol
-    "<unknown>": "[unknown]"       # Question mark for unknown
-}
+def tokenize_and_display(text, theme="light"):
+    """
+    Tokenizes the input text and prepares it for display in Gradio's HighlightedText component.
+    """
+    if not text:
+        # Return a structure that matches all outputs to avoid errors
+        return [], "", "", "", theme
+
+    tokens, _ = tokenizer.tokenize_text(text)
+
+    # Create the list of (token, label) for HighlightedText
+    highlighted_tokens = []
+    token_stats = {"ROOT": 0, "SUFFIX": 0, "BPE": 0}
+
+    for t in tokens:
+        token_text = t["token"]
+        token_type = t["type"].name
+
+        # Count token types for statistics
+        token_stats[token_type] = token_stats.get(token_type, 0) + 1
+
+        highlighted_tokens.append((token_text, token_type))
+
+    encoded_ids = tokenizer.encode(text)
+    decoded_text = tokenizer.decode(encoded_ids)
+
+    # Calculate statistics
+    total_tokens = len(tokens)
+    total_chars = len(text)
+    compression_ratio = (1 - total_tokens / total_chars) * 100 if total_chars > 0 else 0
+
+    # Define theme-specific colors for the stats block
+    bg_col, text_col, card_col, border_col = ('#2d3748', '#f7fafc', '#4a5568', '#718096')
+
+    # Create statistics HTML
+    stats_html = f"""
+    <div style="background:{bg_col};padding:20px;border-radius:12px;margin:20px 0;">
+        <h4 style="color:{text_col};margin-bottom:15px;">📊 Tokenization Statistics</h4>
+        <div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:15px;margin-bottom:20px;">
+            <div style="background:{card_col};padding:15px;border-radius:8px;text-align:center;border:1px solid {border_col};"><div style="font-size:24px;font-weight:bold;color:#3b82f6;">{total_chars}</div><div style="color:{'#64748b' if theme == 'light' else '#a0aec0'};font-size:14px;">Characters</div></div>
+            <div style="background:{card_col};padding:15px;border-radius:8px;text-align:center;border:1px solid {border_col};"><div style="font-size:24px;font-weight:bold;color:#10b981;">{total_tokens}</div><div style="color:{'#64748b' if theme == 'light' else '#a0aec0'};font-size:14px;">Tokens</div></div>
+            <div style="background:{card_col};padding:15px;border-radius:8px;text-align:center;border:1px solid {border_col};"><div style="font-size:24px;font-weight:bold;color:#f59e0b;">{compression_ratio:.1f}%</div><div style="color:{'#64748b' if theme == 'light' else '#a0aec0'};font-size:14px;">Compression</div></div>
+        </div>
+        <div>
+            <h5 style="color:{text_col};margin-bottom:10px;">Token Type Distribution:</h5>
+            <div style="display:flex;gap:15px;flex-wrap:wrap;">
+                <div style="background:#FFADAD;color:#2d3748;padding:8px 12px;border-radius:6px;font-size:14px;font-weight:600;">🔴 Roots: {token_stats['ROOT']}</div>
+                <div style="background:#A0C4FF;color:#2d3748;padding:8px 12px;border-radius:6px;font-size:14px;font-weight:600;">🔵 Suffixes: {token_stats['SUFFIX']}</div>
+                <div style="background:#FDFFB6;color:#2d3748;padding:8px 12px;border-radius:6px;font-size:14px;font-weight:600;">🟡 BPE: {token_stats['BPE']}</div>
+            </div>
+        </div>
+    </div>"""
+    return highlighted_tokens, str(encoded_ids), decoded_text, stats_html, theme
+
+# Custom CSS for better styling
+custom_css = """
+.gradio-container{font-family:'Inter',-apple-system,BlinkMacSystemFont,sans-serif;}
+.custom-button{background:linear-gradient(135deg,#667eea 0%,#764ba2 100%);border:none;border-radius:8px;padding:12px 24px;color:white;font-weight:600;transition:all .3s ease;}
+.custom-button:hover{transform:translateY(-2px);box-shadow:0 8px 25px rgba(0,0,0,.15);}
+.theme-toggle{background:linear-gradient(135deg,#f093fb 0%,#f5576c 100%);border:none;border-radius:50px;padding:10px 20px;color:white;font-weight:600;transition:all .3s ease;}
+.theme-toggle:hover{transform:scale(1.05);box-shadow:0 4px 15px rgba(0,0,0,.2);}
+.input-textbox{border-radius:12px!important;border:2px solid #e2e8f0!important;transition:all .3s ease;}
+.input-textbox:focus{border-color:#667eea!important;box-shadow:0 0 0 3px rgba(102,126,234,.1)!important;}
+.dark .gradio-container{background:#1a202c!important;}
+.dark .input-textbox{background:#2d3748!important;border-color:#4a5568!important;color:#f7fafc!important;}
+"""
+
+# Create the Gradio Interface
+with gr.Blocks(theme=gr.themes.Soft(), title="Turkish Tokenizer", css=custom_css) as demo:
+    with gr.Row():
+        with gr.Column(scale=3):
+            gr.Markdown("""
+                # Turkish Tokenizer
+                ### Advanced Turkish Text Tokenization with Visual Analysis
+                Enter text to see how it's tokenized. Tokens are color-coded by type.
+            """)
+
+    theme_state = gr.State("light")
+
+    input_text = gr.Textbox(
+        label="📝 Input Text",
+        placeholder="Merhaba Dünya, kitapları okumak güzeldir.",
+        lines=4,
+        elem_classes=["input-textbox"]
+    )
 
-# Colors for special tokens
-SPECIAL_COLORS = {
-    "<uppercase>": "#FF9999",  # Light red for uppercase markers
-    "<space>": "#CCCCCC",      # Gray for spaces
-    "<newline>": "#CCCCCC",    # Gray for newlines
-    "<tab>": "#CCCCCC",        # Gray for tabs
-    "<unknown>": "#FF0000"     # Red for unknown tokens
-}
+    with gr.Row():
+        process_button = gr.Button("🚀 Tokenize", variant="primary", elem_classes=["custom-button"], size="lg")
+        clear_button = gr.Button("🗑️ Clear", variant="secondary", size="lg")
+
+    gr.Markdown("---")
+    gr.Markdown("### 🔄 Encoded & Decoded Output")
+    with gr.Row():
+        encoded_output = gr.Textbox(label="🔢 Encoded Token IDs", interactive=False, lines=2)
+        decoded_output = gr.Textbox(label="📝 Decoded Text", interactive=False, lines=2)
+
+    gr.Markdown("### 💡 Example Texts")
+    gr.Examples(
+        examples=[
+            ["Merhaba Dünya! Bu bir gelişmiş Türkçe tokenizer testidir."],
+            ["İstanbul'da yaşıyorum ve Türkçe dilini öğreniyorum."],
+            ["KitapOkumak çok güzeldir ve bilgi verir."],
+            ["Türkiye Cumhuriyeti'nin başkenti Ankara'dır."],
+            ["Yapay zeka ve makine öğrenmesi teknolojileri gelişiyor."],
+        ],
+        inputs=input_text,
+        label="Try these examples:"
+    )
 
-# Required files mapping
-REQUIRED_FILES = {
-    'tokenizer.py': 'turkish_tokenizer/turkish_tokenizer.py',
-    'kokler_v08.json': 'turkish_tokenizer/kokler_v08.json',
-    'ekler_v05.json': 'turkish_tokenizer/ekler_v05.json',
-    'bpe_v06.json': 'turkish_tokenizer/bpe_v06.json'
-}
+    gr.Markdown("---")
+    gr.Markdown("### 🎨 Tokenization Output")
+    highlighted_output = gr.HighlightedText(
+        label="Colorized Tokens",
+        color_map=dark_color_map, # This will be updated dynamically if needed
+        show_legend=True
+    )
+
+    gr.Markdown("---")
+    gr.Markdown("### 📊 Statistics")
+    stats_output = gr.HTML(label="")
 
-# Token ID ranges
-TOKEN_RANGES = {
-    'special': (0, 4),          # Special tokens
-    'root_words': (5, 20000),   # Root words
-    'suffixes': (22268, 22767), # Suffixes
-    'bpe': (20000, None)        # BPE tokens (20000+)
-}
 
-def generate_colors(n):
-    """Generate n visually distinct colors."""
-    colors = []
-    for i in range(n):
-        hue = i / n
-        saturation = 0.3 + (i % 3) * 0.1  # Vary saturation between 0.3-0.5
-        value = 0.95 - (i % 2) * 0.1      # Vary value between 0.85-0.95
-        rgb = colorsys.hsv_to_rgb(hue, saturation, value)
-        hex_color = "#{:02x}{:02x}{:02x}".format(
-            int(rgb[0] * 255),
-            int(rgb[1] * 255),
-            int(rgb[2] * 255)
-        )
-        colors.append(hex_color)
-    return colors
-
-def fetch_github_file(path, ref=GITHUB_BRANCH):
-    """Fetch file content from GitHub repository."""
-    url = f"https://api.github.com/repos/{GITHUB_REPO}/contents/{path}?ref={ref}"
-    response = requests.get(url)
-    if response.status_code == 200:
-        content = base64.b64decode(response.json()['content']).decode('utf-8')
-        return content
-    else:
-        st.error(f"Could not fetch {path} from GitHub: {response.status_code}")
-        return None
-
-@st.cache_resource
-def load_tokenizer():
-    """Load and initialize the tokenizer from GitHub."""
-    temp_dir = Path("temp_tokenizer")
-    temp_dir.mkdir(exist_ok=True)
-    
-    # Fetch required files
-    for local_name, github_path in REQUIRED_FILES.items():
-        content = fetch_github_file(github_path)
-        if content is None:
-            return None
-        
-        with open(temp_dir / local_name, 'w', encoding='utf-8') as f:
-            f.write(content)
-    
-    # Modify tokenizer to use correct paths
-    tokenizer_path = temp_dir / "tokenizer.py"
-    with open(tokenizer_path, 'r', encoding='utf-8') as f:
-        tokenizer_code = f.read()
-    
-    modified_code = tokenizer_code.replace(
-        'def load_json(filename):',
-        f'''def load_json(filename):
-    full_path = os.path.join("{temp_dir.absolute()}", filename)
-    with open(full_path, 'r', encoding='utf-8') as file:
-        return json.load(file)'''
+    gr.Markdown("--- \n **Turkish Tokenizer Pro** - Advanced tokenization for Turkish text.")
+
+    # --- Event Handlers ---
+    def process_with_theme(text, theme):
+        return tokenize_and_display(text, theme)
+
+    def clear_all():
+        return "", [], "", "", ""
+
+    # Connect the buttons to the functions
+    process_button.click(
+        fn=process_with_theme,
+        inputs=[input_text, theme_state],
+        outputs=[highlighted_output, encoded_output, decoded_output, stats_output, theme_state]
     )
-    
-    with open(tokenizer_path, 'w', encoding='utf-8') as f:
-        f.write(modified_code)
-    
-    # Load module
-    spec = importlib.util.spec_from_file_location("tokenizer", str(temp_dir / "tokenizer.py"))
-    module = importlib.util.module_from_spec(spec)
-    sys.modules["tokenizer"] = module
-    spec.loader.exec_module(module)
-    
-    return module.tokenize
-
-@st.cache_data(ttl=3600)
-def get_commit_history():
-    """Fetch commit history from GitHub."""
-    url = f"https://api.github.com/repos/{GITHUB_REPO}/commits"
-    try:
-        response = requests.get(url)
-        if response.status_code == 200:
-            commits = response.json()
-            versions = []
-            for commit in commits[:10]:
-                date = datetime.strptime(commit['commit']['author']['date'], '%Y-%m-%dT%H:%M:%SZ').strftime('%Y-%m-%d')
-                sha = commit['sha'][:7]
-                message = commit['commit']['message'].split('\n')[0][:50]
-                versions.append(f"{date} - {sha} - {message}")
-            return versions
-        return ["latest"]
-    except Exception as e:
-        st.warning(f"Could not fetch commit history: {str(e)}")
-        return ["latest"]
-
-def render_tokens(tokens, token_colors):
-    """Render colored token visualization."""
-    html_tokens = []
-    for token in tokens:
-        color = token_colors[token]
-        display_text = SPECIAL_TOKEN_SYMBOLS.get(token, token)  # Use symbol for special tokens
-        html_tokens.append(
-            f'<span style="background-color: {color}; padding: 2px 4px; margin: 2px; border-radius: 3px;" title="{token}">{display_text}</span>'
-        )
-    return " ".join(html_tokens)
-
-# Load tokenizer
-tokenize = load_tokenizer()
-if tokenize is None:
-    st.error("Failed to load tokenizer from GitHub")
-    st.stop()
-
-# Tokenize example text on startup if no results exist
-if st.session_state.token_results is None and st.session_state.text:
-    try:
-        st.session_state.token_results = tokenize(st.session_state.text)
-    except Exception as e:
-        st.error(f"Error tokenizing text: {str(e)}")
-
-# UI Layout
-st.title("🇹🇷 Turkish Tiktokenizer")
-
-# Model selection
-versions = get_commit_history()
-model = st.selectbox("", versions, key="model_selection", label_visibility="collapsed")
-
-# Main layout
-col1, col2 = st.columns([0.4, 0.6])
-
-# Input column
-with col1:
-    text = st.text_area(
-        "Enter Turkish text to tokenize",
-        value=st.session_state.text,
-        height=200,
-        key="text_input",
-        label_visibility="collapsed",
-        placeholder="Enter Turkish text to tokenize"
+
+    clear_button.click(
+        fn=clear_all,
+        outputs=[input_text, highlighted_output, encoded_output, decoded_output, stats_output]
     )
-    
-    if st.button("Tokenize", type="primary"):
-        st.session_state.text = text
-        if text.strip():
-            try:
-                st.session_state.token_results = tokenize(text)
-            except Exception as e:
-                st.session_state.token_results = None
-                st.error(f"Error tokenizing text: {str(e)}")
-        else:
-            st.session_state.token_results = None
-
-# Results column
-with col2:
-    st.markdown("Token count")
-    if st.session_state.token_results is not None:
-        result = st.session_state.token_results
-        token_count = len(result["tokens"])
-        st.markdown(f"### {token_count}")
-        
-        st.markdown("Tokenized text")
-        
-        # Generate token colors
-        regular_tokens = [t for t in result["tokens"] if t not in SPECIAL_COLORS]
-        regular_token_colors = dict(zip(regular_tokens, generate_colors(len(regular_tokens))))
-        token_colors = {**SPECIAL_COLORS, **regular_token_colors}
-        
-        # Render tokens
-        with st.container():
-            st.markdown(render_tokens(result["tokens"], token_colors), unsafe_allow_html=True)
-        
-        st.markdown("Token IDs")
-        st.code(", ".join(map(str, result["ids"])), language=None)
-    else:
-        st.markdown("### 0")
-        st.markdown("Tokenized text")
-        st.markdown("")
-        st.markdown("Token IDs")
-        st.text("")
-
-# Footer
-st.markdown("""
-<div style="position: fixed; bottom: 0; width: 100%; text-align: center; padding: 10px; background-color: white;">
-    <a href="https://github.com/malibayram/tokenizer" target="_blank">View on GitHub</a>
-</div>
-""", unsafe_allow_html=True)
-
-# Cleanup
-def cleanup():
-    if Path("temp_tokenizer").exists():
-        shutil.rmtree("temp_tokenizer")
-
-atexit.register(cleanup) 
+
+    # Auto-process on load with a default example
+    demo.load(
+        fn=lambda theme: tokenize_and_display("Merhaba Dünya!", theme),
+        inputs=[theme_state],
+        outputs=[highlighted_output, encoded_output, decoded_output, stats_output, theme_state]
+    )
+
+if __name__ == "__main__":
+    demo.launch(show_error=True)

ekler.json

@@ -0,0 +1,363 @@
+{
+  "lar": 20000,
+  "ler": 20000,
+  "ya": 20001,
+  "ye": 20001,
+  "ma": 20002,
+  "me": 20002,
+  "malı": 20003,
+  "meli": 20003,
+  "laş": 20004,
+  "leş": 20004,
+  "ça": 20005,
+  "çe": 20005,
+  "şar": 20006,
+  "şer": 20006,
+  "kan": 20007,
+  "ken": 20007,
+  "lak": 20008,
+  "lek": 20008,
+  "layın": 20009,
+  "leyin": 20009,
+  "sak": 20010,
+  "sek": 20010,
+  "arak": 20011,
+  "erek": 20011,
+  "an": 20012,
+  "en": 20012,
+  "ım": 20013,
+  "im": 20013,
+  "um": 20013,
+  "üm": 20013,
+  "ız": 20014,
+  "iz": 20014,
+  "uz": 20014,
+  "üz": 20014,
+  "sı": 20015,
+  "si": 20015,
+  "su": 20015,
+  "sü": 20015,
+  "mış": 20016,
+  "miş": 20016,
+  "muş": 20016,
+  "müş": 20016,
+  "yı": 20017,
+  "yi": 20017,
+  "yu": 20017,
+  "yü": 20017,
+  "lı": 20018,
+  "li": 20018,
+  "lu": 20018,
+  "lü": 20018,
+  "sız": 20019,
+  "siz": 20019,
+  "suz": 20019,
+  "süz": 20019,
+  "ncı": 20020,
+  "nci": 20020,
+  "ncu": 20020,
+  "ncü": 20020,
+  "ın": 20021,
+  "in": 20021,
+  "un": 20021,
+  "ün": 20021,
+  "nın": 20022,
+  "nin": 20022,
+  "nun": 20022,
+  "nün": 20022,
+  "la": 20023,
+  "le": 20023,
+  "yla": 20023,
+  "yle": 20023,
+  "da": 20024,
+  "de": 20024,
+  "ta": 20024,
+  "te": 20024,
+  "dan": 20025,
+  "den": 20025,
+  "tan": 20025,
+  "ten": 20025,
+  "dı": 20026,
+  "di": 20026,
+  "du": 20026,
+  "dü": 20026,
+  "tı": 20026,
+  "ti": 20026,
+  "tu": 20026,
+  "tü": 20026,
+  "cı": 20027,
+  "ci": 20027,
+  "cu": 20027,
+  "cü": 20027,
+  "çı": 20027,
+  "çi": 20027,
+  "çu": 20027,
+  "çü": 20027,
+  "dır": 20028,
+  "dir": 20028,
+  "dur": 20028,
+  "dür": 20028,
+  "tır": 20028,
+  "tir": 20028,
+  "tur": 20028,
+  "tür": 20028,
+  "lık": 20029,
+  "lik": 20029,
+  "luk": 20029,
+  "lük": 20029,
+  "lığ": 20029,
+  "liğ": 20029,
+  "luğ": 20029,
+  "lüğ": 20029,
+  "cık": 20030,
+  "cik": 20030,
+  "cuk": 20030,
+  "cük": 20030,
+  "çık": 20030,
+  "çik": 20030,
+  "çuk": 20030,
+  "çük": 20030,
+  "cığ": 20030,
+  "ciğ": 20030,
+  "cuğ": 20030,
+  "cüğ": 20030,
+  "çığ": 20030,
+  "çiğ": 20030,
+  "çuğ": 20030,
+  "çüğ": 20030,
+  "mak": 20031,
+  "mek": 20031,
+  "may": 20031,
+  "mey": 20031,
+  "acak": 20032,
+  "ecek": 20032,
+  "acağ": 20032,
+  "eceğ": 20032,
+  "yacak": 20032,
+  "yecek": 20032,
+  "yacağ": 20032,
+  "yeceğ": 20032,
+  "i": 20033,
+  "ı": 20034,
+  "u": 20035,
+  "ü": 20036,
+  "a": 20037,
+  "e": 20038,
+  "m": 20039,
+  "n": 20040,
+  "yor": 20041,
+  "ar": 20042,
+  "er": 20043,
+  "sa": 20044,
+  "se": 20045,
+  "r": 20046,
+  "ce": 20047,
+  "daş": 20048,
+  "deş": 20049,
+  "msı": 20050,
+  "msi": 20051,
+  "msu": 20052,
+  "gil": 20053,
+  "ımsa": 20054,
+  "ıcık": 20055,
+  "nç": 20056,
+  "sal": 20057,
+  "sel": 20058,
+  "ki": 20059,
+  "y": 20060,
+  "idi": 20061,
+  "imiş": 20062,
+  "ise": 20063,
+  "s": 20064,
+  "gül": 20065,
+  "kıl": 20066,
+  "kil": 20067,
+  "ka": 20068,
+  "ge": 20069,
+  "z": 20070,
+  "ek_temp_20071": 20071,
+  "ek_temp_20072": 20072,
+  "ek_temp_20073": 20073,
+  "ek_temp_20074": 20074,
+  "ek_temp_20075": 20075,
+  "ek_temp_20076": 20076,
+  "ek_temp_20077": 20077,
+  "ek_temp_20078": 20078,
+  "ek_temp_20079": 20079,
+  "ek_temp_20080": 20080,
+  "ek_temp_20081": 20081,
+  "ek_temp_20082": 20082,
+  "ek_temp_20083": 20083,
+  "ek_temp_20084": 20084,
+  "ek_temp_20085": 20085,
+  "ek_temp_20086": 20086,
+  "ek_temp_20087": 20087,
+  "ek_temp_20088": 20088,
+  "ek_temp_20089": 20089,
+  "ek_temp_20090": 20090,
+  "ek_temp_20091": 20091,
+  "ek_temp_20092": 20092,
+  "ek_temp_20093": 20093,
+  "ek_temp_20094": 20094,
+  "ek_temp_20095": 20095,
+  "ek_temp_20096": 20096,
+  "ek_temp_20097": 20097,
+  "ek_temp_20098": 20098,
+  "ek_temp_20099": 20099,
+  "ek_temp_20100": 20100,
+  "ek_temp_20101": 20101,
+  "ek_temp_20102": 20102,
+  "ek_temp_20103": 20103,
+  "ek_temp_20104": 20104,
+  "ek_temp_20105": 20105,
+  "ek_temp_20106": 20106,
+  "ek_temp_20107": 20107,
+  "ek_temp_20108": 20108,
+  "ek_temp_20109": 20109,
+  "ek_temp_20110": 20110,
+  "ek_temp_20111": 20111,
+  "ek_temp_20112": 20112,
+  "ek_temp_20113": 20113,
+  "ek_temp_20114": 20114,
+  "ek_temp_20115": 20115,
+  "ek_temp_20116": 20116,
+  "ek_temp_20117": 20117,
+  "ek_temp_20118": 20118,
+  "ek_temp_20119": 20119,
+  "ek_temp_20120": 20120,
+  "ek_temp_20121": 20121,
+  "ek_temp_20122": 20122,
+  "ek_temp_20123": 20123,
+  "ek_temp_20124": 20124,
+  "ek_temp_20125": 20125,
+  "ek_temp_20126": 20126,
+  "ek_temp_20127": 20127,
+  "ek_temp_20128": 20128,
+  "ek_temp_20129": 20129,
+  "ek_temp_20130": 20130,
+  "ek_temp_20131": 20131,
+  "ek_temp_20132": 20132,
+  "ek_temp_20133": 20133,
+  "ek_temp_20134": 20134,
+  "ek_temp_20135": 20135,
+  "ek_temp_20136": 20136,
+  "ek_temp_20137": 20137,
+  "ek_temp_20138": 20138,
+  "ek_temp_20139": 20139,
+  "ek_temp_20140": 20140,
+  "ek_temp_20141": 20141,
+  "ek_temp_20142": 20142,
+  "ek_temp_20143": 20143,
+  "ek_temp_20144": 20144,
+  "ek_temp_20145": 20145,
+  "ek_temp_20146": 20146,
+  "ek_temp_20147": 20147,
+  "ek_temp_20148": 20148,
+  "ek_temp_20149": 20149,
+  "ek_temp_20150": 20150,
+  "ek_temp_20151": 20151,
+  "ek_temp_20152": 20152,
+  "ek_temp_20153": 20153,
+  "ek_temp_20154": 20154,
+  "ek_temp_20155": 20155,
+  "ek_temp_20156": 20156,
+  "ek_temp_20157": 20157,
+  "ek_temp_20158": 20158,
+  "ek_temp_20159": 20159,
+  "ek_temp_20160": 20160,
+  "ek_temp_20161": 20161,
+  "ek_temp_20162": 20162,
+  "ek_temp_20163": 20163,
+  "ek_temp_20164": 20164,
+  "ek_temp_20165": 20165,
+  "ek_temp_20166": 20166,
+  "ek_temp_20167": 20167,
+  "ek_temp_20168": 20168,
+  "ek_temp_20169": 20169,
+  "ek_temp_20170": 20170,
+  "ek_temp_20171": 20171,
+  "ek_temp_20172": 20172,
+  "ek_temp_20173": 20173,
+  "ek_temp_20174": 20174,
+  "ek_temp_20175": 20175,
+  "ek_temp_20176": 20176,
+  "ek_temp_20177": 20177,
+  "ek_temp_20178": 20178,
+  "ek_temp_20179": 20179,
+  "ek_temp_20180": 20180,
+  "ek_temp_20181": 20181,
+  "ek_temp_20182": 20182,
+  "ek_temp_20183": 20183,
+  "ek_temp_20184": 20184,
+  "ek_temp_20185": 20185,
+  "ek_temp_20186": 20186,
+  "ek_temp_20187": 20187,
+  "ek_temp_20188": 20188,
+  "ek_temp_20189": 20189,
+  "ek_temp_20190": 20190,
+  "ek_temp_20191": 20191,
+  "ek_temp_20192": 20192,
+  "ek_temp_20193": 20193,
+  "ek_temp_20194": 20194,
+  "ek_temp_20195": 20195,
+  "ek_temp_20196": 20196,
+  "ek_temp_20197": 20197,
+  "ek_temp_20198": 20198,
+  "ek_temp_20199": 20199,
+  "ek_temp_20200": 20200,
+  "ek_temp_20201": 20201,
+  "ek_temp_20202": 20202,
+  "ek_temp_20203": 20203,
+  "ek_temp_20204": 20204,
+  "ek_temp_20205": 20205,
+  "ek_temp_20206": 20206,
+  "ek_temp_20207": 20207,
+  "ek_temp_20208": 20208,
+  "ek_temp_20209": 20209,
+  "ek_temp_20210": 20210,
+  "ek_temp_20211": 20211,
+  "ek_temp_20212": 20212,
+  "ek_temp_20213": 20213,
+  "ek_temp_20214": 20214,
+  "ek_temp_20215": 20215,
+  "ek_temp_20216": 20216,
+  "ek_temp_20217": 20217,
+  "ek_temp_20218": 20218,
+  "ek_temp_20219": 20219,
+  "ek_temp_20220": 20220,
+  "ek_temp_20221": 20221,
+  "ek_temp_20222": 20222,
+  "ek_temp_20223": 20223,
+  "ek_temp_20224": 20224,
+  "ek_temp_20225": 20225,
+  "ek_temp_20226": 20226,
+  "ek_temp_20227": 20227,
+  "ek_temp_20228": 20228,
+  "ek_temp_20229": 20229,
+  "ek_temp_20230": 20230,
+  "ek_temp_20231": 20231,
+  "ek_temp_20232": 20232,
+  "ek_temp_20233": 20233,
+  "ek_temp_20234": 20234,
+  "ek_temp_20235": 20235,
+  "ek_temp_20236": 20236,
+  "ek_temp_20237": 20237,
+  "ek_temp_20238": 20238,
+  "ek_temp_20239": 20239,
+  "ek_temp_20240": 20240,
+  "ek_temp_20241": 20241,
+  "ek_temp_20242": 20242,
+  "ek_temp_20243": 20243,
+  "ek_temp_20244": 20244,
+  "ek_temp_20245": 20245,
+  "ek_temp_20246": 20246,
+  "ek_temp_20247": 20247,
+  "ek_temp_20248": 20248,
+  "ek_temp_20249": 20249,
+  "ek_temp_20250": 20250,
+  "ek_temp_20251": 20251,
+  "ek_temp_20252": 20252,
+  "ek_temp_20253": 20253,
+  "ek_temp_20254": 20254,
+  "ek_temp_20255": 20255
+}

requirements.txt

@@ -1,5 +1 @@
-streamlit>=1.24.0
-numpy>=1.21.0
-json5>=0.9.0
-requests>=2.31.0
-pathlib>=1.0.1 
+gradio

tr_decoder.py

@@ -0,0 +1,232 @@
+from typing import List
+
+
+class TRDecoder:
+    # Define vowel sets as class constants for better performance
+    ALL_VOWELS = "aeıioöuü"
+    INCE_VOWELS = "eiöü"  # Front vowels
+    AI_VOWELS = "aı"      # Back unrounded
+    EI_VOWELS = "ei"      # Front unrounded  
+    OU_VOWELS = "ou"      # Back rounded
+    HARD_CONSONANTS = "fstkçşhp"  # Sert ünsüzler
+    WHITESPACE = " \n\t"
+
+    def __init__(self, reverse_dict):
+        self.reverse_dict = reverse_dict
+
+    def _starts_with_vowel(self, word: str) -> bool:
+        """Check if word starts with a vowel."""
+        return word and word[0] in self.ALL_VOWELS
+
+    def _ends_with_vowel(self, word: str) -> bool:
+        """Check if word ends with a vowel."""
+        return word and word[-1] in self.ALL_VOWELS
+
+    def _ends_with_any(self, word: str, charset: str) -> bool:
+       # recursively check until first vowel starts from the end       
+       i = len(word) - 1
+       while i >= 0:
+           if word[i] in charset:
+               return True
+           if word[i] in self.ALL_VOWELS:
+               return False
+           i -= 1
+       return False
+
+    def _ends_with_ince(self, word: str) -> bool:
+        """Check if word ends with front vowels (ince ünlü)."""
+        if word in ("saat", "kilovatsaat", "ziraat", "itaat"):
+            return True
+        # check until first vowel recursively
+        return self._ends_with_any(word, self.INCE_VOWELS)
+
+    def _ends_with_sert_unsuz(self, word: str) -> bool:
+        """Check if word ends with a hard consonant."""
+        return word and word[-1] in self.HARD_CONSONANTS
+
+    def _get_vowel_suffix_index(self, prev_token: str) -> int:
+        """Get suffix index based on vowel harmony rules."""
+        if self._ends_with_any(prev_token, self.AI_VOWELS):
+            return 0
+        elif self._ends_with_any(prev_token, self.EI_VOWELS):
+            return 1
+        elif self._ends_with_any(prev_token, self.OU_VOWELS):
+            return 2
+        return 3
+
+    def _select_correct_suffix(self, i: int, ids: List[int], prev_token: str) -> str:
+        """Select the correct suffix based on morphological rules."""
+        suffixes = self.reverse_dict[ids[i]]
+        token_id = ids[i]        
+        # Handle different suffix types with cleaner logic
+        if token_id < 20013:
+            # Basic suffix selection based on vowel harmony
+            return suffixes[1] if self._ends_with_ince(prev_token) else suffixes[0]
+            
+        elif token_id < 20023:  # nın, nin, nun, nün
+            return suffixes[self._get_vowel_suffix_index(prev_token)]
+            
+        elif token_id == 20023:  # la, le, yla, yle
+            end_of_word = True
+            if i < len(ids) - 1:
+                next_token = self.reverse_dict[ids[i + 1]][0]
+                if next_token not in self.WHITESPACE:
+                    end_of_word = False
+            return self._handle_la_le_suffix(prev_token, suffixes, end_of_word)
+            
+        elif token_id <= 20025:  # da, de, ta, te, dan, den, tan, ten
+            return self._handle_da_de_suffix(prev_token, suffixes)
+            
+        elif 20025 < token_id < 20029:  # dı, di, du, dü, tı, ti, tu, tü, etc.
+            return self._handle_di_du_suffix(prev_token, suffixes)
+            
+        elif token_id == 20029:  # lık, lik, luk, lük, etc.
+            return self._handle_lik_suffix(i, ids, prev_token, suffixes)
+            
+        elif token_id == 20030:  # cık, cik, cuk, cük, etc.
+            return self._handle_cik_suffix(i, ids, prev_token, suffixes)
+            
+        elif token_id == 20031:  # mak, mek, may, mey
+            return self._handle_mak_suffix(i, ids, prev_token, suffixes)
+            
+        elif token_id == 20032:  # acak, ecek, etc.
+            return self._handle_acak_suffix(i, ids, prev_token, suffixes)
+            
+        return suffixes[0]
+
+    def _handle_la_le_suffix(self, prev_token: str, suffixes: List[str], end_of_word: bool) -> str:
+        """Handle la/le/yla/yle suffix selection."""
+        if self._ends_with_vowel(prev_token) and end_of_word:
+            return suffixes[3] if self._ends_with_ince(prev_token) else suffixes[2]
+        else:
+            return suffixes[1] if self._ends_with_ince(prev_token) else suffixes[0]
+
+    def _handle_da_de_suffix(self, prev_token: str, suffixes: List[str]) -> str:
+        """Handle da/de/ta/te suffix selection."""
+        if self._ends_with_sert_unsuz(prev_token):
+            return suffixes[3] if self._ends_with_ince(prev_token) else suffixes[2]
+        return suffixes[1] if self._ends_with_ince(prev_token) else suffixes[0]
+
+    def _handle_di_du_suffix(self, prev_token: str, suffixes: List[str]) -> str:
+        """Handle dı/di/du/dü suffix selection."""
+        base_index = self._get_vowel_suffix_index(prev_token)
+        return suffixes[base_index + 4] if self._ends_with_sert_unsuz(prev_token) else suffixes[base_index]
+
+    def _handle_lik_suffix(self, i: int, ids: List[int], prev_token: str, suffixes: List[str]) -> str:
+        """Handle lık/lik/luk/lük suffix selection."""
+        if i >= len(ids) - 1:
+            return suffixes[0]
+        
+        next_token = self.reverse_dict[ids[i + 1]][0]
+        base_index = self._get_vowel_suffix_index(prev_token)
+        return suffixes[base_index + 4] if self._starts_with_vowel(next_token) else suffixes[base_index]
+
+    def _handle_cik_suffix(self, i: int, ids: List[int], prev_token: str, suffixes: List[str]) -> str:
+        """Handle cık/cik/cuk/cük suffix selection."""
+        if i >= len(ids) - 1:
+            return suffixes[0]
+        
+        next_token = self.reverse_dict[ids[i + 1]][0]
+        base_index = self._get_vowel_suffix_index(prev_token)
+        
+        if self._starts_with_vowel(next_token):
+            offset = 12 if self._ends_with_sert_unsuz(prev_token) else 8
+        else:
+            offset = 4 if self._ends_with_sert_unsuz(prev_token) else 0
+        
+        return suffixes[base_index + offset]
+
+    def _handle_mak_suffix(self, i: int, ids: List[int], prev_token: str, suffixes: List[str]) -> str:
+        """Handle mak/mek/may/mey suffix selection."""
+        if i >= len(ids) - 1:
+            return suffixes[0]
+        
+        next_token = self.reverse_dict[ids[i + 1]][0]
+        base_index = 1 if self._ends_with_ince(prev_token) else 0
+        return suffixes[base_index + 2] if self._starts_with_vowel(next_token) else suffixes[base_index]
+
+    def _handle_acak_suffix(self, i: int, ids: List[int], prev_token: str, suffixes: List[str]) -> str:
+        """Handle acak/ecek/yacak/yecek suffix selection."""
+        is_vowel_ending = self._ends_with_vowel(prev_token)
+        is_ince = self._ends_with_ince(prev_token)
+
+        is_vowel_starting = False
+        if i < len(ids) - 1:
+          next_token = self.reverse_dict[ids[i + 1]][0]
+          is_vowel_starting = self._starts_with_vowel(next_token)
+        
+        if is_vowel_starting:
+            if is_vowel_ending:
+                return suffixes[7] if is_ince else suffixes[6]
+            else:
+                return suffixes[3] if is_ince else suffixes[2]
+        else:
+            if is_vowel_ending:
+                return suffixes[5] if is_ince else suffixes[4]
+            else:
+                return suffixes[1] if is_ince else suffixes[0]
+
+    def _select_correct_root(self, i: int, ids: List[int]) -> str:
+        """Select the correct root form based on morphological context."""
+        token_id = ids[i]
+        
+        if i >= len(ids) - 2:
+            return self.reverse_dict[token_id][0]
+        
+        next_token = self.reverse_dict[ids[i + 1]][0]
+        
+        if 100 <= token_id < 2080:
+            if self._starts_with_vowel(next_token):
+                return self.reverse_dict[token_id][1]
+            elif token_id <= 110 and ids[i + 1] == 20034:
+                return self.reverse_dict[token_id][2]
+            else:
+                return self.reverse_dict[token_id][0]
+                
+        elif 2080 <= token_id < 2315:
+            if ids[i + 1] == 20021:  # yor
+                return self.reverse_dict[token_id][1]
+            else:
+                return self.reverse_dict[token_id][0]
+        
+        return self.reverse_dict[token_id][0]
+
+    def decode(self, ids: List[int]) -> str:
+        """Decode a list of token IDs to text."""
+        if not ids:
+            return ""
+        
+        text_parts = []
+        i = 0
+        
+        while i < len(ids):
+            token_id = ids[i]
+            # Handle special tokens
+            if token_id == 0 and i < len(ids) - 1:  # uppercase
+                next_token = self.reverse_dict[ids[i + 1]][0]
+                text_parts.append(next_token.capitalize())
+                i += 2
+                continue
+            elif token_id == 1:  # unknown
+                text_parts.append("▁u▁")
+            elif token_id in self.reverse_dict:
+                tokens = self.reverse_dict[token_id]
+                if len(tokens) > 1 and i > 0:
+                    if token_id < 20000:  # root token
+                        text_parts.append(self._select_correct_root(i, ids))
+                    else:  # suffix token
+                        j = -1
+                        prev_token = text_parts[j]
+                        # while prev_token is not a word, get the previous token
+                        while not prev_token.isalpha() and j > -len(text_parts):
+                            prev_token = text_parts[j]
+                            j -= 1
+                        text_parts.append(self._select_correct_suffix(i, ids, prev_token))
+                else:
+                    text_parts.append(tokens[0])
+            else:
+                text_parts.append("▁")
+            
+            i += 1
+        
+        return "".join(text_parts)

tr_tokenizer.py

@@ -0,0 +1,137 @@
+import json
+from enum import Enum
+from typing import Dict, List, Optional, Tuple
+
+from tr_decoder import TRDecoder
+
+
+class TokenType(Enum):
+    ROOT = "ROOT"
+    SUFFIX = "SUFFIX"
+    BPE = "BPE"
+
+class TRTokenizer:
+    def __init__(self):
+        with open("kokler.json", "r") as f:
+            roots = json.load(f)
+        with open("ekler.json", "r") as f:
+            suffixes = json.load(f)
+        with open("bpe_tokenler.json", "r") as f:
+            bpe_tokens = json.load(f)
+        self.reverse_dict = {}
+
+        for key, value in roots.items():
+            if value not in self.reverse_dict:
+                self.reverse_dict[value] = []
+            self.reverse_dict[value].append(key)
+        for key, value in suffixes.items():
+            if value not in self.reverse_dict:
+                self.reverse_dict[value] = []
+            self.reverse_dict[value].append(key)
+        for key, value in bpe_tokens.items():
+            if value not in self.reverse_dict:
+                self.reverse_dict[value] = []
+            self.reverse_dict[value].append(key)
+
+        self.decoder = TRDecoder(self.reverse_dict)
+
+        self.roots = roots
+        self.suffixes = suffixes
+        self.bpe_tokens = bpe_tokens
+        self.max_root_len = max(len(k) for k in roots) if roots else 0
+        self.max_suffix_len = max(len(k) for k in suffixes) if suffixes else 0
+        self.max_bpe_len = max(len(k) for k in bpe_tokens) if bpe_tokens else 0
+        
+        self.uppercase_marker = {"token": "<uppercase>", "id": 0, "type": TokenType.ROOT}
+        self.unknown_marker = {"token": "<unknown>", "id": 1, "type": TokenType.ROOT}
+        self.space_marker = {"token": " ", "id": 2, "type": TokenType.ROOT}
+
+    def _tokenize_word(self, word: str) -> Tuple[List[dict], List[int]]:
+        uppercase_indices = [i for i, c in enumerate(word) if c.isupper()]
+        result = []
+        
+        segments = self._camel_split_with_positions(word)
+        
+        for seg, orig_pos in segments:
+            if orig_pos < len(word) and word[orig_pos].isupper():
+                result.append(self.uppercase_marker)
+            
+            s = seg
+            pos = 0
+            
+            while pos < len(s):
+                substr = s[pos:]
+                
+                rid, rtok = self._longest_prefix_lookup(substr, self.roots, self.max_root_len)
+                if rid is not None:
+                    result.append({"token": rtok, "id": rid, "type": TokenType.ROOT})
+                    pos += len(rtok)
+                    continue
+                
+                sid, stok = self._longest_prefix_lookup(substr, self.suffixes, self.max_suffix_len)
+                if sid is not None:
+                    result.append({"token": stok, "id": sid, "type": TokenType.SUFFIX})
+                    pos += len(stok)
+                    continue
+                
+                bid, btok = self._longest_prefix_lookup(substr, self.bpe_tokens, self.max_bpe_len)
+                if bid is not None:
+                    result.append({"token": btok, "id": bid, "type": TokenType.BPE})
+                    pos += len(btok)
+                    continue
+                
+                result.append(self.unknown_marker)
+                pos += 1
+        
+        return result, uppercase_indices
+
+    def tokenize_text(self, text: str) -> Tuple[List[dict], List[int]]:
+        final_tokens = []
+        uppercase_indices = [i for i, c in enumerate(text) if c.isupper()]
+        
+        parts = text.split(" ")
+        for idx, part in enumerate(parts):
+            if part.strip():
+                tokens, _ = self._tokenize_word(part)
+                final_tokens.extend(tokens)
+            if idx < len(parts) - 1:
+                final_tokens.append(self.space_marker)
+        
+        return final_tokens, uppercase_indices
+    
+    def encode(self, text: str) -> List[int]:
+        tokens, _ = self.tokenize_text(text)
+        return [t["id"] for t in tokens]
+    
+    def tokenize(self, text: str) -> List[str]:
+        tokens, _ = self.tokenize_text(text)
+        return [t["token"] for t in tokens]
+    
+    def _longest_prefix_lookup(self, s: str, table: Dict[str, int], max_len: int = None) -> Tuple[Optional[int], str]:
+        end = min(len(s), max_len) if max_len else len(s)
+        for i in range(end, 0, -1):
+            cand = s[:i]
+            if cand in table:
+                return table[cand], cand
+        return None, ""
+
+    def _camel_split_with_positions(self, word: str) -> List[Tuple[str, int]]:
+        if not word:
+            return []
+        
+        parts = []
+        start = 0
+        
+        for i in range(1, len(word)):
+            if word[i].isupper():
+                if start < i:
+                    parts.append((word[start:i].lower(), start))
+                start = i
+        
+        if start < len(word):
+            parts.append((word[start:].lower(), start))
+        
+        return parts if parts else [(word.lower(), 0)]
+
+    def decode(self, ids: List[int]) -> str:
+        return TRDecoder(self.reverse_dict).decode(ids)