Initial commit

.gitattributes

@@ -23,13 +23,11 @@
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text 

README.md

@@ -1,6 +1,38 @@
----
-license: mit
-sdk: gradio
-colorFrom: indigo
-colorTo: gray
----
+# Awadhi BPE Tokenizer
+
+This space provides a Byte Pair Encoding (BPE) implementation for Awadhi text compression. It features:
+
+- Custom BPE implementation for Awadhi text
+- Vocabulary size < 5000 tokens
+- Compression ratio > 3.2
+- Interactive web interface
+
+## Usage
+
+1. Enter Awadhi text in the input box
+2. Click "Tokenize"
+3. View tokenization results and statistics
+
+## Implementation Details
+
+- Uses character-level tokenization as base
+- Implements BPE merging strategy
+- Handles UTF-8 encoded Awadhi text
+- Provides compression statistics
+
+## Model Details
+
+- Base tokenization: Character-level
+- Maximum vocabulary size: 4500 tokens
+- Training corpus: Sunderkand in Awadhi
+- Compression target: > 3.2x
+
+## Technical Requirements
+
+- Python 3.10+
+- PyTorch
+- Gradio 3.50.2+
+
+## License
+
+This project is licensed under the MIT License. 

bpe_Awadhi.py

@@ -0,0 +1,106 @@
+import torch
+from collections import defaultdict, Counter
+import re
+from typing import Dict, List, Tuple, Set
+import json
+
+class AwadhiBPE:
+    def __init__(self, vocab_size: int = 5000):
+        self.vocab_size = vocab_size
+        self.merges: Dict[Tuple[str, str], str] = {}
+        self.vocab: Set[str] = set()
+        
+    def get_stats(self, vocab: Dict[str, int]) -> Dict[Tuple[str, str], int]:
+        pairs = defaultdict(int)
+        for word, freq in vocab.items():
+            symbols = word.split()
+            for i in range(len(symbols)-1):
+                pairs[symbols[i], symbols[i+1]] += freq
+        return pairs
+    
+    def merge_vocab(self, pair: Tuple[str, str], v_in: Dict[str, int]) -> Dict[str, int]:
+        v_out = {}
+        bigram = ' '.join(pair)
+        replacement = ''.join(pair)
+        for word in v_in:
+            w_out = word.replace(bigram, replacement)
+            v_out[w_out] = v_in[word]
+        return v_out
+    
+    def fit(self, text: str) -> None:
+        # Initial character-level tokenization
+        words = text.split()
+        word_freqs = Counter(words)
+        
+        # Initialize vocabulary with characters
+        vocab = {}
+        for word, freq in word_freqs.items():
+            chars = ' '.join(list(word))
+            vocab[chars] = freq
+            self.vocab.update(set(word))
+
+        num_merges = min(self.vocab_size - len(self.vocab), len(vocab))
+        
+        for i in range(num_merges):
+            pairs = self.get_stats(vocab)
+            if not pairs:
+                break
+                
+            best = max(pairs, key=pairs.get)
+            vocab = self.merge_vocab(best, vocab)
+            self.merges[best] = ''.join(best)
+            self.vocab.add(self.merges[best])
+            
+    def tokenize(self, text: str) -> List[str]:
+        words = text.split()
+        tokens = []
+        
+        for word in words:
+            chars = ' '.join(list(word))
+            for pair, merge in self.merges.items():
+                chars = chars.replace(' '.join(pair), merge)
+            tokens.extend(chars.split())
+            
+        return tokens
+
+    def save(self, path: str) -> None:
+        with open(path, 'w', encoding='utf-8') as f:
+            json.dump({
+                'merges': {' '.join(k): v for k, v in self.merges.items()},
+                'vocab': list(self.vocab)
+            }, f, ensure_ascii=False)
+            
+    def load(self, path: str) -> None:
+        with open(path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+            self.merges = {tuple(k.split()): v for k, v in data['merges'].items()}
+            self.vocab = set(data['vocab'])
+
+# Training and evaluation code
+def main():
+    # Read the text file
+    with open('sunderkand_awdhi.txt', 'r', encoding='utf-8') as f:
+        text = f.read()
+    
+    # Create and train BPE
+    bpe = AwadhiBPE(vocab_size=4500)  # Using slightly less than 5000 to be safe
+    bpe.fit(text)
+    
+    # Save the trained model
+    bpe.save('Awadhi_bpe.json')
+    
+    # Tokenize the text
+    tokens = bpe.tokenize(text)
+    
+    # Calculate compression ratio
+    original_size = len(text.encode('utf-8'))
+    tokenized_size = len(tokens) * 2  # Assuming average 2 bytes per token
+    compression_ratio = original_size / tokenized_size
+    
+    print(f"Original size (bytes): {original_size}")
+    print(f"Tokenized size (bytes): {tokenized_size}")
+    print(f"Compression ratio: {compression_ratio:.2f}")
+    print(f"Vocabulary size: {len(bpe.vocab)}")
+
+if __name__ == "__main__":
+    main() 

config.yaml

@@ -0,0 +1,18 @@
+title: Awadhi BPE Tokenizer
+emoji: 🇮🇳
+colorFrom: blue
+colorTo: red
+sdk: gradio
+sdk_version: 3.50.2
+app_file: app.py
+pinned: false
+license: mit
+python_version: "3.10"
+app_port: 7860
+tags:
+    - awadhi
+    - tokenizer
+    - bpe
+    - text-compression
+datasets:
+    - sunderkand_awdhi 

requirements.txt

@@ -0,0 +1,2 @@
+gradio
+torch