class HindiTokenizer:
    def __init__(self):
        # Base Varnmala (वर्णमाला)
        self.VYANJAN = set(
            [  # व्यंजन (Consonants)
                "क",
                "ख",
                "ग",
                "घ",
                "ङ",
                "च",
                "छ",
                "ज",
                "झ",
                "ञ",
                "ट",
                "ठ",
                "ड",
                "ढ",
                "ण",
                "त",
                "थ",
                "द",
                "ध",
                "न",
                "प",
                "फ",
                "ब",
                "भ",
                "म",
                "य",
                "र",
                "ल",
                "व",
                "श",
                "ष",
                "स",
                "ह",
            ]
        )

        self.SWAR = set(
            [  # स्वर (Vowels)
                "अ",
                "आ",
                "इ",
                "ई",
                "उ",
                "ऊ",
                "ऋ",
                "ए",
                "ऐ",
                "ओ",
                "औ",
            ]
        )

        self.MATRAS = set(
            [  # मात्राएँ
                "ा",
                "ि",
                "ी",
                "ु",
                "ू",
                "ृ",
                "े",
                "ै",
                "ो",
                "ौ",
            ]
        )

        self.SPECIAL_CHARS = set(
            [
                "्",  # Halant (विराम)
                "ं",  # Anusvara (अनुस्वार)
                "ः",  # Visarga (विसर्ग)
                "ँ",  # Chandrabindu (चन्द्रबिन्दु)
                "़",  # Nukta (नुक्ता)
            ]
        )

        # Combined sets for convenience
        self.CONSONANTS = self.VYANJAN  # For backward compatibility
        self.VOWELS = self.SWAR  # For backward compatibility
        self.BASE_VOCAB = self.VYANJAN | self.SWAR | self.MATRAS | self.SPECIAL_CHARS

        self.base_vocab_stats = {
            "vyanjan": len(self.VYANJAN),
            "swar": len(self.SWAR),
            "matras": len(self.MATRAS),
            "special": len(self.SPECIAL_CHARS),
            "total": len(self.BASE_VOCAB),
        }

    def _get_token_type(self, token: str) -> str:
        """Determine the type of a token"""
        if len(token) == 1:
            if token in self.VYANJAN:
                return "consonant"
            elif token in self.SWAR:
                return "vowel"
            elif token in self.MATRAS:
                return "matra"
            elif token in self.SPECIAL_CHARS:
                return "special"
        return "compound"

    def tokenize(self, text: str) -> list:
        """Basic character-level tokenization"""
        return list(text)

    def is_hindi_char(self, char: str) -> bool:
        """Check if a character is a Hindi character"""
        return char in self.BASE_VOCAB