

@classmethod
def normalize(cls, ch):
    block = unicode_block(ch)
    if (block == UNICODE_BASIC_LATIN):
        if ((ch < 'A') or ('Z' < ch < 'a') or ('z' < ch)):
            ch = ' '
    elif (block == UNICODE_LATIN_1_SUPPLEMENT):
        if (cls.LATIN1_EXCLUDED.find(ch) >= 0):
            ch = ' '
    elif (block == UNICODE_LATIN_EXTENDED_B):
        if (ch == six.u('ș')):
            ch = six.u('ş')
        if (ch == six.u('ț')):
            ch = six.u('ţ')
    elif (block == UNICODE_GENERAL_PUNCTUATION):
        ch = ' '
    elif (block == UNICODE_ARABIC):
        if (ch == six.u('ی')):
            ch = six.u('ي')
    elif (block == UNICODE_LATIN_EXTENDED_ADDITIONAL):
        if (ch >= six.u('Ạ')):
            ch = six.u('ể')
    elif (block == UNICODE_HIRAGANA):
        ch = six.u('あ')
    elif (block == UNICODE_KATAKANA):
        ch = six.u('ア')
    elif (block in (UNICODE_BOPOMOFO, UNICODE_BOPOMOFO_EXTENDED)):
        ch = six.u('ㄅ')
    elif (block == UNICODE_CJK_UNIFIED_IDEOGRAPHS):
        ch = cls.CJK_MAP.get(ch, ch)
    elif (block == UNICODE_HANGUL_SYLLABLES):
        ch = six.u('가')
    return ch

dataset/ETHPy150Open Mimino666/langdetect/langdetect/utils/ngram.py NGram.normalize/original