def mecab_tokenize(text, lang):
    """
    Use the mecab-python3 package to tokenize the given text. The `lang`
    must be 'ja' for Japanese or 'ko' for Korean.

    The simplest output from mecab-python3 is the single-string form, which
    contains the same table that the command-line version of MeCab would output.
    We find the tokens in the first column of this table.
    """
    if lang not in MECAB_DICTIONARY_NAMES:
        raise ValueError("Can't run MeCab on language %r" % lang)
    if lang not in MECAB_ANALYZERS:
        MECAB_ANALYZERS[lang] = make_mecab_analyzer(MECAB_DICTIONARY_NAMES[lang])

    analyzer = MECAB_ANALYZERS[lang]
    text = unicodedata.normalize('NFKC', text.strip())
    analyzed = analyzer.parse(text)
    if not analyzed:
        return []
    return [line.split('\t')[0]
            for line in analyzed.split('\n')
            if line != '' and line != 'EOS']
def mecab_tokenize ( text , lang ) : if lang not in MECAB_DICTIONARY_NAMES : raise ValueError ( "Can't run MeCab on language %r" % lang ) if lang not in MECAB_ANALYZERS : MECAB_ANALYZERS [ lang ] = make_mecab_analyzer ( MECAB_DICTIONARY_NAMES [ lang ] ) analyzer = MECAB_ANALYZERS [ lang ] text = unicodedata . normalize ( 'NFKC' , text . strip ( ) ) analyzed = analyzer . parse ( text ) if not analyzed : return [ ] return [ line . split ( '\t' ) [ 0 ] for line in analyzed . split ( '\n' ) if line != '' and line != 'EOS' ]
Use the mecab-python3 package to tokenize the given text. The `lang`
    must be 'ja' for Japanese or 'ko' for Korean.

    The simplest output from mecab-python3 is the single-string form, which
    contains the same table that the command-line version of MeCab would output.
    We find the tokens in the first column of this table.