def tokenizer(self):
        """
        Datasets can provide support for segmentation (aka tokenization) in two ways:
        - by providing an orthography profile at etc/orthography.tsv or
        - by overwriting this method to return a custom tokenizer callable.

        :return: A callable to do segmentation.

        The expected signature of the callable is

            def t(item, string, **kw)

        where
        - `item` is a `dict` representing the complete CLDF FormTable row
        - `string` is the string to be segmented
        - `kw` may be used to pass any context info to the tokenizer, when called
          explicitly.
        """
        profile = self.dir / 'etc' / 'orthography.tsv'
        if profile.exists():
            profile = Profile.from_file(str(profile), form='NFC')
            default_spec = list(next(iter(profile.graphemes.values())).keys())
            for grapheme in ['^', '$']:
                if grapheme not in profile.graphemes:
                    profile.graphemes[grapheme] = {k: None for k in default_spec}
            profile.tree = Tree(list(profile.graphemes.keys()))
            tokenizer = Tokenizer(profile=profile, errors_replace=lambda c: '<{0}>'.format(c))

            def _tokenizer(item, string, **kw):
                kw.setdefault("column", "IPA")
                kw.setdefault("separator", " + ")
                return tokenizer(unicodedata.normalize('NFC', '^' + string + '$'), **kw).split()
            return _tokenizer
def tokenizer ( self ) : profile = self . dir / 'etc' / 'orthography.tsv' if profile . exists ( ) : profile = Profile . from_file ( str ( profile ) , form = 'NFC' ) default_spec = list ( next ( iter ( profile . graphemes . values ( ) ) ) . keys ( ) ) for grapheme in [ '^' , '$' ] : if grapheme not in profile . graphemes : profile . graphemes [ grapheme ] = { k : None for k in default_spec } profile . tree = Tree ( list ( profile . graphemes . keys ( ) ) ) tokenizer = Tokenizer ( profile = profile , errors_replace = lambda c : '<{0}>' . format ( c ) ) def _tokenizer ( item , string , ** kw ) : kw . setdefault ( "column" , "IPA" ) kw . setdefault ( "separator" , " + " ) return tokenizer ( unicodedata . normalize ( 'NFC' , '^' + string + '$' ) , ** kw ) . split ( ) return _tokenizer
Datasets can provide support for segmentation (aka tokenization) in two ways:
        - by providing an orthography profile at etc/orthography.tsv or
        - by overwriting this method to return a custom tokenizer callable.

        :return: A callable to do segmentation.

        The expected signature of the callable is

            def t(item, string, **kw)

        where
        - `item` is a `dict` representing the complete CLDF FormTable row
        - `string` is the string to be segmented
        - `kw` may be used to pass any context info to the tokenizer, when called
          explicitly.