def tokenize(self, string):
        """Tokenize incoming string."""

        if self.language == 'akkadian':
            tokens = tokenize_akkadian_words(string)
        elif self.language == 'arabic':
            tokens = tokenize_arabic_words(string)
        elif self.language == 'french':
            tokens = tokenize_french_words(string)
        elif self.language == 'greek':
            tokens = tokenize_greek_words(string)
        elif self.language == 'latin':
            tokens = tokenize_latin_words(string)
        elif self.language == 'old_norse':
            tokens = tokenize_old_norse_words(string)
        elif self.language == 'middle_english':
            tokens = tokenize_middle_english_words(string)
        elif self.language == 'middle_high_german':
            tokens = tokenize_middle_high_german_words(string)
        else:
            tokens = nltk_tokenize_words(string)

        return tokens
def tokenize ( self , string ) : if self . language == 'akkadian' : tokens = tokenize_akkadian_words ( string ) elif self . language == 'arabic' : tokens = tokenize_arabic_words ( string ) elif self . language == 'french' : tokens = tokenize_french_words ( string ) elif self . language == 'greek' : tokens = tokenize_greek_words ( string ) elif self . language == 'latin' : tokens = tokenize_latin_words ( string ) elif self . language == 'old_norse' : tokens = tokenize_old_norse_words ( string ) elif self . language == 'middle_english' : tokens = tokenize_middle_english_words ( string ) elif self . language == 'middle_high_german' : tokens = tokenize_middle_high_german_words ( string ) else : tokens = nltk_tokenize_words ( string ) return tokens
Tokenize incoming string.