ktext/preprocess.py
def process_text(self, text: List[str]) -> List[List[str]]:
        """Combine the cleaner and tokenizer."""
        process_text = process_text_constructor(cleaner=self.cleaner,
                                                tokenizer=self.tokenizer,
                                                append_indicators=self.append_indicators,
                                                start_tok=self.start_tok,
                                                end_tok=self.end_tok)
        return process_text(text)
def process_text(self, text: List[str]) -> List[List[str]]:
        process_text = process_text_constructor(cleaner=self.cleaner,
                                                tokenizer=self.tokenizer,
                                                append_indicators=self.append_indicators,
                                                start_tok=self.start_tok,
                                                end_tok=self.end_tok)
        return process_text(text)
def process_text ( self , text : List [ str ] ) -> List [ List [ str ] ] : process_text = process_text_constructor ( cleaner = self . cleaner , tokenizer = self . tokenizer , append_indicators = self . append_indicators , start_tok = self . start_tok , end_tok = self . end_tok ) return process_text ( text )
Combine the cleaner and tokenizer.
Combine the cleaner and tokenizer .