ktext/preprocess.py
def parallel_process_text(self, data: List[str]) -> List[List[str]]:
        """Apply cleaner -> tokenizer."""
        process_text = process_text_constructor(cleaner=self.cleaner,
                                                tokenizer=self.tokenizer,
                                                append_indicators=self.append_indicators,
                                                start_tok=self.start_tok,
                                                end_tok=self.end_tok)
        n_cores = self.num_cores
        return flattenlist(apply_parallel(process_text, data, n_cores))
def parallel_process_text(self, data: List[str]) -> List[List[str]]:
        process_text = process_text_constructor(cleaner=self.cleaner,
                                                tokenizer=self.tokenizer,
                                                append_indicators=self.append_indicators,
                                                start_tok=self.start_tok,
                                                end_tok=self.end_tok)
        n_cores = self.num_cores
        return flattenlist(apply_parallel(process_text, data, n_cores))
def parallel_process_text ( self , data : List [ str ] ) -> List [ List [ str ] ] : process_text = process_text_constructor ( cleaner = self . cleaner , tokenizer = self . tokenizer , append_indicators = self . append_indicators , start_tok = self . start_tok , end_tok = self . end_tok ) n_cores = self . num_cores return flattenlist ( apply_parallel ( process_text , data , n_cores ) )
Apply cleaner -> tokenizer.
Apply cleaner - > tokenizer .