ktext/preprocess.py
def process_text_constructor(cleaner: Callable,
                             tokenizer: Callable,
                             append_indicators: bool,
                             start_tok: str,
                             end_tok: str):
    """Generate a function that will clean and tokenize text."""
    def process_text(text):
        if append_indicators:
            return [[start_tok] + tokenizer(cleaner(doc)) + [end_tok] for doc in text]
        return [tokenizer(cleaner(doc)) for doc in text]

    return process_text
def process_text_constructor(cleaner: Callable,
                             tokenizer: Callable,
                             append_indicators: bool,
                             start_tok: str,
                             end_tok: str):
    def process_text(text):
        if append_indicators:
            return [[start_tok] + tokenizer(cleaner(doc)) + [end_tok] for doc in text]
        return [tokenizer(cleaner(doc)) for doc in text]

    return process_text
def process_text_constructor ( cleaner : Callable , tokenizer : Callable , append_indicators : bool , start_tok : str , end_tok : str ) : def process_text ( text ) : if append_indicators : return [ [ start_tok ] + tokenizer ( cleaner ( doc ) ) + [ end_tok ] for doc in text ] return [ tokenizer ( cleaner ( doc ) ) for doc in text ] return process_text
Generate a function that will clean and tokenize text.
Generate a function that will clean and tokenize text .