ktext/preprocess.py
def apply_parallel(func: Callable,
                   data: List[Any],
                   cpu_cores: int = None) -> List[Any]:
    """
    Apply function to list of elements.

    Automatically determines the chunk size.
    """
    if not cpu_cores:
        cpu_cores = cpu_count()

    try:
        chunk_size = ceil(len(data) / cpu_cores)
        pool = Pool(cpu_cores)
        transformed_data = pool.map(func, chunked(data, chunk_size), chunksize=1)
    finally:
        pool.close()
        pool.join()
        return transformed_data
def apply_parallel(func: Callable,
                   data: List[Any],
                   cpu_cores: int = None) -> List[Any]:
    if not cpu_cores:
        cpu_cores = cpu_count()

    try:
        chunk_size = ceil(len(data) / cpu_cores)
        pool = Pool(cpu_cores)
        transformed_data = pool.map(func, chunked(data, chunk_size), chunksize=1)
    finally:
        pool.close()
        pool.join()
        return transformed_data
def apply_parallel ( func : Callable , data : List [ Any ] , cpu_cores : int = None ) -> List [ Any ] : if not cpu_cores : cpu_cores = cpu_count ( ) try : chunk_size = ceil ( len ( data ) / cpu_cores ) pool = Pool ( cpu_cores ) transformed_data = pool . map ( func , chunked ( data , chunk_size ) , chunksize = 1 ) finally : pool . close ( ) pool . join ( ) return transformed_data
Apply function to list of elements.

    Automatically determines the chunk size.
Apply function to list of elements .