Process items in a list in parallel (optionally, one smaller batch at a time).

    Args:
        input_list: An input object that has a list-like interface (indexing and slicing).
        mapper: A function to apply to each item of the input list.
        project: An instance of pygoose project.
        n_jobs: The number of parallel processing jobs. -1 will use the number of CPUs on the system.
        batch_size: The maximum number of input items in each batch. -1 will store all data as a single batch.
        checkpoint: Whether to save each batch and its corresponding output to disk.
        cleanup: Whether to remove the batch checkpoints from the disk after all batches are processed.
        **kwargs: Additional keyword arguments to joblib.Parallel.

    Returns:
        A list representing the combined output from the mapper function called on all input items.

def map_embarrassingly_parallel(input_list, mapper, project, n_jobs=-1, batch_size=-1,
                                checkpoint=False, cleanup=True, **kwargs):
    """
    Process items in a list in parallel (optionally, one smaller batch at a time).

    Args:
        input_list: An input object that has a list-like interface (indexing and slicing).
        mapper: A function to apply to each item of the input list.
        project: An instance of pygoose project.
        n_jobs: The number of parallel processing jobs. -1 will use the number of CPUs on the system.
        batch_size: The maximum number of input items in each batch. -1 will store all data as a single batch.
        checkpoint: Whether to save each batch and its corresponding output to disk.
        cleanup: Whether to remove the batch checkpoints from the disk after all batches are processed.
        **kwargs: Additional keyword arguments to joblib.Parallel.

    Returns:
        A list representing the combined output from the mapper function called on all input items.
    """

    if batch_size < 0:
        batch_size = len(input_list)

    # Partition the data.
    job_id = _create_job_id()
    print('Creating job ID:', job_id)

    batch_storage_dir = os.path.join(project.temp_dir, job_id)
    batches = split_into_batches(input_list, batch_size, batch_storage_dir, checkpoint)

    # The results will be collected here.
    # TODO: collecting lists like this may be memory inefficient. Perhaps we could use another callback function.
    combined_results = []

    # Process data one batch at a time.
    for batch in batches:
        description = 'Batch {}/{}'.format(batch['index'] + 1, len(batches))

        # Process each item in the batch in parallel.
        batch_result = Parallel(n_jobs=n_jobs, **kwargs)(
            delayed(mapper)(input_item)
            for input_item in progressbar(
                batch['data'],
                desc=description,
                total=len(batch['data']),
                file=sys.stdout,
            )
        )
        if checkpoint:
            save(batch_result, batch['result_filename'])

        combined_results.extend(batch_result)

    # Remove the temporary files.
    if checkpoint and cleanup:
        shutil.rmtree(batch_storage_dir)

    return combined_results