loky/reusable_executor.py
def get_reusable_executor(max_workers=None, context=None, timeout=10,
                          kill_workers=False, reuse="auto",
                          job_reducers=None, result_reducers=None,
                          initializer=None, initargs=()):
    """Return the current ReusableExectutor instance.

    Start a new instance if it has not been started already or if the previous
    instance was left in a broken state.

    If the previous instance does not have the requested number of workers, the
    executor is dynamically resized to adjust the number of workers prior to
    returning.

    Reusing a singleton instance spares the overhead of starting new worker
    processes and importing common python packages each time.

    ``max_workers`` controls the maximum number of tasks that can be running in
    parallel in worker processes. By default this is set to the number of
    CPUs on the host.

    Setting ``timeout`` (in seconds) makes idle workers automatically shutdown
    so as to release system resources. New workers are respawn upon submission
    of new tasks so that ``max_workers`` are available to accept the newly
    submitted tasks. Setting ``timeout`` to around 100 times the time required
    to spawn new processes and import packages in them (on the order of 100ms)
    ensures that the overhead of spawning workers is negligible.

    Setting ``kill_workers=True`` makes it possible to forcibly interrupt
    previously spawned jobs to get a new instance of the reusable executor
    with new constructor argument values.

    The ``job_reducers`` and ``result_reducers`` are used to customize the
    pickling of tasks and results send to the executor.

    When provided, the ``initializer`` is run first in newly spawned
    processes with argument ``initargs``.
    """
    with _executor_lock:
        global _executor, _executor_kwargs
        executor = _executor

        if max_workers is None:
            if reuse is True and executor is not None:
                max_workers = executor._max_workers
            else:
                max_workers = cpu_count()
        elif max_workers <= 0:
            raise ValueError(
                "max_workers must be greater than 0, got {}."
                .format(max_workers))

        if isinstance(context, STRING_TYPE):
            context = get_context(context)
        if context is not None and context.get_start_method() == "fork":
            raise ValueError("Cannot use reusable executor with the 'fork' "
                             "context")

        kwargs = dict(context=context, timeout=timeout,
                      job_reducers=job_reducers,
                      result_reducers=result_reducers,
                      initializer=initializer, initargs=initargs)
        if executor is None:
            mp.util.debug("Create a executor with max_workers={}."
                          .format(max_workers))
            executor_id = _get_next_executor_id()
            _executor_kwargs = kwargs
            _executor = executor = _ReusablePoolExecutor(
                _executor_lock, max_workers=max_workers,
                executor_id=executor_id, **kwargs)
        else:
            if reuse == 'auto':
                reuse = kwargs == _executor_kwargs
            if (executor._flags.broken or executor._flags.shutdown
                    or not reuse):
                if executor._flags.broken:
                    reason = "broken"
                elif executor._flags.shutdown:
                    reason = "shutdown"
                else:
                    reason = "arguments have changed"
                mp.util.debug(
                    "Creating a new executor with max_workers={} as the "
                    "previous instance cannot be reused ({})."
                    .format(max_workers, reason))
                executor.shutdown(wait=True, kill_workers=kill_workers)
                _executor = executor = _executor_kwargs = None
                # Recursive call to build a new instance
                return get_reusable_executor(max_workers=max_workers,
                                             **kwargs)
            else:
                mp.util.debug("Reusing existing executor with max_workers={}."
                              .format(executor._max_workers))
                executor._resize(max_workers)

    return executor
def get_reusable_executor(max_workers=None, context=None, timeout=10,
                          kill_workers=False, reuse="auto",
                          job_reducers=None, result_reducers=None,
                          initializer=None, initargs=()):
    with _executor_lock:
        global _executor, _executor_kwargs
        executor = _executor

        if max_workers is None:
            if reuse is True and executor is not None:
                max_workers = executor._max_workers
            else:
                max_workers = cpu_count()
        elif max_workers <= 0:
            raise ValueError(
                "max_workers must be greater than 0, got {}."
                .format(max_workers))

        if isinstance(context, STRING_TYPE):
            context = get_context(context)
        if context is not None and context.get_start_method() == "fork":
            raise ValueError("Cannot use reusable executor with the 'fork' "
                             "context")

        kwargs = dict(context=context, timeout=timeout,
                      job_reducers=job_reducers,
                      result_reducers=result_reducers,
                      initializer=initializer, initargs=initargs)
        if executor is None:
            mp.util.debug("Create a executor with max_workers={}."
                          .format(max_workers))
            executor_id = _get_next_executor_id()
            _executor_kwargs = kwargs
            _executor = executor = _ReusablePoolExecutor(
                _executor_lock, max_workers=max_workers,
                executor_id=executor_id, **kwargs)
        else:
            if reuse == 'auto':
                reuse = kwargs == _executor_kwargs
            if (executor._flags.broken or executor._flags.shutdown
                    or not reuse):
                if executor._flags.broken:
                    reason = "broken"
                elif executor._flags.shutdown:
                    reason = "shutdown"
                else:
                    reason = "arguments have changed"
                mp.util.debug(
                    "Creating a new executor with max_workers={} as the "
                    "previous instance cannot be reused ({})."
                    .format(max_workers, reason))
                executor.shutdown(wait=True, kill_workers=kill_workers)
                _executor = executor = _executor_kwargs = None
                # Recursive call to build a new instance
                return get_reusable_executor(max_workers=max_workers,
                                             **kwargs)
            else:
                mp.util.debug("Reusing existing executor with max_workers={}."
                              .format(executor._max_workers))
                executor._resize(max_workers)

    return executor
def get_reusable_executor ( max_workers = None , context = None , timeout = 10 , kill_workers = False , reuse = "auto" , job_reducers = None , result_reducers = None , initializer = None , initargs = ( ) ) : with _executor_lock : global _executor , _executor_kwargs executor = _executor if max_workers is None : if reuse is True and executor is not None : max_workers = executor . _max_workers else : max_workers = cpu_count ( ) elif max_workers <= 0 : raise ValueError ( "max_workers must be greater than 0, got {}." . format ( max_workers ) ) if isinstance ( context , STRING_TYPE ) : context = get_context ( context ) if context is not None and context . get_start_method ( ) == "fork" : raise ValueError ( "Cannot use reusable executor with the 'fork' " "context" ) kwargs = dict ( context = context , timeout = timeout , job_reducers = job_reducers , result_reducers = result_reducers , initializer = initializer , initargs = initargs ) if executor is None : mp . util . debug ( "Create a executor with max_workers={}." . format ( max_workers ) ) executor_id = _get_next_executor_id ( ) _executor_kwargs = kwargs _executor = executor = _ReusablePoolExecutor ( _executor_lock , max_workers = max_workers , executor_id = executor_id , * * kwargs ) else : if reuse == 'auto' : reuse = kwargs == _executor_kwargs if ( executor . _flags . broken or executor . _flags . shutdown or not reuse ) : if executor . _flags . broken : reason = "broken" elif executor . _flags . shutdown : reason = "shutdown" else : reason = "arguments have changed" mp . util . debug ( "Creating a new executor with max_workers={} as the " "previous instance cannot be reused ({})." . format ( max_workers , reason ) ) executor . shutdown ( wait = True , kill_workers = kill_workers ) _executor = executor = _executor_kwargs = None # Recursive call to build a new instance return get_reusable_executor ( max_workers = max_workers , * * kwargs ) else : mp . util . debug ( "Reusing existing executor with max_workers={}." . format ( executor . _max_workers ) ) executor . _resize ( max_workers ) return executor
Return the current ReusableExectutor instance.

    Start a new instance if it has not been started already or if the previous
    instance was left in a broken state.

    If the previous instance does not have the requested number of workers, the
    executor is dynamically resized to adjust the number of workers prior to
    returning.

    Reusing a singleton instance spares the overhead of starting new worker
    processes and importing common python packages each time.

    ``max_workers`` controls the maximum number of tasks that can be running in
    parallel in worker processes. By default this is set to the number of
    CPUs on the host.

    Setting ``timeout`` (in seconds) makes idle workers automatically shutdown
    so as to release system resources. New workers are respawn upon submission
    of new tasks so that ``max_workers`` are available to accept the newly
    submitted tasks. Setting ``timeout`` to around 100 times the time required
    to spawn new processes and import packages in them (on the order of 100ms)
    ensures that the overhead of spawning workers is negligible.

    Setting ``kill_workers=True`` makes it possible to forcibly interrupt
    previously spawned jobs to get a new instance of the reusable executor
    with new constructor argument values.

    The ``job_reducers`` and ``result_reducers`` are used to customize the
    pickling of tasks and results send to the executor.

    When provided, the ``initializer`` is run first in newly spawned
    processes with argument ``initargs``.
Return the current ReusableExectutor instance .