addok/helpers/__init__.py
def imap_unordered(self, func, iterable, chunksize):
        """Customized version of imap_unordered.

        Directly send chunks to func, instead of iterating in each process and
        sending one by one.

        Original:
        https://hg.python.org/cpython/file/tip/Lib/multiprocessing/pool.py#l271

        Other tried options:
        - map_async: makes a list(iterable), so it loads all the data for each
          process into RAM
        - apply_async: needs manual chunking
        """
        assert self._state == RUN
        task_batches = Pool._get_tasks(func, iterable, chunksize)
        result = IMapUnorderedIterator(self._cache)
        tasks = ((result._job, i, func, chunk, {})
                 for i, (_, chunk) in enumerate(task_batches))
        self._taskqueue.put((tasks, result._set_length))
        return result
def imap_unordered(self, func, iterable, chunksize):
        assert self._state == RUN
        task_batches = Pool._get_tasks(func, iterable, chunksize)
        result = IMapUnorderedIterator(self._cache)
        tasks = ((result._job, i, func, chunk, {})
                 for i, (_, chunk) in enumerate(task_batches))
        self._taskqueue.put((tasks, result._set_length))
        return result
def imap_unordered ( self , func , iterable , chunksize ) : assert self . _state == RUN task_batches = Pool . _get_tasks ( func , iterable , chunksize ) result = IMapUnorderedIterator ( self . _cache ) tasks = ( ( result . _job , i , func , chunk , { } ) for i , ( _ , chunk ) in enumerate ( task_batches ) ) self . _taskqueue . put ( ( tasks , result . _set_length ) ) return result
Customized version of imap_unordered.

        Directly send chunks to func, instead of iterating in each process and
        sending one by one.

        Original:
        https://hg.python.org/cpython/file/tip/Lib/multiprocessing/pool.py#l271

        Other tried options:
        - map_async: makes a list(iterable), so it loads all the data for each
          process into RAM
        - apply_async: needs manual chunking
Customized version of imap_unordered .