"""These are utilities designed for carefully handling communication between
processes while multithreading.

The code for ``pool_imap_unordered`` is copied nearly wholesale from GrantJ's 
`Stack Overflow answer here
<https://stackoverflow.com/questions/5318936/python-multiprocessing-pool-lazy-iteration?noredirect=1&lq=1>`_.
It allows for a lazy imap over an iterable and the return of very large objects
"""

from multiprocessing import Process, Queue, cpu_count

try:
    from Queue import Full as QueueFull
    from Queue import Empty as QueueEmpty
except ImportError:  # python3
    from queue import Full as QueueFull
    from queue import Empty as QueueEmpty

__all__ = ["pool_imap_unordered"]


def worker(func, recvq, sendq):
    for args in iter(recvq.get, None):
        # The args are training_data, scoring_data, var_idx
        # Thus, we want to return the var_idx and then 
        # send those args to the abstract runner. 
        result = (args[-1], func(*args))
        
        sendq.put(result)


def pool_imap_unordered(func, iterable, procs=cpu_count()):
    """Lazily imaps in an unordered manner over an iterable in parallel as a
    generator

    :Author: <NAME> <https://stackoverflow.com/users/232571/grantj>

    :param func: function to perform on each iterable
    :param iterable: iterable which has items to map over
    :param procs: number of workers in the pool. Defaults to the cpu count
    :yields: the results of the mapping
    """
    # Create queues for sending/receiving items from iterable.

    sendq = Queue(procs)
    recvq = Queue()

    # Start worker processes.

    for rpt in range(procs):
        Process(target=worker, args=(func, sendq, recvq)).start()

    # Iterate iterable and communicate with worker processes.

    send_len = 0
    recv_len = 0
    itr = iter(iterable)

    try:
        value = next(itr)
        while True:
            try:
                sendq.put(value, True, 0.1)
                send_len += 1
                value = next(itr)
            except QueueFull:
                while True:
                    try:
                        result = recvq.get(False)
                        recv_len += 1
                        yield result
                    except QueueEmpty:
                        break
    except StopIteration:
        pass

    # Collect all remaining results.

    while recv_len < send_len:
        result = recvq.get()
        recv_len += 1
        yield result

    # Terminate worker processes.

    for rpt in range(procs):
        sendq.put(None)

"""These are utilities designed for carefully handling communication between
processes while multithreading.

The code for ``pool_imap_unordered`` is copied nearly wholesale from GrantJ's 
`Stack Overflow answer here
<https://stackoverflow.com/questions/5318936/python-multiprocessing-pool-lazy-iteration?noredirect=1&lq=1>`_.
It allows for a lazy imap over an iterable and the return of very large objects
"""

from multiprocessing import Process, Queue, cpu_count

try:
    from Queue import Full as QueueFull
    from Queue import Empty as QueueEmpty
except ImportError:  # python3
    from queue import Full as QueueFull
    from queue import Empty as QueueEmpty

__all__ = ["pool_imap_unordered"]


def worker(func, recvq, sendq):
    for args in iter(recvq.get, None):
        # The args are training_data, scoring_data, var_idx
        # Thus, we want to return the var_idx and then 
        # send those args to the abstract runner. 
        result = (args[-1], func(*args))
        
        sendq.put(result)


def pool_imap_unordered(func, iterable, procs=cpu_count()):
    """Lazily imaps in an unordered manner over an iterable in parallel as a
    generator

    :Author: <NAME> <https://stackoverflow.com/users/232571/grantj>

    :param func: function to perform on each iterable
    :param iterable: iterable which has items to map over
    :param procs: number of workers in the pool. Defaults to the cpu count
    :yields: the results of the mapping
    """
    # Create queues for sending/receiving items from iterable.

    sendq = Queue(procs)
    recvq = Queue()

    # Start worker processes.

    for rpt in range(procs):
        Process(target=worker, args=(func, sendq, recvq)).start()

    # Iterate iterable and communicate with worker processes.

    send_len = 0
    recv_len = 0
    itr = iter(iterable)

    try:
        value = next(itr)
        while True:
            try:
                sendq.put(value, True, 0.1)
                send_len += 1
                value = next(itr)
            except QueueFull:
                while True:
                    try:
                        result = recvq.get(False)
                        recv_len += 1
                        yield result
                    except QueueEmpty:
                        break
    except StopIteration:
        pass

    # Collect all remaining results.

    while recv_len < send_len:
        result = recvq.get()
        recv_len += 1
        yield result

    # Terminate worker processes.

    for rpt in range(procs):
        sendq.put(None)

These are utilities designed for carefully handling communication between processes while multithreading.  The code for ``pool_imap_unordered`` is copied nearly wholesale from GrantJ's  `Stack Overflow answer here <https://stackoverflow.com/questions/5318936/python-multiprocessing-pool-lazy-iteration?noredirect=1&lq=1>`_. It allows for a lazy imap over an iterable and the return of very large objects # python3 # The args are training_data, scoring_data, var_idx # Thus, we want to return the var_idx and then # send those args to the abstract runner. Lazily imaps in an unordered manner over an iterable in parallel as a     generator      :Author: <NAME> <https://stackoverflow.com/users/232571/grantj>      :param func: function to perform on each iterable     :param iterable: iterable which has items to map over     :param procs: number of workers in the pool. Defaults to the cpu count     :yields: the results of the mapping # Create queues for sending/receiving items from iterable. # Start worker processes. # Iterate iterable and communicate with worker processes. # Collect all remaining results. # Terminate worker processes.