r"""
    Interfaces to either multiprocessing or futures.
    Esentially maps ``args_gen`` onto ``func`` using pool.imap.
    However, args_gen must be a tuple of args that will be unpacked and send to
    the function. Thus, the function can take multiple args. Also specifing
    keyword args is supported.

    Useful for embarrassingly parallel loops. Currently does not work with
    opencv3

    CommandLine:
        python -m utool.util_parallel generate2

    Args:
        func (function):  live python function
        args_gen (?):
        kw_gen (None): (default = None)
        ntasks (None): (default = None)
        ordered (bool): (default = True)
        force_serial (bool): (default = False)
        verbose (bool):  verbosity flag(default = None)

    CommandLine:
        python -m utool.util_parallel generate2

    Example:
        >>> # DISABLE_DOCTEST
        >>> from utool.util_parallel import *  # NOQA
        >>> from utool.util_parallel import _kw_wrap_worker  # NOQA
        >>> import utool as ut
        >>> args_gen = list(zip(range(10000)))
        >>> kw_gen = [{}] * len(args_gen)
        >>> func = ut.is_prime
        >>> _ = list(generate2(func, args_gen))
        >>> _ = list(generate2(func, args_gen, ordered=False))
        >>> _ = list(generate2(func, args_gen, force_serial=True))
        >>> _ = list(generate2(func, args_gen, use_pool=True))
        >>> _ = list(generate2(func, args_gen, ordered=False, verbose=False))

    Example0:
        >>> # ENABLE_DOCTEST
        >>> import utool as ut
        >>> #num = 8700  # parallel is slower for smaller numbers
        >>> num = 500  # parallel has an initial (~.1 second startup overhead)
        >>> print('TESTING SERIAL')
        >>> func = ut.is_prime
        >>> args_list = list(range(0, num))
        >>> flag_generator0 = ut.generate2(ut.is_prime, zip(range(0, num)), force_serial=True)
        >>> flag_list0 = list(flag_generator0)
        >>> print('TESTING PARALLEL')
        >>> flag_generator1 = ut.generate2(ut.is_prime, zip(range(0, num)))
        >>> flag_list1 = list(flag_generator1)
        >>> print('ASSERTING')
        >>> assert len(flag_list1) == num
        >>> assert flag_list0 == flag_list1

    Example1:
        >>> # ENABLE_DOCTEST
        >>> # Trying to recreate the freeze seen in IBEIS
        >>> import utool as ut
        >>> print('TESTING SERIAL')
        >>> flag_generator0 = ut.generate2(ut.is_prime, zip(range(0, 1)))
        >>> flag_list0 = list(flag_generator0)
        >>> flag_generator1 = ut.generate2(ut.fibonacci_recursive, zip(range(0, 1)))
        >>> flag_list1 = list(flag_generator1)
        >>> print('TESTING PARALLEL')
        >>> flag_generator2 = ut.generate2(ut.is_prime, zip(range(0, 12)))
        >>> flag_list2 = list(flag_generator2)
        >>> flag_generator3 = ut.generate2(ut.fibonacci_recursive, zip(range(0, 12)))
        >>> flag_list3 = list(flag_generator3)
        >>> print('flag_list0 = %r' % (flag_list0,))
        >>> print('flag_list1 = %r' % (flag_list1,))
        >>> print('flag_list2 = %r' % (flag_list1,))
        >>> print('flag_list3 = %r' % (flag_list1,))

    Example2:
        >>> # DISABLE_DOCTEST
        >>> # UNSTABLE_DOCTEST
        >>> # Trying to recreate the freeze seen in IBEIS
        >>> import vtool as vt
        >>> #def gen_chip(tup):
        >>> #    import vtool as vt
        >>> #    cfpath, gfpath, bbox, theta, new_size, filter_list = tup
        >>> #    chipBGR = vt.compute_chip(gfpath, bbox, theta, new_size, filter_list)
        >>> #    height, width = chipBGR.shape[0:2]
        >>> #    vt.imwrite(cfpath, chipBGR)
        >>> #    return cfpath, width, height
        >>> import utool as ut
        >>> from ibeis.algo.preproc.preproc_chip import gen_chip
        >>> #from ibeis.algo.preproc.preproc_feat import gen_feat_worker
        >>> key_list = ['grace.jpg', 'easy1.png', 'ada2.jpg', 'easy3.png',
        >>>             'hard3.png', 'zebra.png', 'patsy.jpg', 'ada.jpg',
        >>>             'carl.jpg', 'lena.png', 'easy2.png']
        >>> img_fpath_list = [ut.grab_test_imgpath(key) for key in key_list]
        >>> arg_list1 = [(ut.augpath(img_fpath, '_testgen'), img_fpath, (0, 0, 100, 100), 0.0, (545, 372), []) for img_fpath in img_fpath_list[0:1]]
        >>> arg_list2 = [(ut.augpath(img_fpath, '_testgen'), img_fpath, (0, 0, 100, 100), 0.0, (545, 372), []) for img_fpath in img_fpath_list]
        >>> #arg_list3 = [(count, fpath, {}) for count, fpath in enumerate(ut.get_list_column(arg_list1, 0))]
        >>> #arg_list4 = [(count, fpath, {}) for count, fpath in enumerate(ut.get_list_column(arg_list2, 0))]
        >>> ut.remove_file_list(ut.get_list_column(arg_list2, 0))
        >>> chips1 = [x for x in ut.generate2(gen_chip, arg_list1)]
        >>> chips2 = [y for y in ut.generate2(gen_chip, arg_list2, force_serial=True)]
        >>> #feats3 = [z for z in ut.generate2(gen_feat_worker, arg_list3)]
        >>> #feats4 = [w for w in ut.generate2(gen_feat_worker, arg_list4)]

    Example3:
        >>> # DISABLE_DOCTEST
        >>> # FAILING_DOCTEST
        >>> # Trying to recreate the freeze seen in IBEIS
        >>> # Extremely weird case: freezes only if dsize > (313, 313) AND __testwarp was called beforehand.
        >>> # otherwise the parallel loop works fine. Could be an opencv 3.0.0-dev issue.
        >>> import vtool as vt
        >>> import utool as ut
        >>> from ibeis.algo.preproc.preproc_chip import gen_chip
        >>> import cv2
        >>> from utool.util_parallel import __testwarp
        >>> key_list = ['grace.jpg', 'easy1.png', 'ada2.jpg', 'easy3.png',
        >>>             'hard3.png', 'zebra.png', 'patsy.jpg', 'ada.jpg',
        >>>             'carl.jpg', 'lena.png', 'easy2.png']
        >>> img_fpath_list = [ut.grab_test_imgpath(key) for key in key_list]
        >>> arg_list1 = [(vt.imread(fpath),) for fpath in img_fpath_list[0:1]]
        >>> arg_list2 = [(vt.imread(fpath),) for fpath in img_fpath_list]
        >>> #new1 = [x for x in ut.generate2(__testwarp, arg_list1)]
        >>> new1 =  __testwarp(arg_list1[0])
        >>> new2 = [y for y in ut.generate2(__testwarp, arg_list2, force_serial=False)]
        >>> #print('new2 = %r' % (new2,))

    #Example4:
    #    >>> # Freakin weird. When IBEIS Runs generate it doesn't close the processes
    #    >>> # UNSTABLE_DOCTEST
    #    >>> # python -m utool.util_parallel --test-generate:4
    #    >>> # Trying to see if we can recreate the problem where there are
    #    >>> # defunct python processes
    #    >>> import utool as ut
    #    >>> #num = 8700  # parallel is slower for smaller numbers
    #    >>> num = 70000  # parallel has an initial (~.1 second startup overhead)
    #    >>> print('TESTING PARALLEL')
    #    >>> flag_generator1 = list(ut.generate2(ut.is_prime, range(0, num)))
    #    >>> flag_generator1 = list(ut.generate2(ut.is_prime, range(0, num)))
    #    >>> import time
    #    >>> time.sleep(10)

def generate2(func, args_gen, kw_gen=None, ntasks=None, ordered=True,
              force_serial=False, use_pool=False, chunksize=None, nprocs=None,
              progkw={}, nTasks=None, verbose=None):
    r"""
    Interfaces to either multiprocessing or futures.
    Esentially maps ``args_gen`` onto ``func`` using pool.imap.
    However, args_gen must be a tuple of args that will be unpacked and send to
    the function. Thus, the function can take multiple args. Also specifing
    keyword args is supported.

    Useful for embarrassingly parallel loops. Currently does not work with
    opencv3

    CommandLine:
        python -m utool.util_parallel generate2

    Args:
        func (function):  live python function
        args_gen (?):
        kw_gen (None): (default = None)
        ntasks (None): (default = None)
        ordered (bool): (default = True)
        force_serial (bool): (default = False)
        verbose (bool):  verbosity flag(default = None)

    CommandLine:
        python -m utool.util_parallel generate2

    Example:
        >>> # DISABLE_DOCTEST
        >>> from utool.util_parallel import *  # NOQA
        >>> from utool.util_parallel import _kw_wrap_worker  # NOQA
        >>> import utool as ut
        >>> args_gen = list(zip(range(10000)))
        >>> kw_gen = [{}] * len(args_gen)
        >>> func = ut.is_prime
        >>> _ = list(generate2(func, args_gen))
        >>> _ = list(generate2(func, args_gen, ordered=False))
        >>> _ = list(generate2(func, args_gen, force_serial=True))
        >>> _ = list(generate2(func, args_gen, use_pool=True))
        >>> _ = list(generate2(func, args_gen, ordered=False, verbose=False))

    Example0:
        >>> # ENABLE_DOCTEST
        >>> import utool as ut
        >>> #num = 8700  # parallel is slower for smaller numbers
        >>> num = 500  # parallel has an initial (~.1 second startup overhead)
        >>> print('TESTING SERIAL')
        >>> func = ut.is_prime
        >>> args_list = list(range(0, num))
        >>> flag_generator0 = ut.generate2(ut.is_prime, zip(range(0, num)), force_serial=True)
        >>> flag_list0 = list(flag_generator0)
        >>> print('TESTING PARALLEL')
        >>> flag_generator1 = ut.generate2(ut.is_prime, zip(range(0, num)))
        >>> flag_list1 = list(flag_generator1)
        >>> print('ASSERTING')
        >>> assert len(flag_list1) == num
        >>> assert flag_list0 == flag_list1

    Example1:
        >>> # ENABLE_DOCTEST
        >>> # Trying to recreate the freeze seen in IBEIS
        >>> import utool as ut
        >>> print('TESTING SERIAL')
        >>> flag_generator0 = ut.generate2(ut.is_prime, zip(range(0, 1)))
        >>> flag_list0 = list(flag_generator0)
        >>> flag_generator1 = ut.generate2(ut.fibonacci_recursive, zip(range(0, 1)))
        >>> flag_list1 = list(flag_generator1)
        >>> print('TESTING PARALLEL')
        >>> flag_generator2 = ut.generate2(ut.is_prime, zip(range(0, 12)))
        >>> flag_list2 = list(flag_generator2)
        >>> flag_generator3 = ut.generate2(ut.fibonacci_recursive, zip(range(0, 12)))
        >>> flag_list3 = list(flag_generator3)
        >>> print('flag_list0 = %r' % (flag_list0,))
        >>> print('flag_list1 = %r' % (flag_list1,))
        >>> print('flag_list2 = %r' % (flag_list1,))
        >>> print('flag_list3 = %r' % (flag_list1,))

    Example2:
        >>> # DISABLE_DOCTEST
        >>> # UNSTABLE_DOCTEST
        >>> # Trying to recreate the freeze seen in IBEIS
        >>> import vtool as vt
        >>> #def gen_chip(tup):
        >>> #    import vtool as vt
        >>> #    cfpath, gfpath, bbox, theta, new_size, filter_list = tup
        >>> #    chipBGR = vt.compute_chip(gfpath, bbox, theta, new_size, filter_list)
        >>> #    height, width = chipBGR.shape[0:2]
        >>> #    vt.imwrite(cfpath, chipBGR)
        >>> #    return cfpath, width, height
        >>> import utool as ut
        >>> from ibeis.algo.preproc.preproc_chip import gen_chip
        >>> #from ibeis.algo.preproc.preproc_feat import gen_feat_worker
        >>> key_list = ['grace.jpg', 'easy1.png', 'ada2.jpg', 'easy3.png',
        >>>             'hard3.png', 'zebra.png', 'patsy.jpg', 'ada.jpg',
        >>>             'carl.jpg', 'lena.png', 'easy2.png']
        >>> img_fpath_list = [ut.grab_test_imgpath(key) for key in key_list]
        >>> arg_list1 = [(ut.augpath(img_fpath, '_testgen'), img_fpath, (0, 0, 100, 100), 0.0, (545, 372), []) for img_fpath in img_fpath_list[0:1]]
        >>> arg_list2 = [(ut.augpath(img_fpath, '_testgen'), img_fpath, (0, 0, 100, 100), 0.0, (545, 372), []) for img_fpath in img_fpath_list]
        >>> #arg_list3 = [(count, fpath, {}) for count, fpath in enumerate(ut.get_list_column(arg_list1, 0))]
        >>> #arg_list4 = [(count, fpath, {}) for count, fpath in enumerate(ut.get_list_column(arg_list2, 0))]
        >>> ut.remove_file_list(ut.get_list_column(arg_list2, 0))
        >>> chips1 = [x for x in ut.generate2(gen_chip, arg_list1)]
        >>> chips2 = [y for y in ut.generate2(gen_chip, arg_list2, force_serial=True)]
        >>> #feats3 = [z for z in ut.generate2(gen_feat_worker, arg_list3)]
        >>> #feats4 = [w for w in ut.generate2(gen_feat_worker, arg_list4)]

    Example3:
        >>> # DISABLE_DOCTEST
        >>> # FAILING_DOCTEST
        >>> # Trying to recreate the freeze seen in IBEIS
        >>> # Extremely weird case: freezes only if dsize > (313, 313) AND __testwarp was called beforehand.
        >>> # otherwise the parallel loop works fine. Could be an opencv 3.0.0-dev issue.
        >>> import vtool as vt
        >>> import utool as ut
        >>> from ibeis.algo.preproc.preproc_chip import gen_chip
        >>> import cv2
        >>> from utool.util_parallel import __testwarp
        >>> key_list = ['grace.jpg', 'easy1.png', 'ada2.jpg', 'easy3.png',
        >>>             'hard3.png', 'zebra.png', 'patsy.jpg', 'ada.jpg',
        >>>             'carl.jpg', 'lena.png', 'easy2.png']
        >>> img_fpath_list = [ut.grab_test_imgpath(key) for key in key_list]
        >>> arg_list1 = [(vt.imread(fpath),) for fpath in img_fpath_list[0:1]]
        >>> arg_list2 = [(vt.imread(fpath),) for fpath in img_fpath_list]
        >>> #new1 = [x for x in ut.generate2(__testwarp, arg_list1)]
        >>> new1 =  __testwarp(arg_list1[0])
        >>> new2 = [y for y in ut.generate2(__testwarp, arg_list2, force_serial=False)]
        >>> #print('new2 = %r' % (new2,))

    #Example4:
    #    >>> # Freakin weird. When IBEIS Runs generate it doesn't close the processes
    #    >>> # UNSTABLE_DOCTEST
    #    >>> # python -m utool.util_parallel --test-generate:4
    #    >>> # Trying to see if we can recreate the problem where there are
    #    >>> # defunct python processes
    #    >>> import utool as ut
    #    >>> #num = 8700  # parallel is slower for smaller numbers
    #    >>> num = 70000  # parallel has an initial (~.1 second startup overhead)
    #    >>> print('TESTING PARALLEL')
    #    >>> flag_generator1 = list(ut.generate2(ut.is_prime, range(0, num)))
    #    >>> flag_generator1 = list(ut.generate2(ut.is_prime, range(0, num)))
    #    >>> import time
    #    >>> time.sleep(10)
    """
    if verbose is None:
        verbose = 2
    if ntasks is None:
        ntasks = nTasks
    if ntasks is None:
        try:
            ntasks = len(args_gen)
        except TypeError:
            # Cast to a list
            args_gen = list(args_gen)
            ntasks = len(args_gen)
    if ntasks == 1 or ntasks < __MIN_PARALLEL_TASKS__:
        force_serial = True
    if __FORCE_SERIAL__:
        force_serial = __FORCE_SERIAL__
    if ntasks == 0:
        if verbose:
            print('[ut.generate2] submitted 0 tasks')
        raise StopIteration
    if nprocs is None:
        nprocs = min(ntasks, get_default_numprocs())
    if nprocs == 1:
        force_serial = True

    if kw_gen is None:
        kw_gen = [{}] * ntasks
    if isinstance(kw_gen, dict):
        # kw_gen can be a single dict applied to everything
        kw_gen = [kw_gen] * ntasks

    if force_serial:
        for result in _generate_serial2(func, args_gen, kw_gen,
                                        ntasks=ntasks, progkw=progkw,
                                        verbose=verbose):
            yield result
    else:
        if verbose:
            gentype = 'mp' if use_pool else 'futures'
            fmtstr = '[generate2] executing {} {} tasks using {} {} procs'
            print(fmtstr.format(ntasks, get_funcname(func), nprocs, gentype))

        if verbose > 1:
            lbl = '(pargen) %s: ' % (get_funcname(func),)
            progkw_ = dict(freq=None, bs=True, adjust=False, freq_est='absolute')
            progkw_.update(progkw)
            # print('progkw_.update = {!r}'.format(progkw_.update))
            progpart = util_progress.ProgPartial(length=ntasks, lbl=lbl, **progkw_)

        if use_pool:
            # Use multiprocessing
            if chunksize is None:
                chunksize = max(min(4, ntasks), min(8, ntasks // (nprocs ** 2)))

            try:
                pool = multiprocessing.Pool(nprocs)
                if ordered:
                    pmap_func = pool.imap
                else:
                    pmap_func = pool.imap_unordered

                wrapped_arg_gen = zip([func] * len(args_gen), args_gen, kw_gen)
                res_gen = pmap_func(_kw_wrap_worker, wrapped_arg_gen,
                                    chunksize)
                if verbose > 1:
                    res_gen = progpart(res_gen)
                for res in res_gen:
                    yield res
            finally:
                pool.close()
                pool.join()
        else:
            # Use futures
            executor = futures.ProcessPoolExecutor(nprocs)
            try:
                fs_list = [executor.submit(func, *a, **k)
                           for a, k in zip(args_gen, kw_gen)]
                fs_gen = fs_list
                if not ordered:
                    fs_gen = futures.as_completed(fs_gen)
                if verbose > 1:
                    fs_gen = progpart(fs_gen)
                for fs in fs_gen:
                    yield fs.result()
            finally:
                executor.shutdown(wait=True)