def enqueue_sources(queue, the_set, sources):
    expected_paths = set()
    commit_sha = None
    
    #
    # Enqueue each source if there is nothing else in the queue.
    #
    for source in sources:
        while len(queue) >= 1:
            yield len(expected_paths)
        
        with queue as db:
            _L.info(u'Sending {path} to task queue'.format(**source))
            task_data = dict(job_id=None, url=None, set_id=the_set.id,
                             name=source['path'],
                             content_b64=source['content'],
                             commit_sha=source['commit_sha'],
                             file_id=source['blob_sha'])
        
            task_id = queue.put(task_data)
            expected_paths.add(source['path'])
            commit_sha = source['commit_sha']
    
    while len(expected_paths):
        with queue as db:
            _update_expected_paths(db, expected_paths, the_set)

        yield len(expected_paths)

    with queue as db:
        complete_set(db, the_set.id, commit_sha)

    yield 0
Batch task generator, yields counts of remaining expected paths.
dataset/ETHPy150Open openaddresses/machine/openaddr/ci/__init__.py/enqueue_sources