def consume_queue(queue, action, thread_pool_size, max_retry):
    thread_pool = []

    while queue.is_queuing() or not queue.is_empty() or len(thread_pool) != 0:
        # de-pool the done threads
        for t in thread_pool:
            if not t.is_alive():
                thread_pool.remove(t)

        # en-pool new threads
        if not queue.is_empty() and len(thread_pool) <= thread_pool_size:
            t = threading.Thread(target=process_a_key, args=[queue, action, max_retry])
            t.start()
            thread_pool.append(t)

    queue.all_processed = True
Consumes the queue with the designated thread poll size by uploading/downloading the keys to
dataset/ETHPy150Open quid/s3concurrent/s3concurrent/s3concurrent.py/consume_queue