Retrieve all s3 records for the given policy output url

    From the given start date.

def record_set(session_factory, bucket, key_prefix, start_date, specify_hour=False):
    """Retrieve all s3 records for the given policy output url

    From the given start date.
    """

    s3 = local_session(session_factory).client('s3')

    records = []
    key_count = 0

    date = start_date.strftime('%Y/%m/%d')
    if specify_hour:
        date += "/{}".format(start_date.hour)
    else:
        date += "/00"

    marker = "{}/{}/resources.json.gz".format(key_prefix.strip("/"), date)

    p = s3.get_paginator('list_objects_v2').paginate(
        Bucket=bucket,
        Prefix=key_prefix.strip('/') + '/',
        StartAfter=marker,
    )

    with ThreadPoolExecutor(max_workers=20) as w:
        for key_set in p:
            if 'Contents' not in key_set:
                continue
            keys = [k for k in key_set['Contents']
                    if k['Key'].endswith('resources.json.gz')]
            key_count += len(keys)
            futures = map(lambda k: w.submit(
                get_records, bucket, k, session_factory), keys)

            for f in as_completed(futures):
                records.extend(f.result())

    log.info("Fetched %d records across %d files" % (
        len(records), key_count))
    return records