index traildbs directly from s3 for multiple accounts.

    context: assumes a daily traildb file in s3 with key path
             specified by key_template in config file for each account

def index(config, start, end, incremental=False, concurrency=5, accounts=None,
          verbose=False):
    """index traildbs directly from s3 for multiple accounts.

    context: assumes a daily traildb file in s3 with key path
             specified by key_template in config file for each account
    """
    with open(config) as fh:
        config = yaml.safe_load(fh.read())
    jsonschema.validate(config, CONFIG_SCHEMA)

    if verbose:
        logging.root.setLevel(logging.DEBUG)
    log.info("tmpdir %s" % os.environ.get('TMPDIR'))

    with ProcessPoolExecutor(max_workers=concurrency) as w:
        futures = {}

        if incremental:
            account_starts = get_incremental_starts(config, start)
        else:
            account_starts = defaultdict(lambda : start) # NOQA E203

        for account in config.get('accounts'):
            if accounts and account['name'] not in accounts:
                continue
            for region in account.get('regions'):
                for d in get_date_range(
                        account_starts[(account['name'], region)], end):
                    i = bool(d.hour or d.minute)
                    p = (config, region, account, d, i)
                    futures[w.submit(index_account, *p)] = p

        for f in as_completed(futures):
            _, region, account, d, incremental = futures[f]

            result = f.result()
            if result is None:
                continue
            log.info(
                ("processed account:%(account)s day:%(day)s region:%(region)s "
                 "records:%(records)s time:%(time)0.2f db-date:%(db-date)s"
                 ) % result)