def _prep_s3_directories(args, buckets):
    dirs = set([])
    def _get_dirs(fname, context, remap_dict):
        dirs.add(os.path.normpath(os.path.dirname(os.path.abspath(fname))))
    remap.walk_files(args, _get_dirs, {}, pass_dirs=True)
    work_dir, biodata_dir = _get_known_dirs(args)
    out = {}
    external_count = 0
    for d in sorted(dirs):
        if work_dir and d.startswith(work_dir):
            folder = d.replace(work_dir, "")
            folder = folder[1:] if folder.startswith("/") else folder
            out[d] = {"bucket": buckets["run"],
                      "folder": folder}
        elif biodata_dir and d.startswith(biodata_dir):
            folder = d.replace(biodata_dir, "")
            folder = folder[1:] if folder.startswith("/") else folder
            out[d] = {"bucket": buckets["biodata"],
                      "folder": folder}
        else:
            folder = os.path.join("externalmap", str(external_count))
            out[d] = {"bucket": buckets["run"],
                      "folder": folder}
            external_count += 1
    return out
Map input directories into stable S3 buckets and folders for storing files.
dataset/ETHPy150Open chapmanb/bcbio-nextgen-vm/bcbiovm/ship/pack.py/_prep_s3_directories