

@db.non_transactional
def get_bucket(self, bucketpath, prefix, marker, max_keys, delimiter):
    "Get bucket listing with a GET.\n\n    How GCS listbucket work in production:\n    GCS tries to return as many items as possible in a single response. If\n    there are more items satisfying user's query and the current request\n    took too long (e.g spent on skipping files in a subdir) or items to return\n    gets too big (> max_keys), it returns fast and sets IsTruncated\n    and NextMarker for continuation. They serve redundant purpose: if\n    NextMarker is set, IsTruncated is True.\n\n    Note NextMarker is not where GCS scan left off. It is\n    only valid for the exact same type of query the marker was generated from.\n    For example, if a marker is generated from query with delimiter, the marker\n    is the name of a subdir (instead of the last file within the subdir). Thus\n    you can't use this marker to issue a query without delimiter.\n\n    Args:\n      bucketpath: gcs bucket path of form '/bucket'\n      prefix: prefix to limit listing.\n      marker: a str after which to start listing. Exclusive.\n      max_keys: max items we scan & return.\n      delimiter: delimiter for directory.\n\n    See https://developers.google.com/storage/docs/reference-methods#getbucket\n    for details.\n\n    Returns:\n      A tuple of (a list of GCSFileStat for files or directories sorted by\n      filename, next_marker to use as next marker, is_truncated boolean to\n      indicate if there are more results satisfying query).\n    "
    common.validate_bucket_path(bucketpath)
    q = _AE_GCSFileInfo_.all(namespace='')
    fully_qualified_prefix = '/'.join([bucketpath, prefix])
    if marker:
        q.filter('filename >', '/'.join([bucketpath, marker]))
    else:
        q.filter('filename >=', fully_qualified_prefix)
    result = set()
    name = None
    first = True
    first_dir = None
    for info in q.run():
        if (not info.filename.startswith(fully_qualified_prefix)):
            break
        if (len(result) == max_keys):
            break
        info = db.get(info.key())
        if (not info):
            continue
        name = info.filename
        if delimiter:
            start_index = name.find(delimiter, len(fully_qualified_prefix))
            if (start_index != (- 1)):
                name = name[:(start_index + len(delimiter))]
                if (marker and (first or (name == first_dir))):
                    first = False
                    first_dir = name
                else:
                    result.add(common.GCSFileStat(name, st_size=None, st_ctime=None, etag=None, is_dir=True))
                continue
        if info.finalized:
            first = False
            result.add(common.GCSFileStat(filename=name, st_size=info.size, st_ctime=calendar.timegm(info.creation.utctimetuple()), etag=info.etag))

    def is_truncated():
        'Check if there are more results satisfying the query.'
        if (not result):
            return False
        q = _AE_GCSFileInfo_.all(namespace='')
        q.filter('filename >', name)
        info = None
        if (delimiter and name.endswith(delimiter)):
            for info in q.run():
                if (not info.filename.startswith(name)):
                    break
            if ((info is not None) and info.filename.startswith(name)):
                info = None
        else:
            info = q.get()
        if ((info is None) or (not info.filename.startswith(fully_qualified_prefix))):
            return False
        return True
    result = list(result)
    result.sort()
    truncated = is_truncated()
    next_marker = (name if truncated else None)
    return (result, next_marker, truncated)

dataset/ETHPy150Open GoogleCloudPlatform/python-compat-runtime/appengine-compat/exported_appengine_sdk/google/appengine/ext/cloudstorage/cloudstorage_stub.py CloudStorageStub.get_bucket/original