s4cmd.py
def s3walk(self, s3url, s3dir, filter_path, result):
    '''Thread worker for s3walk.
       Recursively walk into all subdirectories if they still match the filter
       path partially.
    '''

    paginator = self.s3.get_paginator('list_objects')
    filter_path_level = filter_path.count(PATH_SEP)

    for page in paginator.paginate(Bucket=s3url.bucket, Prefix=s3dir, Delimiter=PATH_SEP, PaginationConfig={'PageSize': 1000}):
      # Get subdirectories first.
      for obj in page.get('CommonPrefixes') or []:
        obj_name = obj['Prefix']

        if not self.partial_match(obj_name, filter_path):
          continue

        if self.opt.recursive or (obj_name.count(PATH_SEP) != filter_path_level + 1):
          self.pool.s3walk(s3url, obj_name, filter_path, result)
        else:
          self.conditional(result, {
            'name': S3URL.combine(s3url.proto, s3url.bucket, obj_name),
            'is_dir': True,
            'size': 0,
            'last_modified': None
          })

      # Then get all items in this folder.
      for obj in page.get('Contents') or []:
        obj_name = obj['Key']
        if not self.partial_match(obj_name, filter_path):
          continue

        if self.opt.recursive or obj_name.count(PATH_SEP) == filter_path_level:
          self.conditional(result, {
            'name': S3URL.combine(s3url.proto, s3url.bucket, obj_name),
            'is_dir': False,
            'size': obj['Size'],
            'last_modified': obj['LastModified']
          })
def s3walk(self, s3url, s3dir, filter_path, result):
    '''Thread worker for s3walk.
       Recursively walk into all subdirectories if they still match the filter
       path partially.
    '''

    paginator = self.s3.get_paginator('list_objects')
    filter_path_level = filter_path.count(PATH_SEP)

    for page in paginator.paginate(Bucket=s3url.bucket, Prefix=s3dir, Delimiter=PATH_SEP, PaginationConfig={'PageSize': 1000}):
      # Get subdirectories first.
      for obj in page.get('CommonPrefixes') or []:
        obj_name = obj['Prefix']

        if not self.partial_match(obj_name, filter_path):
          continue

        if self.opt.recursive or (obj_name.count(PATH_SEP) != filter_path_level + 1):
          self.pool.s3walk(s3url, obj_name, filter_path, result)
        else:
          self.conditional(result, {
            'name': S3URL.combine(s3url.proto, s3url.bucket, obj_name),
            'is_dir': True,
            'size': 0,
            'last_modified': None
          })

      # Then get all items in this folder.
      for obj in page.get('Contents') or []:
        obj_name = obj['Key']
        if not self.partial_match(obj_name, filter_path):
          continue

        if self.opt.recursive or obj_name.count(PATH_SEP) == filter_path_level:
          self.conditional(result, {
            'name': S3URL.combine(s3url.proto, s3url.bucket, obj_name),
            'is_dir': False,
            'size': obj['Size'],
            'last_modified': obj['LastModified']
          })
def s3walk ( self , s3url , s3dir , filter_path , result ) : paginator = self . s3 . get_paginator ( 'list_objects' ) filter_path_level = filter_path . count ( PATH_SEP ) for page in paginator . paginate ( Bucket = s3url . bucket , Prefix = s3dir , Delimiter = PATH_SEP , PaginationConfig = { 'PageSize' : 1000 } ) : # Get subdirectories first. for obj in page . get ( 'CommonPrefixes' ) or [ ] : obj_name = obj [ 'Prefix' ] if not self . partial_match ( obj_name , filter_path ) : continue if self . opt . recursive or ( obj_name . count ( PATH_SEP ) != filter_path_level + 1 ) : self . pool . s3walk ( s3url , obj_name , filter_path , result ) else : self . conditional ( result , { 'name' : S3URL . combine ( s3url . proto , s3url . bucket , obj_name ) , 'is_dir' : True , 'size' : 0 , 'last_modified' : None } ) # Then get all items in this folder. for obj in page . get ( 'Contents' ) or [ ] : obj_name = obj [ 'Key' ] if not self . partial_match ( obj_name , filter_path ) : continue if self . opt . recursive or obj_name . count ( PATH_SEP ) == filter_path_level : self . conditional ( result , { 'name' : S3URL . combine ( s3url . proto , s3url . bucket , obj_name ) , 'is_dir' : False , 'size' : obj [ 'Size' ] , 'last_modified' : obj [ 'LastModified' ] } )
Thread worker for s3walk.
       Recursively walk into all subdirectories if they still match the filter
       path partially.
Thread worker for s3walk . Recursively walk into all subdirectories if they still match the filter path partially .