s4cmd.py
def dsync_files(self, source, target):
    '''Sync directory to directory.'''
    src_s3_url = S3URL.is_valid(source)
    dst_s3_url = S3URL.is_valid(target)

    source_list = self.relative_dir_walk(source)
    if len(source_list) == 0 or '.' in source_list:
      raise Failure('Sync command need to sync directory to directory.')

    sync_list = [(os.path.join(source, f), os.path.join(target, f)) for f in source_list]

    pool = ThreadPool(ThreadUtil, self.opt)
    if src_s3_url and not dst_s3_url:
      for src, dest in sync_list:
        pool.download(src, dest)
    elif not src_s3_url and dst_s3_url:
      for src, dest in sync_list:
        pool.upload(src, dest)
    elif src_s3_url and dst_s3_url:
      for src, dest in sync_list:
        pool.copy(src, dest)
    else:
      raise InvalidArgument('Cannot sync two local directories.')
    pool.join()

    if self.opt.delete_removed:
      target_list = self.relative_dir_walk(target)
      remove_list = [os.path.join(target, f) for f in (set(target_list) - set(source_list))]

      if S3URL.is_valid(target):
        pool = ThreadPool(ThreadUtil, self.opt)
        pool.batch_delete(remove_list)
        pool.join()
      else:
        for f in remove_list:
          try:
            os.unlink(f)
            message('Delete %s', f)
          except:
            pass
def dsync_files(self, source, target):
    '''Sync directory to directory.'''
    src_s3_url = S3URL.is_valid(source)
    dst_s3_url = S3URL.is_valid(target)

    source_list = self.relative_dir_walk(source)
    if len(source_list) == 0 or '.' in source_list:
      raise Failure('Sync command need to sync directory to directory.')

    sync_list = [(os.path.join(source, f), os.path.join(target, f)) for f in source_list]

    pool = ThreadPool(ThreadUtil, self.opt)
    if src_s3_url and not dst_s3_url:
      for src, dest in sync_list:
        pool.download(src, dest)
    elif not src_s3_url and dst_s3_url:
      for src, dest in sync_list:
        pool.upload(src, dest)
    elif src_s3_url and dst_s3_url:
      for src, dest in sync_list:
        pool.copy(src, dest)
    else:
      raise InvalidArgument('Cannot sync two local directories.')
    pool.join()

    if self.opt.delete_removed:
      target_list = self.relative_dir_walk(target)
      remove_list = [os.path.join(target, f) for f in (set(target_list) - set(source_list))]

      if S3URL.is_valid(target):
        pool = ThreadPool(ThreadUtil, self.opt)
        pool.batch_delete(remove_list)
        pool.join()
      else:
        for f in remove_list:
          try:
            os.unlink(f)
            message('Delete %s', f)
          except:
            pass
def dsync_files ( self , source , target ) : src_s3_url = S3URL . is_valid ( source ) dst_s3_url = S3URL . is_valid ( target ) source_list = self . relative_dir_walk ( source ) if len ( source_list ) == 0 or '.' in source_list : raise Failure ( 'Sync command need to sync directory to directory.' ) sync_list = [ ( os . path . join ( source , f ) , os . path . join ( target , f ) ) for f in source_list ] pool = ThreadPool ( ThreadUtil , self . opt ) if src_s3_url and not dst_s3_url : for src , dest in sync_list : pool . download ( src , dest ) elif not src_s3_url and dst_s3_url : for src , dest in sync_list : pool . upload ( src , dest ) elif src_s3_url and dst_s3_url : for src , dest in sync_list : pool . copy ( src , dest ) else : raise InvalidArgument ( 'Cannot sync two local directories.' ) pool . join ( ) if self . opt . delete_removed : target_list = self . relative_dir_walk ( target ) remove_list = [ os . path . join ( target , f ) for f in ( set ( target_list ) - set ( source_list ) ) ] if S3URL . is_valid ( target ) : pool = ThreadPool ( ThreadUtil , self . opt ) pool . batch_delete ( remove_list ) pool . join ( ) else : for f in remove_list : try : os . unlink ( f ) message ( 'Delete %s' , f ) except : pass
Sync directory to directory.
Sync directory to directory .