<s> """<STR_LIT>""" <EOL> from __future__ import absolute_import <EOL> import os <EOL> import time <EOL> import traceback <EOL> from gslib import copy_helper <EOL> from gslib . cat_helper import CatHelper <EOL> from gslib . command import Command <EOL> from gslib . command_argument import CommandArgument <EOL> from gslib . commands . compose import MAX_COMPONENT_COUNT <EOL> from gslib . copy_helper import CreateCopyHelperOpts <EOL> from gslib . copy_helper import GZIP_ALL_FILES <EOL> from gslib . copy_helper import ItemExistsError <EOL> from gslib . copy_helper import Manifest <EOL> from gslib . copy_helper import PARALLEL_UPLOAD_TEMP_NAMESPACE <EOL> from gslib . copy_helper import SkipUnsupportedObjectError <EOL> from gslib . cs_api_map import ApiSelector <EOL> from gslib . exception import CommandException <EOL> from gslib . name_expansion import NameExpansionIterator <EOL> from gslib . storage_url import ContainsWildcard <EOL> from gslib . util import CreateLock <EOL> from gslib . util import DEBUGLEVEL_DUMP_REQUESTS <EOL> from gslib . util import GetCloudApiInstance <EOL> from gslib . util import IsCloudSubdirPlaceholder <EOL> from gslib . util import MakeHumanReadable <EOL> from gslib . util import NO_MAX <EOL> from gslib . util import RemoveCRLFFromString <EOL> from gslib . util import StdinIterator <EOL> _SYNOPSIS = """<STR_LIT>""" <EOL> _SYNOPSIS_TEXT = """<STR_LIT>""" + _SYNOPSIS <EOL> _DESCRIPTION_TEXT = """<STR_LIT>""" <EOL> _NAME_CONSTRUCTION_TEXT = """<STR_LIT>""" <EOL> _SUBDIRECTORIES_TEXT = """<STR_LIT>""" <EOL> _COPY_IN_CLOUD_TEXT = """<STR_LIT>""" <EOL> _CHECKSUM_VALIDATION_TEXT = """<STR_LIT>""" <EOL> _RETRY_HANDLING_TEXT = """<STR_LIT>""" <EOL> _RESUMABLE_TRANSFERS_TEXT = """<STR_LIT>""" <EOL> _STREAMING_TRANSFERS_TEXT = """<STR_LIT>""" <EOL> _SLICED_OBJECT_DOWNLOADS_TEXT = """<STR_LIT>""" <EOL> _PARALLEL_COMPOSITE_UPLOADS_TEXT = """<STR_LIT>""" % ( PARALLEL_UPLOAD_TEMP_NAMESPACE , <NUM_LIT:10> , MAX_COMPONENT_COUNT - <NUM_LIT:9> , <EOL> MAX_COMPONENT_COUNT ) <EOL> _CHANGING_TEMP_DIRECTORIES_TEXT = """<STR_LIT>""" <EOL> _COPYING_SPECIAL_FILES_TEXT = """<STR_LIT>""" <EOL> _OPTIONS_TEXT = """<STR_LIT>""" <EOL> _DETAILED_HELP_TEXT = '<STR_LIT>' . join ( [ _SYNOPSIS_TEXT , <EOL> _DESCRIPTION_TEXT , <EOL> _NAME_CONSTRUCTION_TEXT , <EOL> _SUBDIRECTORIES_TEXT , <EOL> _COPY_IN_CLOUD_TEXT , <EOL> _CHECKSUM_VALIDATION_TEXT , <EOL> _RETRY_HANDLING_TEXT , <EOL> _RESUMABLE_TRANSFERS_TEXT , <EOL> _STREAMING_TRANSFERS_TEXT , <EOL> _SLICED_OBJECT_DOWNLOADS_TEXT , <EOL> _PARALLEL_COMPOSITE_UPLOADS_TEXT , <EOL> _CHANGING_TEMP_DIRECTORIES_TEXT , <EOL> _COPYING_SPECIAL_FILES_TEXT , <EOL> _OPTIONS_TEXT ] ) <EOL> CP_SUB_ARGS = '<STR_LIT>' <EOL> def _CopyFuncWrapper ( cls , args , thread_state = None ) : <EOL> cls . CopyFunc ( args , thread_state = thread_state ) <EOL> def _CopyExceptionHandler ( cls , e ) : <EOL> """<STR_LIT>""" <EOL> cls . logger . error ( str ( e ) ) <EOL> cls . op_failure_count += <NUM_LIT:1> <EOL> cls . logger . debug ( '<STR_LIT>' , <EOL> traceback . format_exc ( ) ) <EOL> def _RmExceptionHandler ( cls , e ) : <EOL> """<STR_LIT>""" <EOL> cls . logger . error ( str ( e ) ) <EOL> class CpCommand ( Command ) : <EOL> """<STR_LIT>""" <EOL> command_spec = Command . CreateCommandSpec ( <EOL> '<STR_LIT>' , <EOL> command_name_aliases = [ '<STR_LIT>' ] , <EOL> usage_synopsis = _SYNOPSIS , <EOL> min_args = <NUM_LIT:1> , <EOL> max_args = NO_MAX , <EOL> supported_sub_args = CP_SUB_ARGS , <EOL> file_url_ok = True , <EOL> provider_url_ok = False , <EOL> urls_start_arg = <NUM_LIT:0> , <EOL> gs_api_support = [ ApiSelector . XML , ApiSelector . JSON ] , <EOL> gs_default_api = ApiSelector . JSON , <EOL> supported_private_args = [ '<STR_LIT>' ] , <EOL> argparse_arguments = [ <EOL> CommandArgument . MakeZeroOrMoreCloudOrFileURLsArgument ( ) <EOL> ] <EOL> ) <EOL> help_spec = Command . HelpSpec ( <EOL> help_name = '<STR_LIT>' , <EOL> help_name_aliases = [ '<STR_LIT>' ] , <EOL> help_type = '<STR_LIT>' , <EOL> help_one_line_summary = '<STR_LIT>' , <EOL> help_text = _DETAILED_HELP_TEXT , <EOL> subcommand_help_text = { } , <EOL> ) <EOL> def CopyFunc ( self , name_expansion_result , thread_state = None ) : <EOL> """<STR_LIT>""" <EOL> gsutil_api = GetCloudApiInstance ( self , thread_state = thread_state ) <EOL> copy_helper_opts = copy_helper . GetCopyHelperOpts ( ) <EOL> if copy_helper_opts . perform_mv : <EOL> cmd_name = '<STR_LIT>' <EOL> else : <EOL> cmd_name = self . command_name <EOL> src_url = name_expansion_result . source_storage_url <EOL> exp_src_url = name_expansion_result . expanded_storage_url <EOL> src_url_names_container = name_expansion_result . names_container <EOL> have_multiple_srcs = name_expansion_result . is_multi_source_request <EOL> if src_url . IsCloudUrl ( ) and src_url . IsProvider ( ) : <EOL> raise CommandException ( <EOL> '<STR_LIT>' % <EOL> ( cmd_name , src_url ) ) <EOL> if have_multiple_srcs : <EOL> copy_helper . InsistDstUrlNamesContainer ( <EOL> self . exp_dst_url , self . have_existing_dst_container , cmd_name ) <EOL> if IsCloudSubdirPlaceholder ( exp_src_url ) : <EOL> return <EOL> if copy_helper_opts . use_manifest and self . manifest . WasSuccessful ( <EOL> exp_src_url . url_string ) : <EOL> return <EOL> if copy_helper_opts . perform_mv : <EOL> if name_expansion_result . names_container : <EOL> self . recursion_requested = True <EOL> if ContainsWildcard ( src_url . url_string ) : <EOL> raise CommandException ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if ( self . exp_dst_url . IsFileUrl ( ) <EOL> and not os . path . exists ( self . exp_dst_url . object_name ) <EOL> and have_multiple_srcs ) : <EOL> os . makedirs ( self . exp_dst_url . object_name ) <EOL> dst_url = copy_helper . ConstructDstUrl ( <EOL> src_url , exp_src_url , src_url_names_container , have_multiple_srcs , <EOL> self . exp_dst_url , self . have_existing_dst_container , <EOL> self . recursion_requested ) <EOL> dst_url = copy_helper . FixWindowsNaming ( src_url , dst_url ) <EOL> copy_helper . CheckForDirFileConflict ( exp_src_url , dst_url ) <EOL> if copy_helper . SrcDstSame ( exp_src_url , dst_url ) : <EOL> raise CommandException ( '<STR_LIT>' <EOL> '<STR_LIT>' % ( cmd_name , exp_src_url , dst_url ) ) <EOL> if dst_url . IsCloudUrl ( ) and dst_url . HasGeneration ( ) : <EOL> raise CommandException ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> % ( cmd_name , dst_url ) ) <EOL> elapsed_time = bytes_transferred = <NUM_LIT:0> <EOL> try : <EOL> if copy_helper_opts . use_manifest : <EOL> self . manifest . Initialize ( <EOL> exp_src_url . url_string , dst_url . url_string ) <EOL> ( elapsed_time , bytes_transferred , result_url , md5 ) = ( <EOL> copy_helper . PerformCopy ( <EOL> self . logger , exp_src_url , dst_url , gsutil_api , <EOL> self , _CopyExceptionHandler , allow_splitting = True , <EOL> headers = self . headers , manifest = self . manifest , <EOL> gzip_exts = self . gzip_exts ) ) <EOL> if copy_helper_opts . use_manifest : <EOL> if md5 : <EOL> self . manifest . Set ( exp_src_url . url_string , '<STR_LIT>' , md5 ) <EOL> self . manifest . SetResult ( <EOL> exp_src_url . url_string , bytes_transferred , '<STR_LIT:OK>' ) <EOL> if copy_helper_opts . print_ver : <EOL> self . logger . info ( '<STR_LIT>' , result_url ) <EOL> except ItemExistsError : <EOL> message = '<STR_LIT>' % dst_url <EOL> self . logger . info ( message ) <EOL> if copy_helper_opts . use_manifest : <EOL> self . manifest . SetResult ( exp_src_url . url_string , <NUM_LIT:0> , '<STR_LIT>' , message ) <EOL> except SkipUnsupportedObjectError , e : <EOL> message = ( '<STR_LIT>' % <EOL> ( exp_src_url . url_string , e . unsupported_type ) ) <EOL> self . logger . info ( message ) <EOL> if copy_helper_opts . use_manifest : <EOL> self . manifest . SetResult ( exp_src_url . url_string , <NUM_LIT:0> , '<STR_LIT>' , message ) <EOL> except copy_helper . FileConcurrencySkipError , e : <EOL> self . logger . warn ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> % ( src_url , dst_url ) ) <EOL> except Exception , e : <EOL> if ( copy_helper_opts . no_clobber and <EOL> copy_helper . IsNoClobberServerException ( e ) ) : <EOL> message = '<STR_LIT>' % dst_url <EOL> self . logger . info ( message ) <EOL> if copy_helper_opts . use_manifest : <EOL> self . manifest . SetResult ( <EOL> exp_src_url . url_string , <NUM_LIT:0> , '<STR_LIT>' , message ) <EOL> elif self . continue_on_error : <EOL> message = '<STR_LIT>' % ( src_url , str ( e ) ) <EOL> self . op_failure_count += <NUM_LIT:1> <EOL> self . logger . error ( message ) <EOL> if copy_helper_opts . use_manifest : <EOL> self . manifest . SetResult ( <EOL> exp_src_url . url_string , <NUM_LIT:0> , '<STR_LIT:error>' , <EOL> RemoveCRLFFromString ( message ) ) <EOL> else : <EOL> if copy_helper_opts . use_manifest : <EOL> self . manifest . SetResult ( <EOL> exp_src_url . url_string , <NUM_LIT:0> , '<STR_LIT:error>' , str ( e ) ) <EOL> raise <EOL> else : <EOL> if copy_helper_opts . perform_mv : <EOL> self . logger . info ( '<STR_LIT>' , exp_src_url ) <EOL> if exp_src_url . IsCloudUrl ( ) : <EOL> gsutil_api . DeleteObject ( exp_src_url . bucket_name , <EOL> exp_src_url . object_name , <EOL> generation = exp_src_url . generation , <EOL> provider = exp_src_url . scheme ) <EOL> else : <EOL> os . unlink ( exp_src_url . object_name ) <EOL> with self . stats_lock : <EOL> self . total_elapsed_time += elapsed_time <EOL> self . total_bytes_transferred += bytes_transferred <EOL> def RunCommand ( self ) : <EOL> copy_helper_opts = self . _ParseOpts ( ) <EOL> self . total_elapsed_time = self . total_bytes_transferred = <NUM_LIT:0> <EOL> if self . args [ - <NUM_LIT:1> ] == '<STR_LIT:->' or self . args [ - <NUM_LIT:1> ] == '<STR_LIT>' : <EOL> return CatHelper ( self ) . CatUrlStrings ( self . args [ : - <NUM_LIT:1> ] ) <EOL> if copy_helper_opts . read_args_from_stdin : <EOL> if len ( self . args ) != <NUM_LIT:1> : <EOL> raise CommandException ( '<STR_LIT>' ) <EOL> url_strs = StdinIterator ( ) <EOL> else : <EOL> if len ( self . args ) < <NUM_LIT:2> : <EOL> raise CommandException ( '<STR_LIT>' ) <EOL> url_strs = self . args [ : - <NUM_LIT:1> ] <EOL> ( self . exp_dst_url , self . have_existing_dst_container ) = ( <EOL> copy_helper . ExpandUrlToSingleBlr ( self . args [ - <NUM_LIT:1> ] , self . gsutil_api , <EOL> self . debug , self . project_id ) ) <EOL> name_expansion_iterator = NameExpansionIterator ( <EOL> self . command_name , self . debug , <EOL> self . logger , self . gsutil_api , url_strs , <EOL> self . recursion_requested or copy_helper_opts . perform_mv , <EOL> project_id = self . project_id , all_versions = self . all_versions , <EOL> continue_on_error = self . continue_on_error or self . parallel_operations ) <EOL> self . stats_lock = CreateLock ( ) <EOL> self . op_failure_count = <NUM_LIT:0> <EOL> start_time = time . time ( ) <EOL> shared_attrs = ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> self . Apply ( _CopyFuncWrapper , name_expansion_iterator , <EOL> _CopyExceptionHandler , shared_attrs , <EOL> fail_on_error = ( not self . continue_on_error ) ) <EOL> self . logger . debug ( <EOL> '<STR_LIT>' , self . total_bytes_transferred ) <EOL> end_time = time . time ( ) <EOL> self . total_elapsed_time = end_time - start_time <EOL> try : <EOL> float ( self . total_bytes_transferred ) / float ( self . total_elapsed_time ) <EOL> except ZeroDivisionError : <EOL> self . total_elapsed_time = <NUM_LIT> <EOL> self . total_bytes_per_second = ( float ( self . total_bytes_transferred ) / <EOL> float ( self . total_elapsed_time ) ) <EOL> if self . debug >= DEBUGLEVEL_DUMP_REQUESTS : <EOL> if self . total_bytes_transferred != <NUM_LIT:0> : <EOL> self . logger . info ( <EOL> '<STR_LIT>' , <EOL> self . total_bytes_transferred , self . total_elapsed_time , <EOL> MakeHumanReadable ( self . total_bytes_per_second ) ) <EOL> if self . op_failure_count : <EOL> plural_str = '<STR_LIT:s>' if self . op_failure_count > <NUM_LIT:1> else '<STR_LIT>' <EOL> raise CommandException ( '<STR_LIT>' % ( <EOL> self . op_failure_count , plural_str , plural_str ) ) <EOL> return <NUM_LIT:0> <EOL> def _ParseOpts ( self ) : <EOL> perform_mv = False <EOL> self . exclude_symlinks = False <EOL> no_clobber = False <EOL> self . continue_on_error = False <EOL> daisy_chain = False <EOL> read_args_from_stdin = False <EOL> print_ver = False <EOL> use_manifest = False <EOL> preserve_acl = False <EOL> canned_acl = None <EOL> self . canned = None <EOL> self . all_versions = False <EOL> self . skip_unsupported_objects = False <EOL> gzip_arg_exts = None <EOL> gzip_arg_all = None <EOL> test_callback_file = None <EOL> self . manifest = None <EOL> if self . sub_opts : <EOL> for o , a in self . sub_opts : <EOL> if o == '<STR_LIT>' : <EOL> canned_acl = a <EOL> self . canned = True <EOL> if o == '<STR_LIT>' : <EOL> self . all_versions = True <EOL> if o == '<STR_LIT:-c>' : <EOL> self . continue_on_error = True <EOL> elif o == '<STR_LIT>' : <EOL> daisy_chain = True <EOL> elif o == '<STR_LIT>' : <EOL> self . exclude_symlinks = True <EOL> elif o == '<STR_LIT>' : <EOL> test_callback_file = a <EOL> elif o == '<STR_LIT>' : <EOL> read_args_from_stdin = True <EOL> elif o == '<STR_LIT>' : <EOL> use_manifest = True <EOL> self . manifest = Manifest ( a ) <EOL> elif o == '<STR_LIT>' : <EOL> perform_mv = True <EOL> elif o == '<STR_LIT>' : <EOL> no_clobber = True <EOL> elif o == '<STR_LIT>' : <EOL> preserve_acl = True <EOL> elif o == '<STR_LIT>' or o == '<STR_LIT>' : <EOL> self . recursion_requested = True <EOL> elif o == '<STR_LIT>' : <EOL> self . skip_unsupported_objects = True <EOL> elif o == '<STR_LIT>' : <EOL> print_ver = True <EOL> elif o == '<STR_LIT>' : <EOL> gzip_arg_exts = [ x . strip ( ) for x in a . split ( '<STR_LIT:U+002C>' ) ] <EOL> elif o == '<STR_LIT>' : <EOL> gzip_arg_all = GZIP_ALL_FILES <EOL> if preserve_acl and canned_acl : <EOL> raise CommandException ( <EOL> '<STR_LIT>' ) <EOL> if self . all_versions and self . parallel_operations : <EOL> raise CommandException ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if gzip_arg_exts and gzip_arg_all : <EOL> raise CommandException ( <EOL> '<STR_LIT>' ) <EOL> self . gzip_exts = gzip_arg_exts or gzip_arg_all <EOL> return CreateCopyHelperOpts ( <EOL> perform_mv = perform_mv , <EOL> no_clobber = no_clobber , <EOL> daisy_chain = daisy_chain , <EOL> read_args_from_stdin = read_args_from_stdin , <EOL> print_ver = print_ver , <EOL> use_manifest = use_manifest , <EOL> preserve_acl = preserve_acl , <EOL> canned_acl = canned_acl , <EOL> skip_unsupported_objects = self . skip_unsupported_objects , <EOL> test_callback_file = test_callback_file ) </s>
