MAX_COMPOSE_ARITY = 32 DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE = '50M' DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD = '0' DEFAULT_SLICED_OBJECT_DOWNLOAD_COMPONENT_SIZE = '200M' DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS = 4 DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD = '150M' DEFAULT_GZIP_COMPRESSION_LEVEL = 9 class CommandException(StandardError):
  """Exception raised when a problem is encountered running a gsutil command.

  This exception should be used to signal user errors or system failures
  (like timeouts), not bugs (like an incorrect param value). For the
  latter you should raise Exception so we can see where/how it happened
  via gsutil -D (which will include a stack trace for raised Exceptions).
  """

  def __init__(self, reason, informational=False):
    """Instantiate a CommandException.

    Args:
      reason: Text describing the problem.
      informational: Indicates reason should be printed as FYI, not a failure.
    """
    StandardError.__init__(self)
    self.reason = reason
    self.informational = informational

  def __repr__(self):
    return str(self)

  def __str__(self):
    return 'CommandException: %s' % self.reason class HashMismatchException(Exception):
  """Exception raised when data integrity validation fails."""
  pass class InvalidUrlError(Exception):
  """Exception raised when URL is invalid."""

  def __init__(self, message):
    Exception.__init__(self, message)
    self.message = message

  def __repr__(self):
    return str(self)

  def __str__(self):
    return 'InvalidUrlError: %s' % self.message def ContainsWildcard(url_string):
  """Checks whether url_string contains a wildcard.

  Args:
    url_string: URL string to check.

  Returns:
    bool indicator.
  """
  return bool(WILDCARD_REGEX.search(url_string)) def GenerationFromUrlAndString(url, generation):
  """Decodes a generation from a StorageURL and a generation string.

  This is used to represent gs and s3 versioning.

  Args:
    url: StorageUrl representing the object.
    generation: Long or string representing the object's generation or
                version.

  Returns:
    Valid generation string for use in URLs.
  """
  if url.scheme == 's3' and generation:
    return text_util.DecodeLongAsString(generation)
  return generation def IsCloudSubdirPlaceholder(url, blr=None):
  """Determines if a StorageUrl is a cloud subdir placeholder.

  This function is needed because GUI tools (like the GCS cloud console) allow
  users to create empty "folders" by creating a placeholder object; and parts
  of gsutil need to treat those placeholder objects specially. For example,
  gsutil rsync needs to avoid downloading those objects because they can cause
  conflicts (see comments in rsync command for details).

  We currently detect two cases:
    - Cloud objects whose name ends with '_$folder$'
    - Cloud objects whose name ends with '/'

  Args:
    url: (gslib.storage_url.StorageUrl) The URL to be checked.
    blr: (gslib.BucketListingRef or None) The blr to check, or None if not
        available. If `blr` is None, size won't be checked.

  Returns:
    (bool) True if the URL is a cloud subdir placeholder, otherwise False.
  """
  if not url.IsCloudUrl():
    return False
  url_str = url.url_string
  if url_str.endswith('_$folder$'):
    return True
  if blr and blr.IsObject():
    size = blr.root_object.size
  else:
    size = 0
  return size == 0 and url_str.endswith('/') def StorageUrlFromString(url_str):
  """Static factory function for creating a StorageUrl from a string."""

  scheme = GetSchemeFromUrlString(url_str)

  if not IsKnownUrlScheme(scheme):
    raise InvalidUrlError('Unrecognized scheme "%s"' % scheme)
  if scheme == 'file':
    path = _GetPathFromUrlString(url_str)
    is_stream = (path == '-')
    is_fifo = False
    try:
      is_fifo = stat.S_ISFIFO(os.stat(path).st_mode)
    except OSError:
      pass
    return _FileUrl(url_str, is_stream=is_stream, is_fifo=is_fifo)
  return _CloudUrl(url_str) class Bucket(_messages.Message):
  class AutoclassValue(_messages.Message):
  class BillingValue(_messages.Message):
  class CorsValueListEntry(_messages.Message):
  class CustomPlacementConfigValue(_messages.Message):
  class EncryptionValue(_messages.Message):
  class IamConfigurationValue(_messages.Message):
    class BucketPolicyOnlyValue(_messages.Message):
  class LabelsValue(_messages.Message):
    class AdditionalProperty(_messages.Message):
  class LifecycleValue(_messages.Message):
    class RuleValueListEntry(_messages.Message):
      class ActionValue(_messages.Message):
      class ConditionValue(_messages.Message):
  class LoggingValue(_messages.Message):
  class OwnerValue(_messages.Message):
  class RetentionPolicyValue(_messages.Message):
  class VersioningValue(_messages.Message):
  class WebsiteValue(_messages.Message):
class BucketAccessControl(_messages.Message):
  class ProjectTeamValue(_messages.Message):
class BucketAccessControls(_messages.Message):
class Buckets(_messages.Message):
class Channel(_messages.Message):
  class ParamsValue(_messages.Message):
    class AdditionalProperty(_messages.Message):
class Channels(_messages.Message):
  class ItemsValueListEntry(_messages.Message):
class ComposeRequest(_messages.Message):
  class SourceObjectsValueListEntry(_messages.Message):
    class ObjectPreconditionsValue(_messages.Message):
class Expr(_messages.Message):
class Notification(_messages.Message):
  class CustomAttributesValue(_messages.Message):
    class AdditionalProperty(_messages.Message):
class Notifications(_messages.Message):
class Object(_messages.Message):
  class CustomerEncryptionValue(_messages.Message):
  class MetadataValue(_messages.Message):
    class AdditionalProperty(_messages.Message):
  class OwnerValue(_messages.Message):
class ObjectAccessControl(_messages.Message):
  class ProjectTeamValue(_messages.Message):
class ObjectAccessControls(_messages.Message):
class Objects(_messages.Message):
class Policy(_messages.Message):
  class BindingsValueListEntry(_messages.Message):
class RewriteResponse(_messages.Message):
class ServiceAccount(_messages.Message):
class StandardQueryParameters(_messages.Message):
  class AltValueValuesEnum(_messages.Enum):
class StorageBucketAccessControlsDeleteRequest(_messages.Message):
class StorageBucketAccessControlsDeleteResponse(_messages.Message):
class StorageBucketAccessControlsGetRequest(_messages.Message):
class StorageBucketAccessControlsInsertRequest(_messages.Message):
class StorageBucketAccessControlsListRequest(_messages.Message):
class StorageBucketAccessControlsPatchRequest(_messages.Message):
class StorageBucketAccessControlsUpdateRequest(_messages.Message):
class StorageBucketsDeleteRequest(_messages.Message):
class StorageBucketsDeleteResponse(_messages.Message):
class StorageBucketsGetIamPolicyRequest(_messages.Message):
class StorageBucketsGetRequest(_messages.Message):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageBucketsInsertRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class PredefinedDefaultObjectAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageBucketsListRequest(_messages.Message):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageBucketsListChannelsRequest(_messages.Message):
class StorageBucketsLockRetentionPolicyRequest(_messages.Message):
class StorageBucketsPatchRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class PredefinedDefaultObjectAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageBucketsSetIamPolicyRequest(_messages.Message):
class StorageBucketsTestIamPermissionsRequest(_messages.Message):
class StorageBucketsUpdateRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class PredefinedDefaultObjectAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageChannelsStopResponse(_messages.Message):
class StorageDefaultObjectAccessControlsDeleteRequest(_messages.Message):
class StorageDefaultObjectAccessControlsDeleteResponse(_messages.Message):
class StorageDefaultObjectAccessControlsGetRequest(_messages.Message):
class StorageDefaultObjectAccessControlsInsertRequest(_messages.Message):
class StorageDefaultObjectAccessControlsListRequest(_messages.Message):
class StorageDefaultObjectAccessControlsPatchRequest(_messages.Message):
class StorageDefaultObjectAccessControlsUpdateRequest(_messages.Message):
class StorageNotificationsDeleteRequest(_messages.Message):
class StorageNotificationsDeleteResponse(_messages.Message):
class StorageNotificationsGetRequest(_messages.Message):
class StorageNotificationsInsertRequest(_messages.Message):
class StorageNotificationsListRequest(_messages.Message):
class StorageObjectAccessControlsDeleteRequest(_messages.Message):
class StorageObjectAccessControlsDeleteResponse(_messages.Message):
class StorageObjectAccessControlsGetRequest(_messages.Message):
class StorageObjectAccessControlsInsertRequest(_messages.Message):
class StorageObjectAccessControlsListRequest(_messages.Message):
class StorageObjectAccessControlsPatchRequest(_messages.Message):
class StorageObjectAccessControlsUpdateRequest(_messages.Message):
class StorageObjectsComposeRequest(_messages.Message):
  class DestinationPredefinedAclValueValuesEnum(_messages.Enum):
class StorageObjectsCopyRequest(_messages.Message):
  class DestinationPredefinedAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsDeleteRequest(_messages.Message):
class StorageObjectsDeleteResponse(_messages.Message):
class StorageObjectsGetIamPolicyRequest(_messages.Message):
class StorageObjectsGetRequest(_messages.Message):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsInsertRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsListRequest(_messages.Message):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsPatchRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsRewriteRequest(_messages.Message):
  class DestinationPredefinedAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsSetIamPolicyRequest(_messages.Message):
class StorageObjectsTestIamPermissionsRequest(_messages.Message):
class StorageObjectsUpdateRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsWatchAllRequest(_messages.Message):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageProjectsServiceAccountGetRequest(_messages.Message):
class TestIamPermissionsResponse(_messages.Message):
class StorageProjectsHmacKeysCreateRequest(_messages.Message):
class StorageProjectsHmacKeysDeleteRequest(_messages.Message):
class StorageProjectsHmacKeysGetRequest(_messages.Message):
class StorageProjectsHmacKeysListRequest(_messages.Message):
class StorageProjectsHmacKeysUpdateRequest(_messages.Message):
class HmacKeyMetadata(_messages.Message):
class HmacKey(_messages.Message):
class HmacKeyMetadataList(_messages.Message):
class HmacKeysDeleteResponse(_messages.Message): def GetJsonResumableChunkSize():
  chunk_size = config.getint('GSUtil', 'json_resumable_chunk_size',
                             long(1024 * 1024 * 100))
  if chunk_size == 0:
    chunk_size = long(1024 * 256)
  elif chunk_size % long(1024 * 256) != 0:
    chunk_size += (long(1024 * 256) - (chunk_size % (long(1024 * 256))))
  return chunk_size def GetMaxRetryDelay():
  return config.getint('Boto', 'max_retry_delay', 32) def GetNumRetries():
  return config.getint('Boto', 'num_retries', 23) def ResumableThreshold():
  return config.getint('GSUtil', 'resumable_threshold', 8 * ONE_MIB) def UsingCrcmodExtension():
  boto_opt = boto.config.get('GSUtil', 'test_assume_fast_crcmod', None)
  if boto_opt is not None:
    return boto_opt
  # Python 3 makes this attribute tough to access due to the way the top-level
  # crcmod package imports and (identically) names its crcmod module. The only
  # way to get it is "from crcmod.crcmod import _usingExtension". This is the
  # alternative form of that statement, but doesn't pollute this module's
  # namespace with a "_usingExtension" attribute. This also works in both Python
  # 2.7 and 3.5+.
  nested_crcmod = __import__(
      'crcmod.crcmod',
      globals(),
      locals(),
      ['_usingExtension'],
      0,
  )
  return getattr(nested_crcmod, '_usingExtension', False) ATIME_ATTR = 'goog-reserved-file-atime' def ConvertDatetimeToPOSIX(dt):
  """Converts a datetime object to UTC and formats as POSIX.

  Sanitize the timestamp returned in dt, and put it in UTC format. For more
  information see the UTC class.

  Args:
    dt: A Python datetime object.

  Returns:
    A POSIX timestamp according to UTC.
  """
  return long(timegm(dt.replace(tzinfo=UTC()).timetuple())) GID_ATTR = 'goog-reserved-posix-gid' MODE_ATTR = 'goog-reserved-posix-mode' MTIME_ATTR = 'goog-reserved-file-mtime' def ParseAndSetPOSIXAttributes(path,
                               obj_metadata,
                               is_rsync=False,
                               preserve_posix=False):
  """Parses POSIX attributes from obj_metadata and sets them.

  Attributes will only be set if they exist in custom metadata. This function
  should only be called after ValidateFilePermissionAccess has been called for
  the specific file/object so as not to orphan files.

  Args:
    path: The local filesystem path for the file. Valid metadata attributes will
          be set for the file located at path, some attributes will only be set
          if preserve_posix is set to True.
    obj_metadata: The metadata for the file/object.
    is_rsync: Whether or not the caller is the rsync command. Used to determine
              if timeCreated should be used.
    preserve_posix: Whether or not all POSIX attributes should be set.
  """
  if obj_metadata is None:
    # This exception is meant for debugging purposes as it should never be
    # thrown unless there are unexpected code changes.
    raise CommandException('obj_metadata cannot be None for %s' % path)
  try:
    found_at, atime = GetValueFromObjectCustomMetadata(obj_metadata,
                                                       ATIME_ATTR,
                                                       default_value=NA_TIME)
    found_mt, mtime = GetValueFromObjectCustomMetadata(obj_metadata,
                                                       MTIME_ATTR,
                                                       default_value=NA_TIME)
    found_uid, uid = GetValueFromObjectCustomMetadata(obj_metadata,
                                                      UID_ATTR,
                                                      default_value=NA_ID)
    found_gid, gid = GetValueFromObjectCustomMetadata(obj_metadata,
                                                      GID_ATTR,
                                                      default_value=NA_ID)
    found_mode, mode = GetValueFromObjectCustomMetadata(obj_metadata,
                                                        MODE_ATTR,
                                                        default_value=NA_MODE)
    if not found_mode:
      mode = int(GetDefaultMode())
    if found_mt:
      mtime = long(mtime)
      if not preserve_posix:
        atime_tmp = os.stat(path).st_atime
        os.utime(path, (atime_tmp, mtime))
        return
    elif is_rsync:
      mtime = ConvertDatetimeToPOSIX(obj_metadata.timeCreated)
      os.utime(path, (mtime, mtime))
    if not preserve_posix:
      return
    if found_at:
      atime = long(atime)
    if atime > NA_TIME and mtime > NA_TIME:
      # Set both atime and mtime.
      os.utime(path, (atime, mtime))
    elif atime > NA_TIME and mtime <= NA_TIME:
      # atime is valid but mtime isn't.
      mtime_tmp = os.stat(path).st_mtime
      os.utime(path, (atime, mtime_tmp))
    elif atime <= NA_TIME and mtime > NA_TIME:
      # mtime is valid but atime isn't
      atime_tmp = os.stat(path).st_atime
      os.utime(path, (atime_tmp, mtime))
    if IS_WINDOWS:
      # Windows doesn't use POSIX uid/gid/mode unlike other systems. So there's
      # no point continuing.
      return
    # Only root can change ownership.
    if found_uid and os.geteuid() == 0:
      uid = int(uid)
    else:
      uid = NA_ID
    if found_gid:
      gid = int(gid)
    if uid > NA_ID and gid > NA_ID:
      # Set both uid and gid.
      os.chown(path, uid, gid)
    elif uid > NA_ID and gid <= NA_ID:
      # uid is valid but gid isn't.
      os.chown(path, uid, -1)
    elif uid <= NA_ID and gid > NA_ID:
      # gid is valid but uid isn't.
      os.chown(path, -1, gid)
    if found_mode:
      mode = int(str(mode), 8)
      os.chmod(path, mode)
  except ValueError:
    raise CommandException('Check POSIX attribute values for %s' %
                           obj_metadata.name) UID_ATTR = 'goog-reserved-posix-uid'
import base64
import csv
import datetime
import errno
import gzip
import json
import logging
import mimetypes
import os
import pickle
import random
import re
import shutil
import six
import stat
import subprocess
import tempfile
import textwrap
import time
import traceback
import six
import crcmod
import gslib
  import msvcrt
from collections import namedtuple
from operator import attrgetter
from six.moves import xrange
from six.moves import range
from apitools.base.protorpclite import protojson
from boto import config
from gslib.cloud_api import AccessDeniedException
from gslib.cloud_api import ArgumentException
from gslib.cloud_api import CloudApi
from gslib.cloud_api import EncryptionException
from gslib.cloud_api import NotFoundException
from gslib.cloud_api import PreconditionException
from gslib.cloud_api import Preconditions
from gslib.cloud_api import ResumableDownloadException
from gslib.cloud_api import ResumableUploadAbortException
from gslib.cloud_api import ResumableUploadException
from gslib.cloud_api import ResumableUploadStartOverException
from gslib.cloud_api import ServiceException
from gslib.commands.compose import MAX_COMPOSE_ARITY
from gslib.commands.config import DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE
from gslib.commands.config import DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD
from gslib.commands.config import DEFAULT_SLICED_OBJECT_DOWNLOAD_COMPONENT_SIZE
from gslib.commands.config import DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS
from gslib.commands.config import DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD
from gslib.commands.config import DEFAULT_GZIP_COMPRESSION_LEVEL
from gslib.cs_api_map import ApiSelector
from gslib.daisy_chain_wrapper import DaisyChainWrapper
from gslib.exception import CommandException
from gslib.exception import HashMismatchException
from gslib.exception import InvalidUrlError
from gslib.file_part import FilePart
from gslib.parallel_tracker_file import GenerateComponentObjectPrefix
from gslib.parallel_tracker_file import ReadParallelUploadTrackerFile
from gslib.parallel_tracker_file import ValidateParallelCompositeTrackerData
from gslib.parallel_tracker_file import WriteComponentToParallelUploadTrackerFile
from gslib.parallel_tracker_file import WriteParallelUploadTrackerFile
from gslib.progress_callback import FileProgressCallbackHandler
from gslib.progress_callback import ProgressCallbackWithTimeout
from gslib.resumable_streaming_upload import ResumableStreamingJsonUploadWrapper
from gslib.storage_url import ContainsWildcard
from gslib.storage_url import GenerationFromUrlAndString
from gslib.storage_url import IsCloudSubdirPlaceholder
from gslib.storage_url import StorageUrlFromString
from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages
from gslib.thread_message import FileMessage
from gslib.thread_message import RetryableErrorMessage
from gslib.tracker_file import DeleteDownloadTrackerFiles
from gslib.tracker_file import DeleteTrackerFile
from gslib.tracker_file import ENCRYPTION_UPLOAD_TRACKER_ENTRY
from gslib.tracker_file import GetDownloadStartByte
from gslib.tracker_file import GetTrackerFilePath
from gslib.tracker_file import GetUploadTrackerData
from gslib.tracker_file import RaiseUnwritableTrackerFileException
from gslib.tracker_file import ReadOrCreateDownloadTrackerFile
from gslib.tracker_file import SERIALIZATION_UPLOAD_TRACKER_ENTRY
from gslib.tracker_file import TrackerFileType
from gslib.tracker_file import WriteDownloadComponentTrackerFile
from gslib.tracker_file import WriteJsonDataToTrackerFile
from gslib.utils import parallelism_framework_util
from gslib.utils import stet_util
from gslib.utils import temporary_file_util
from gslib.utils import text_util
from gslib.utils.boto_util import GetJsonResumableChunkSize
from gslib.utils.boto_util import GetMaxRetryDelay
from gslib.utils.boto_util import GetNumRetries
from gslib.utils.boto_util import ResumableThreshold
from gslib.utils.boto_util import UsingCrcmodExtension
from gslib.utils.cloud_api_helper import GetCloudApiInstance
from gslib.utils.cloud_api_helper import GetDownloadSerializationData
from gslib.utils.constants import DEFAULT_FILE_BUFFER_SIZE
from gslib.utils.constants import MIN_SIZE_COMPUTE_LOGGING
from gslib.utils.constants import UTF8
from gslib.utils.encryption_helper import CryptoKeyType
from gslib.utils.encryption_helper import CryptoKeyWrapperFromKey
from gslib.utils.encryption_helper import FindMatchingCSEKInBotoConfig
from gslib.utils.encryption_helper import GetEncryptionKeyWrapper
from gslib.utils.hashing_helper import Base64EncodeHash
from gslib.utils.hashing_helper import CalculateB64EncodedMd5FromContents
from gslib.utils.hashing_helper import CalculateHashesFromContents
from gslib.utils.hashing_helper import CHECK_HASH_IF_FAST_ELSE_FAIL
from gslib.utils.hashing_helper import CHECK_HASH_NEVER
from gslib.utils.hashing_helper import ConcatCrc32c
from gslib.utils.hashing_helper import GetDownloadHashAlgs
from gslib.utils.hashing_helper import GetMd5
from gslib.utils.hashing_helper import GetUploadHashAlgs
from gslib.utils.hashing_helper import HashingFileUploadWrapper
from gslib.utils.metadata_util import ObjectIsGzipEncoded
from gslib.utils.parallelism_framework_util import AtomicDict
from gslib.utils.parallelism_framework_util import CheckMultiprocessingAvailableAndInit
from gslib.utils.parallelism_framework_util import PutToQueueWithTimeout
from gslib.utils.posix_util import ATIME_ATTR
from gslib.utils.posix_util import ConvertDatetimeToPOSIX
from gslib.utils.posix_util import GID_ATTR
from gslib.utils.posix_util import MODE_ATTR
from gslib.utils.posix_util import MTIME_ATTR
from gslib.utils.posix_util import ParseAndSetPOSIXAttributes
from gslib.utils.posix_util import UID_ATTR
from gslib.utils.system_util import CheckFreeSpace
from gslib.utils.system_util import GetFileSize
from gslib.utils.system_util import GetStreamFromFileUrl
from gslib.utils.system_util import IS_WINDOWS
from gslib.utils.translation_helper import AddS3MarkerAclToObjectMetadata
from gslib.utils.translation_helper import CopyObjectMetadata
from gslib.utils.translation_helper import DEFAULT_CONTENT_TYPE
from gslib.utils.translation_helper import ObjectMetadataFromHeaders
from gslib.utils.translation_helper import PreconditionsFromHeaders
from gslib.utils.translation_helper import S3MarkerAclFromObjectMetadata
from gslib.utils.unit_util import DivideAndCeil
from gslib.utils.unit_util import HumanReadableToBytes
from gslib.utils.unit_util import MakeHumanReadable
from gslib.utils.unit_util import SECONDS_PER_DAY
from gslib.utils.unit_util import TEN_MIB
from gslib.wildcard_iterator import CreateWildcardIterator
    local_b64_digest = six.ensure_binary(local_hashes[alg])
    cloud_b64_digest = cloud_hashes[alg]
    logger.debug('Comparing local vs cloud %s-checksum for %s. (%s/%s)', alg,
                 file_name, local_b64_digest, cloud_b64_digest)
    if local_b64_digest != cloud_b64_digest:
      raise HashMismatchException(
          '%s signature computed for local file (%s) doesn\'t match '
          'cloud-supplied digest (%s). %s (%s) will be deleted.' %
          (alg, local_b64_digest, cloud_b64_digest, 'Cloud object'
           if is_upload else 'Local file', obj_url if is_upload else file_name))
    checked_one = True
  if not checked_one:
    if is_upload:
      logger.warn(
          'WARNING: Found no hashes to validate object uploaded to %s. '
          'Integrity cannot be assured without hashes.', obj_url)
    else:
      # One known way this can currently happen is when downloading objects larger
      # than 5 GB from S3 (for which the etag is not an MD5).
      logger.warn(
          'WARNING: Found no hashes to validate object downloaded to %s. '
          'Integrity cannot be assured without hashes.', file_name)


def IsNoClobberServerException(e):
  """Checks to see if the server attempted to clobber a file.

  In this case we specified via a precondition that we didn't want the file
  clobbered.

  Args:
    e: The Exception that was generated by a failed copy operation

  Returns:
    bool indicator - True indicates that the server did attempt to clobber
        an existing file.
  """
  return ((isinstance(e, PreconditionException)) or
          (isinstance(e, ResumableUploadException) and '412' in e.message))


def CheckForDirFileConflict(exp_src_url, dst_url):
  """Checks whether copying exp_src_url into dst_url is not possible.

     This happens if a directory exists in local file system where a file
     needs to go or vice versa. In that case we print an error message and
     exits. Example: if the file "./x" exists and you try to do:
       gsutil cp gs://mybucket/x/y .
     the request can't succeed because it requires a directory where
     the file x exists.

     Note that we don't enforce any corresponding restrictions for buckets,
     because the flat namespace semantics for buckets doesn't prohibit such
     cases the way hierarchical file systems do. For example, if a bucket
     contains an object called gs://bucket/dir and then you run the command:
       gsutil cp file1 file2 gs://bucket/dir
     you'll end up with objects gs://bucket/dir, gs://bucket/dir/file1, and
     gs://bucket/dir/file2.

  Args:
    exp_src_url: Expanded source StorageUrl.
    dst_url: Destination StorageUrl.

  Raises:
    CommandException: if errors encountered.
  """
  if dst_url.IsCloudUrl():
    # The problem can only happen for file destination URLs.
    return
  dst_path = dst_url.object_name
  final_dir = os.path.dirname(dst_path)
  if os.path.isfile(final_dir):
    raise CommandException('Cannot retrieve %s because a file exists '
                           'where a directory needs to be created (%s).' %
                           (exp_src_url.url_string, final_dir))
  if os.path.isdir(dst_path):
    raise CommandException('Cannot retrieve %s because a directory exists '
                           '(%s) where the file needs to be created.' %
                           (exp_src_url.url_string, dst_path))


def _PartitionFile(fp,
                   file_size,
                   src_url,
                   content_type,
                   canned_acl,
                   dst_bucket_url,
                   random_prefix,
                   tracker_file,
                   tracker_file_lock,
                   encryption_key_sha256=None,
                   gzip_encoded=False):
  """Partitions a file into FilePart objects to be uploaded and later composed.

  These objects, when composed, will match the original file. This entails
  splitting the file into parts, naming and forming a destination URL for each
  part, and also providing the PerformParallelUploadFileToObjectArgs
  corresponding to each part.

  Args:
    fp: The file object to be partitioned.
    file_size: The size of fp, in bytes.
    src_url: Source FileUrl from the original command.
    content_type: content type for the component and final objects.
    canned_acl: The user-provided canned_acl, if applicable.
    dst_bucket_url: CloudUrl for the destination bucket
    random_prefix: The randomly-generated prefix used to prevent collisions
                   among the temporary component names.
    tracker_file: The path to the parallel composite upload tracker file.
    tracker_file_lock: The lock protecting access to the tracker file.
    encryption_key_sha256: Encryption key SHA256 for use in this upload, if any.
    gzip_encoded: Whether to use gzip transport encoding for the upload.

  Returns:
    dst_args: The destination URIs for the temporary component objects.
  """
  parallel_composite_upload_component_size = HumanReadableToBytes(
      config.get('GSUtil', 'parallel_composite_upload_component_size',
                 DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE))
  (num_components, component_size) = _GetPartitionInfo(

      file_size, MAX_COMPOSE_ARITY, parallel_composite_upload_component_size)