_DEFAULT_METADATA_PATH = os.path.expanduser(
    os.path.join('~', '.secureConnect', 'context_aware_metadata.json'))
_CERT_PROVIDER_COMMAND = "cert_provider_command"
_CERT_PROVIDER_COMMAND_PASSPHRASE_OPTION = "--with_passphrase"
class CertProvisionError(Exception):
class ContextConfigSingletonAlreadyExistsError(Exception):
class _ContextConfig(object):
def _is_pem_section_marker(line):
def _split_pem_into_sections(contents, logger):
def _check_path():
def _read_metadata_file(metadata_path):
def _default_command():
  def __init__(self, logger):
  def _provision_client_cert(self, cert_path):
  def _unprovision_client_cert(self):
def create_context_config(logger):
def get_context_config(): class CommandException(StandardError):
  """Exception raised when a problem is encountered running a gsutil command.

  This exception should be used to signal user errors or system failures
  (like timeouts), not bugs (like an incorrect param value). For the
  latter you should raise Exception so we can see where/how it happened
  via gsutil -D (which will include a stack trace for raised Exceptions).
  """

  def __init__(self, reason, informational=False):
    """Instantiate a CommandException.

    Args:
      reason: Text describing the problem.
      informational: Indicates reason should be printed as FYI, not a failure.
    """
    StandardError.__init__(self)
    self.reason = reason
    self.informational = informational

  def __repr__(self):
    return str(self)

  def __str__(self):
    return 'CommandException: %s' % self.reason def SetUpJsonCredentialsAndCache(api, logger, credentials=None):
  """Helper to ensure each GCS API client shares the same credentials."""
  api.credentials = (credentials or _CheckAndGetCredentials(logger) or
                     NoOpCredentials())

  # Notify the user that impersonation credentials are in effect.
  if isinstance(api.credentials, ImpersonationCredentials):
    logger.warn(
        'WARNING: This command is using service account impersonation. All '
        'API calls will be executed as [%s].', _GetImpersonateServiceAccount())

  # Set credential cache so that we don't have to get a new access token for
  # every call we make. All GCS APIs use the same credentials as the JSON API,
  # so we use its version in the key for caching access tokens.
  credential_store_key = (GetCredentialStoreKey(api.credentials,
                                                GetGcsJsonApiVersion()))
  api.credentials.set_store(
      multiprocess_file_storage.MultiprocessFileStorage(
          GetCredentialStoreFilename(), credential_store_key))
  # The cached entry for this credential often contains more context than what
  # we can construct from boto config attributes (e.g. for a user credential,
  # the cached version might also contain a RAPT token and expiry info).
  # Prefer the cached credential if present.
  cached_cred = None
  if not isinstance(api.credentials, NoOpCredentials):
    # A NoOpCredentials object doesn't actually have a store attribute.
    cached_cred = api.credentials.store.get()
  # As of gsutil 4.31, we never use the OAuth2Credentials class for
  # credentials directly; rather, we use subclasses (user credentials were
  # the only ones left using it, but they now use
  # Oauth2WithReauthCredentials).  If we detect that a cached credential is
  # an instance of OAuth2Credentials and not a subclass of it (as might
  # happen when transitioning to version v4.31+), we don't fetch it from the
  # cache. This results in our new-style credential being refreshed and
  # overwriting the old credential cache entry in our credstore.
  if (cached_cred and
      type(cached_cred) != oauth2client.client.OAuth2Credentials):
    api.credentials = cached_cred class Bucket(_messages.Message):
  class AutoclassValue(_messages.Message):
  class BillingValue(_messages.Message):
  class CorsValueListEntry(_messages.Message):
  class CustomPlacementConfigValue(_messages.Message):
  class EncryptionValue(_messages.Message):
  class IamConfigurationValue(_messages.Message):
    class BucketPolicyOnlyValue(_messages.Message):
  class LabelsValue(_messages.Message):
    class AdditionalProperty(_messages.Message):
  class LifecycleValue(_messages.Message):
    class RuleValueListEntry(_messages.Message):
      class ActionValue(_messages.Message):
      class ConditionValue(_messages.Message):
  class LoggingValue(_messages.Message):
  class OwnerValue(_messages.Message):
  class RetentionPolicyValue(_messages.Message):
  class VersioningValue(_messages.Message):
  class WebsiteValue(_messages.Message):
class BucketAccessControl(_messages.Message):
  class ProjectTeamValue(_messages.Message):
class BucketAccessControls(_messages.Message):
class Buckets(_messages.Message):
class Channel(_messages.Message):
  class ParamsValue(_messages.Message):
    class AdditionalProperty(_messages.Message):
class Channels(_messages.Message):
  class ItemsValueListEntry(_messages.Message):
class ComposeRequest(_messages.Message):
  class SourceObjectsValueListEntry(_messages.Message):
    class ObjectPreconditionsValue(_messages.Message):
class Expr(_messages.Message):
class Notification(_messages.Message):
  class CustomAttributesValue(_messages.Message):
    class AdditionalProperty(_messages.Message):
class Notifications(_messages.Message):
class Object(_messages.Message):
  class CustomerEncryptionValue(_messages.Message):
  class MetadataValue(_messages.Message):
    class AdditionalProperty(_messages.Message):
  class OwnerValue(_messages.Message):
class ObjectAccessControl(_messages.Message):
  class ProjectTeamValue(_messages.Message):
class ObjectAccessControls(_messages.Message):
class Objects(_messages.Message):
class Policy(_messages.Message):
  class BindingsValueListEntry(_messages.Message):
class RewriteResponse(_messages.Message):
class ServiceAccount(_messages.Message):
class StandardQueryParameters(_messages.Message):
  class AltValueValuesEnum(_messages.Enum):
class StorageBucketAccessControlsDeleteRequest(_messages.Message):
class StorageBucketAccessControlsDeleteResponse(_messages.Message):
class StorageBucketAccessControlsGetRequest(_messages.Message):
class StorageBucketAccessControlsInsertRequest(_messages.Message):
class StorageBucketAccessControlsListRequest(_messages.Message):
class StorageBucketAccessControlsPatchRequest(_messages.Message):
class StorageBucketAccessControlsUpdateRequest(_messages.Message):
class StorageBucketsDeleteRequest(_messages.Message):
class StorageBucketsDeleteResponse(_messages.Message):
class StorageBucketsGetIamPolicyRequest(_messages.Message):
class StorageBucketsGetRequest(_messages.Message):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageBucketsInsertRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class PredefinedDefaultObjectAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageBucketsListRequest(_messages.Message):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageBucketsListChannelsRequest(_messages.Message):
class StorageBucketsLockRetentionPolicyRequest(_messages.Message):
class StorageBucketsPatchRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class PredefinedDefaultObjectAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageBucketsSetIamPolicyRequest(_messages.Message):
class StorageBucketsTestIamPermissionsRequest(_messages.Message):
class StorageBucketsUpdateRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class PredefinedDefaultObjectAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageChannelsStopResponse(_messages.Message):
class StorageDefaultObjectAccessControlsDeleteRequest(_messages.Message):
class StorageDefaultObjectAccessControlsDeleteResponse(_messages.Message):
class StorageDefaultObjectAccessControlsGetRequest(_messages.Message):
class StorageDefaultObjectAccessControlsInsertRequest(_messages.Message):
class StorageDefaultObjectAccessControlsListRequest(_messages.Message):
class StorageDefaultObjectAccessControlsPatchRequest(_messages.Message):
class StorageDefaultObjectAccessControlsUpdateRequest(_messages.Message):
class StorageNotificationsDeleteRequest(_messages.Message):
class StorageNotificationsDeleteResponse(_messages.Message):
class StorageNotificationsGetRequest(_messages.Message):
class StorageNotificationsInsertRequest(_messages.Message):
class StorageNotificationsListRequest(_messages.Message):
class StorageObjectAccessControlsDeleteRequest(_messages.Message):
class StorageObjectAccessControlsDeleteResponse(_messages.Message):
class StorageObjectAccessControlsGetRequest(_messages.Message):
class StorageObjectAccessControlsInsertRequest(_messages.Message):
class StorageObjectAccessControlsListRequest(_messages.Message):
class StorageObjectAccessControlsPatchRequest(_messages.Message):
class StorageObjectAccessControlsUpdateRequest(_messages.Message):
class StorageObjectsComposeRequest(_messages.Message):
  class DestinationPredefinedAclValueValuesEnum(_messages.Enum):
class StorageObjectsCopyRequest(_messages.Message):
  class DestinationPredefinedAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsDeleteRequest(_messages.Message):
class StorageObjectsDeleteResponse(_messages.Message):
class StorageObjectsGetIamPolicyRequest(_messages.Message):
class StorageObjectsGetRequest(_messages.Message):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsInsertRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsListRequest(_messages.Message):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsPatchRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsRewriteRequest(_messages.Message):
  class DestinationPredefinedAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsSetIamPolicyRequest(_messages.Message):
class StorageObjectsTestIamPermissionsRequest(_messages.Message):
class StorageObjectsUpdateRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsWatchAllRequest(_messages.Message):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageProjectsServiceAccountGetRequest(_messages.Message):
class TestIamPermissionsResponse(_messages.Message):
class StorageProjectsHmacKeysCreateRequest(_messages.Message):
class StorageProjectsHmacKeysDeleteRequest(_messages.Message):
class StorageProjectsHmacKeysGetRequest(_messages.Message):
class StorageProjectsHmacKeysListRequest(_messages.Message):
class StorageProjectsHmacKeysUpdateRequest(_messages.Message):
class HmacKeyMetadata(_messages.Message):
class HmacKey(_messages.Message):
class HmacKeyMetadataList(_messages.Message):
class HmacKeysDeleteResponse(_messages.Message): def GetCertsFile():
  return configured_certs_file def GetGcsJsonApiVersion():
  return config.get('GSUtil', 'json_api_version', DEFAULT_GCS_JSON_API_VERSION) def GetJsonResumableChunkSize():
  chunk_size = config.getint('GSUtil', 'json_resumable_chunk_size',
                             long(1024 * 1024 * 100))
  if chunk_size == 0:
    chunk_size = long(1024 * 256)
  elif chunk_size % long(1024 * 256) != 0:
    chunk_size += (long(1024 * 256) - (chunk_size % (long(1024 * 256))))
  return chunk_size def GetMaxRetryDelay():
  return config.getint('Boto', 'max_retry_delay', 32) def GetNewHttp(http_class=httplib2.Http, **kwargs):
  """Creates and returns a new httplib2.Http instance.

  Args:
    http_class: Optional custom Http class to use.
    **kwargs: Arguments to pass to http_class constructor.

  Returns:
    An initialized httplib2.Http instance.
  """

  ##Get Proxy configuration from boto file, defaults are None, 0 and False
  proxy_host = config.get('Boto', 'proxy', None)  #needed for if clause below
  boto_proxy_config = {
      'proxy_host':
          proxy_host,
      'proxy_type':
          config.get('Boto', 'proxy_type', 'http'),
      'proxy_port':
          config.getint('Boto', 'proxy_port'),
      'proxy_user':
          config.get('Boto', 'proxy_user', None),
      'proxy_pass':
          config.get('Boto', 'proxy_pass', None),
      'proxy_rdns':
          config.get('Boto', 'proxy_rdns', True if proxy_host else None)
  }

  #Use SetProxyInfo to convert boto config to httplib2.proxyinfo object
  proxy_info = SetProxyInfo(boto_proxy_config)

  # Some installers don't package a certs file with httplib2, so use the
  # one included with gsutil.
  kwargs['ca_certs'] = GetCertsFile()
  # Use a non-infinite SSL timeout to avoid hangs during network flakiness.
  kwargs['timeout'] = SSL_TIMEOUT_SEC
  http = http_class(proxy_info=proxy_info, **kwargs)
  http.disable_ssl_certificate_validation = (not config.getbool(
      'Boto', 'https_validate_certificates'))

  global_context_config = context_config.get_context_config()
  if global_context_config and global_context_config.use_client_certificate:
    http.add_certificate(key=global_context_config.client_cert_path,
                         cert=global_context_config.client_cert_path,
                         domain='',
                         password=global_context_config.client_cert_password)

  return http def GetNumRetries():
  return config.getint('Boto', 'num_retries', 23) def JsonResumableChunkSizeDefined():
  chunk_size_defined = config.get('GSUtil', 'json_resumable_chunk_size', None)
  return chunk_size_defined is not None
from contextlib import contextmanager
from six.moves import http_client
from apitools.base.py import encoding
from apitools.base.py import exceptions as apitools_exceptions
from apitools.base.py import http_wrapper as apitools_http_wrapper
from apitools.base.py import transfer as apitools_transfer
from apitools.base.py.util import CalculateWaitForRetry
from boto import config
from gslib import context_config
from gslib.cloud_api import AccessDeniedException
from gslib.cloud_api import ArgumentException
from gslib.cloud_api import BadRequestException
from gslib.cloud_api import CloudApi
from gslib.cloud_api import EncryptionException
from gslib.cloud_api import NotEmptyException
from gslib.cloud_api import NotFoundException
from gslib.cloud_api import PreconditionException
from gslib.cloud_api import Preconditions
from gslib.cloud_api import PublishPermissionDeniedException
from gslib.cloud_api import ResumableDownloadException
from gslib.cloud_api import ResumableUploadAbortException
from gslib.cloud_api import ResumableUploadException
from gslib.cloud_api import ResumableUploadStartOverException
from gslib.cloud_api import ServiceException
from gslib.exception import CommandException
from gslib.gcs_json_credentials import SetUpJsonCredentialsAndCache
from gslib.gcs_json_media import BytesTransferredContainer
from gslib.gcs_json_media import DownloadCallbackConnectionClassFactory
from gslib.gcs_json_media import HttpWithDownloadStream
from gslib.gcs_json_media import HttpWithNoRetries
from gslib.gcs_json_media import UploadCallbackConnectionClassFactory
from gslib.gcs_json_media import WrapDownloadHttpRequest
from gslib.gcs_json_media import WrapUploadHttpRequest
from gslib.impersonation_credentials import ImpersonationCredentials
from gslib.no_op_credentials import NoOpCredentials
from gslib.progress_callback import ProgressCallbackWithTimeout
from gslib.project_id import PopulateProjectId
from gslib.third_party.storage_apitools import storage_v1_client as apitools_client
from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages
from gslib.tracker_file import DeleteTrackerFile
from gslib.tracker_file import GetRewriteTrackerFilePath
from gslib.tracker_file import HashRewriteParameters
from gslib.tracker_file import ReadRewriteTrackerFile
from gslib.tracker_file import WriteRewriteTrackerFile
from gslib.utils.boto_util import GetCertsFile
from gslib.utils.boto_util import GetGcsJsonApiVersion
from gslib.utils.boto_util import GetJsonResumableChunkSize
from gslib.utils.boto_util import GetMaxRetryDelay
from gslib.utils.boto_util import GetNewHttp
from gslib.utils.boto_util import GetNumRetries
from gslib.utils.boto_util import JsonResumableChunkSizeDefined
from gslib.utils.cloud_api_helper import ListToGetFields
from gslib.utils.cloud_api_helper import ValidateDstObjectMetadata
from gslib.utils.constants import IAM_POLICY_VERSION
from gslib.utils.constants import NUM_OBJECTS_PER_LIST_PAGE
from gslib.utils.constants import REQUEST_REASON_ENV_VAR
from gslib.utils.constants import REQUEST_REASON_HEADER_KEY
from gslib.utils.constants import UTF8
from gslib.utils.encryption_helper import Base64Sha256FromBase64EncryptionKey
from gslib.utils.encryption_helper import CryptoKeyType
from gslib.utils.encryption_helper import CryptoKeyWrapperFromKey
from gslib.utils.encryption_helper import FindMatchingCSEKInBotoConfig
from gslib.utils.metadata_util import AddAcceptEncodingGzipIfNeeded
from gslib.utils.retry_util import LogAndHandleRetries
from gslib.utils.signurl_helper import CreatePayload, GetFinalUrl
from gslib.utils.text_util import GetPrintableExceptionString
from gslib.utils.translation_helper import CreateBucketNotFoundException
from gslib.utils.translation_helper import CreateNotFoundExceptionForObjectWrite
from gslib.utils.translation_helper import CreateObjectNotFoundException
from gslib.utils.translation_helper import DEFAULT_CONTENT_TYPE
from gslib.utils.translation_helper import PRIVATE_DEFAULT_OBJ_ACL
from gslib.utils.translation_helper import REMOVE_CORS_CONFIG
from oauth2client.service_account import ServiceAccountCredentials
import functools
import json
import logging
import os
import socket
import ssl
import time
import traceback
import six
import httplib2
import oauth2client
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""JSON gsutil Cloud API implementation for Google Cloud Storage."""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals





if six.PY3:
  long = int

# pylint: disable=invalid-name
Notification = apitools_messages.Notification
NotificationCustomAttributesValue = Notification.CustomAttributesValue
NotificationAdditionalProperty = (
    NotificationCustomAttributesValue.AdditionalProperty)

# Implementation supports only 'gs' URLs, so provider is unused.
# pylint: disable=unused-argument

NUM_BUCKETS_PER_LIST_PAGE = 1000

TRANSLATABLE_APITOOLS_EXCEPTIONS = (apitools_exceptions.HttpError,
                                    apitools_exceptions.StreamExhausted,
                                    apitools_exceptions.TransferError,
                                    apitools_exceptions.TransferInvalidError)

# TODO: Distribute these exceptions better through apitools and here.
# Right now, apitools is configured not to handle any exceptions on
# uploads/downloads.
# oauth2_client tries to JSON-decode the response, which can result
# in a ValueError if the response was invalid. Until that is fixed in
# oauth2_client, need to handle it here.
HTTP_TRANSFER_EXCEPTIONS = (
    apitools_exceptions.TransferRetryError,
    apitools_exceptions.BadStatusCodeError,
    # TODO: Honor retry-after headers.
    apitools_exceptions.RetryAfterError,
    apitools_exceptions.RequestError,
    http_client.BadStatusLine,
    http_client.IncompleteRead,
    http_client.ResponseNotReady,
    httplib2.ServerNotFoundError,
    oauth2client.client.HttpAccessTokenRefreshError,
    socket.error,
    socket.gaierror,
    socket.timeout,
    ssl.SSLError,
    ValueError,
)

# Fields requiring projection=full across all API calls.
_ACL_FIELDS_SET = set([
    'acl',
    'defaultObjectAcl',
    'items/acl',
    'items/defaultObjectAcl',
    'items/owner',
    'owner',
])

# Fields that may be encrypted.
_ENCRYPTED_HASHES_SET = set(['crc32c', 'md5Hash'])

# During listing, if we attempt to decrypt an object but it has been removed,
# we skip including the object in the listing.
_SKIP_LISTING_OBJECT = 'skip'

_INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = (
    'Insufficient OAuth2 scope to perform this operation.')

DEFAULT_HOST = 'storage.googleapis.com'
MTLS_HOST = 'storage.mtls.googleapis.com'


class GcsJsonApi(CloudApi):
  """Google Cloud Storage JSON implementation of gsutil Cloud API."""

  def __init__(self,
               bucket_storage_uri_class,
               logger,
               status_queue,
               provider=None,
               credentials=None,
               debug=0,
               trace_token=None,
               perf_trace_token=None,
               user_project=None):
    """Performs necessary setup for interacting with Google Cloud Storage.

    Args:
      bucket_storage_uri_class: Unused.
      logger: logging.logger for outputting log messages.
      status_queue: Queue for relaying status to UI.
      provider: Unused.  This implementation supports only Google Cloud Storage.
      credentials: Credentials to be used for interacting with Google Cloud
                   Storage.
      debug: Debug level for the API implementation (0..3).
      trace_token: Trace token to pass to the API implementation.
      perf_trace_token: Performance trace token to use when making API calls.
      user_project: Project to be billed for this request.
    """
    # TODO: Plumb host_header for perfdiag / test_perfdiag.
    # TODO: Add jitter to apitools' http_wrapper retry mechanism.
    super(GcsJsonApi, self).__init__(bucket_storage_uri_class,
                                     logger,
                                     status_queue,
                                     provider='gs',
                                     debug=debug,
                                     trace_token=trace_token,
                                     perf_trace_token=perf_trace_token,
                                     user_project=user_project)


    self.certs_file = GetCertsFile()