class CommandException(StandardError):
  """Exception raised when a problem is encountered running a gsutil command.

  This exception should be used to signal user errors or system failures
  (like timeouts), not bugs (like an incorrect param value). For the
  latter you should raise Exception so we can see where/how it happened
  via gsutil -D (which will include a stack trace for raised Exceptions).
  """

  def __init__(self, reason, informational=False):
    """Instantiate a CommandException.

    Args:
      reason: Text describing the problem.
      informational: Indicates reason should be printed as FYI, not a failure.
    """
    StandardError.__init__(self)
    self.reason = reason
    self.informational = informational

  def __repr__(self):
    return str(self)

  def __str__(self):
    return 'CommandException: %s' % self.reason def SkipForS3(reason):
  if RUN_S3_TESTS:
    return unittest.skip(reason)
  else:
    return lambda func: func def SkipForXML(reason):
  """Skips the test if running S3 tests, or if prefer_api isn't set to json."""
  if not USING_JSON_API or RUN_S3_TESTS:
    return unittest.skip(reason)
  else:
    return lambda func: func class Bucket(_messages.Message):
  class AutoclassValue(_messages.Message):
  class BillingValue(_messages.Message):
  class CorsValueListEntry(_messages.Message):
  class CustomPlacementConfigValue(_messages.Message):
  class EncryptionValue(_messages.Message):
  class IamConfigurationValue(_messages.Message):
    class BucketPolicyOnlyValue(_messages.Message):
  class LabelsValue(_messages.Message):
    class AdditionalProperty(_messages.Message):
  class LifecycleValue(_messages.Message):
    class RuleValueListEntry(_messages.Message):
      class ActionValue(_messages.Message):
      class ConditionValue(_messages.Message):
  class LoggingValue(_messages.Message):
  class OwnerValue(_messages.Message):
  class RetentionPolicyValue(_messages.Message):
  class VersioningValue(_messages.Message):
  class WebsiteValue(_messages.Message):
class BucketAccessControl(_messages.Message):
  class ProjectTeamValue(_messages.Message):
class BucketAccessControls(_messages.Message):
class Buckets(_messages.Message):
class Channel(_messages.Message):
  class ParamsValue(_messages.Message):
    class AdditionalProperty(_messages.Message):
class Channels(_messages.Message):
  class ItemsValueListEntry(_messages.Message):
class ComposeRequest(_messages.Message):
  class SourceObjectsValueListEntry(_messages.Message):
    class ObjectPreconditionsValue(_messages.Message):
class Expr(_messages.Message):
class Notification(_messages.Message):
  class CustomAttributesValue(_messages.Message):
    class AdditionalProperty(_messages.Message):
class Notifications(_messages.Message):
class Object(_messages.Message):
  class CustomerEncryptionValue(_messages.Message):
  class MetadataValue(_messages.Message):
    class AdditionalProperty(_messages.Message):
  class OwnerValue(_messages.Message):
class ObjectAccessControl(_messages.Message):
  class ProjectTeamValue(_messages.Message):
class ObjectAccessControls(_messages.Message):
class Objects(_messages.Message):
class Policy(_messages.Message):
  class BindingsValueListEntry(_messages.Message):
class RewriteResponse(_messages.Message):
class ServiceAccount(_messages.Message):
class StandardQueryParameters(_messages.Message):
  class AltValueValuesEnum(_messages.Enum):
class StorageBucketAccessControlsDeleteRequest(_messages.Message):
class StorageBucketAccessControlsDeleteResponse(_messages.Message):
class StorageBucketAccessControlsGetRequest(_messages.Message):
class StorageBucketAccessControlsInsertRequest(_messages.Message):
class StorageBucketAccessControlsListRequest(_messages.Message):
class StorageBucketAccessControlsPatchRequest(_messages.Message):
class StorageBucketAccessControlsUpdateRequest(_messages.Message):
class StorageBucketsDeleteRequest(_messages.Message):
class StorageBucketsDeleteResponse(_messages.Message):
class StorageBucketsGetIamPolicyRequest(_messages.Message):
class StorageBucketsGetRequest(_messages.Message):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageBucketsInsertRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class PredefinedDefaultObjectAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageBucketsListRequest(_messages.Message):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageBucketsListChannelsRequest(_messages.Message):
class StorageBucketsLockRetentionPolicyRequest(_messages.Message):
class StorageBucketsPatchRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class PredefinedDefaultObjectAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageBucketsSetIamPolicyRequest(_messages.Message):
class StorageBucketsTestIamPermissionsRequest(_messages.Message):
class StorageBucketsUpdateRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class PredefinedDefaultObjectAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageChannelsStopResponse(_messages.Message):
class StorageDefaultObjectAccessControlsDeleteRequest(_messages.Message):
class StorageDefaultObjectAccessControlsDeleteResponse(_messages.Message):
class StorageDefaultObjectAccessControlsGetRequest(_messages.Message):
class StorageDefaultObjectAccessControlsInsertRequest(_messages.Message):
class StorageDefaultObjectAccessControlsListRequest(_messages.Message):
class StorageDefaultObjectAccessControlsPatchRequest(_messages.Message):
class StorageDefaultObjectAccessControlsUpdateRequest(_messages.Message):
class StorageNotificationsDeleteRequest(_messages.Message):
class StorageNotificationsDeleteResponse(_messages.Message):
class StorageNotificationsGetRequest(_messages.Message):
class StorageNotificationsInsertRequest(_messages.Message):
class StorageNotificationsListRequest(_messages.Message):
class StorageObjectAccessControlsDeleteRequest(_messages.Message):
class StorageObjectAccessControlsDeleteResponse(_messages.Message):
class StorageObjectAccessControlsGetRequest(_messages.Message):
class StorageObjectAccessControlsInsertRequest(_messages.Message):
class StorageObjectAccessControlsListRequest(_messages.Message):
class StorageObjectAccessControlsPatchRequest(_messages.Message):
class StorageObjectAccessControlsUpdateRequest(_messages.Message):
class StorageObjectsComposeRequest(_messages.Message):
  class DestinationPredefinedAclValueValuesEnum(_messages.Enum):
class StorageObjectsCopyRequest(_messages.Message):
  class DestinationPredefinedAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsDeleteRequest(_messages.Message):
class StorageObjectsDeleteResponse(_messages.Message):
class StorageObjectsGetIamPolicyRequest(_messages.Message):
class StorageObjectsGetRequest(_messages.Message):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsInsertRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsListRequest(_messages.Message):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsPatchRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsRewriteRequest(_messages.Message):
  class DestinationPredefinedAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsSetIamPolicyRequest(_messages.Message):
class StorageObjectsTestIamPermissionsRequest(_messages.Message):
class StorageObjectsUpdateRequest(_messages.Message):
  class PredefinedAclValueValuesEnum(_messages.Enum):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageObjectsWatchAllRequest(_messages.Message):
  class ProjectionValueValuesEnum(_messages.Enum):
class StorageProjectsServiceAccountGetRequest(_messages.Message):
class TestIamPermissionsResponse(_messages.Message):
class StorageProjectsHmacKeysCreateRequest(_messages.Message):
class StorageProjectsHmacKeysDeleteRequest(_messages.Message):
class StorageProjectsHmacKeysGetRequest(_messages.Message):
class StorageProjectsHmacKeysListRequest(_messages.Message):
class StorageProjectsHmacKeysUpdateRequest(_messages.Message):
class HmacKeyMetadata(_messages.Message):
class HmacKey(_messages.Message):
class HmacKeyMetadataList(_messages.Message):
class HmacKeysDeleteResponse(_messages.Message): def BindingsToDict(bindings):
  """Converts a list of BindingsValueListEntry to a dictionary.

  Args:
    bindings: A list of BindingsValueListEntry instances.

  Returns:
    A {role: set(members)} dictionary.
  """

  tmp_bindings = defaultdict(set)
  for binding in bindings:
    tmp_bindings[binding.role].update(binding.members)
  return tmp_bindings def BindingStringToTuple(is_grant, input_str):
  """Parses an iam ch bind string to a list of binding tuples.

  Args:
    is_grant: If true, binding is to be appended to IAM policy; else, delete
              this binding from the policy.
    input_str: A string representing a member-role binding.
               e.g. user:foo@bar.com:objectAdmin
                    user:foo@bar.com:objectAdmin,objectViewer
                    user:foo@bar.com
                    allUsers
                    deleted:user:foo@bar.com?uid=123:objectAdmin,objectViewer
                    deleted:serviceAccount:foo@bar.com?uid=123

  Raises:
    CommandException in the case of invalid input.

  Returns:
    A BindingsTuple instance.
  """

  if not input_str.count(':'):
    input_str += ':'

  # Allows user specified PUBLIC_MEMBERS, DISCOURAGED_TYPES, and TYPES to be
  # case insensitive.
  tokens = input_str.split(":")
  public_members = {s.lower(): s for s in PUBLIC_MEMBERS}
  types = {s.lower(): s for s in TYPES}
  discouraged_types = {s.lower(): s for s in DISCOURAGED_TYPES}
  possible_public_member_or_type = tokens[0].lower()
  possible_type = '%s:%s' % (tokens[0].lower(), tokens[1].lower())

  if possible_public_member_or_type in public_members:
    tokens[0] = public_members[possible_public_member_or_type]
  elif possible_public_member_or_type in types:
    tokens[0] = types[possible_public_member_or_type]
  elif possible_public_member_or_type in discouraged_types:
    tokens[0] = discouraged_types[possible_public_member_or_type]
  elif possible_type in types:
    (tokens[0], tokens[1]) = types[possible_type].split(':')
  input_str = ":".join(tokens)

  # We can remove project convenience members, but not add them.
  removing_discouraged_type = not is_grant and tokens[0] in DISCOURAGED_TYPES

  if input_str.count(':') == 1:
    if '%s:%s' % (tokens[0], tokens[1]) in TYPES:
      raise CommandException('Incorrect public member type for binding %s' %
                             input_str)
    elif tokens[0] in PUBLIC_MEMBERS:
      (member, roles) = tokens
    elif tokens[0] in TYPES or removing_discouraged_type:
      member = input_str
      roles = DROP_ALL
    else:
      raise CommandException('Incorrect public member type for binding %s' %
                             input_str)
  elif input_str.count(':') == 2:
    if '%s:%s' % (tokens[0], tokens[1]) in TYPES:
      # case "deleted:user:foo@bar.com?uid=1234"
      member = input_str
      roles = DROP_ALL
    elif removing_discouraged_type:
      (member_type, project_id, roles) = tokens
      member = '%s:%s' % (member_type, project_id)
    else:
      (member_type, member_id, roles) = tokens
      _check_member_type(member_type, input_str)
      member = '%s:%s' % (member_type, member_id)
  elif input_str.count(':') == 3:
    # case "deleted:user:foo@bar.com?uid=1234:objectAdmin,objectViewer"
    (member_type_p1, member_type_p2, member_id, roles) = input_str.split(':')
    member_type = '%s:%s' % (member_type_p1, member_type_p2)
    _check_member_type(member_type, input_str)
    member = '%s:%s' % (member_type, member_id)
  else:
    raise CommandException('Invalid ch format %s' % input_str)

  if is_grant and not roles:
    raise CommandException('Must specify a role to grant.')

  roles = [ResolveRole(r) for r in roles.split(',')]

  bindings = [
      apitools_messages.Policy.BindingsValueListEntry(members=[member], role=r)
      for r in set(roles)
  ]
  return BindingsTuple(is_grant=is_grant, bindings=bindings) TYPES = set([
    'user', 'deleted:user', 'serviceAccount', 'deleted:serviceAccount', 'group',
    'deleted:group', 'domain', 'principal', 'principalSet', 'principalHierarchy'
])
DISCOURAGED_TYPES = set([
    'projectOwner',
    'projectEditor',
    'projectViewer',
])
DISCOURAGED_TYPES_MSG = (
    'Assigning roles (e.g. objectCreator, legacyBucketOwner) for project '
    'convenience groups is not supported by gsutil, as it goes against the '
    'principle of least privilege. Consider creating and using more granular '
    'groups with which to assign permissions. See '
    'https://cloud.google.com/iam/docs/using-iam-securely for more '
    'information. Assigning a role to a project group can be achieved by '
    'setting the IAM policy directly (see gsutil help iam for specifics).')
PUBLIC_MEMBERS = set([
    'allUsers',
    'allAuthenticatedUsers',
])
DROP_ALL = ''
def SerializeBindingsTuple(bindings_tuple):
def DeserializeBindingsTuple(serialized_bindings_tuple):
def BindingsToDict(bindings):
def IsEqualBindings(a, b):
def DiffBindings(old, new):
def PatchBindings(base, diff):
def BindingStringToTuple(is_grant, input_str):
def _check_member_type(member_type, input_str):
def ResolveRole(role): def DiffBindings(old, new):
  """Computes the difference between two BindingsValueListEntry lists.

  Args:
    old: The original list of BindingValuesListEntry instances
    new: The updated list of BindingValuesListEntry instances

  Returns:
    A pair of BindingsTuple instances, one for roles granted between old and
      new, and one for roles removed between old and new.
  """
  tmp_old = BindingsToDict(old)
  tmp_new = BindingsToDict(new)

  granted = BindingsToDict([])
  removed = BindingsToDict([])

  for (role, members) in six.iteritems(tmp_old):
    removed[role].update(members.difference(tmp_new[role]))
  for (role, members) in six.iteritems(tmp_new):
    granted[role].update(members.difference(tmp_old[role]))

  granted = [
      apitools_messages.Policy.BindingsValueListEntry(role=r, members=list(m))
      for (r, m) in six.iteritems(granted)
      if m
  ]
  removed = [
      apitools_messages.Policy.BindingsValueListEntry(role=r, members=list(m))
      for (r, m) in six.iteritems(removed)
      if m
  ]

  return (BindingsTuple(True, granted), BindingsTuple(False, removed)) def IsEqualBindings(a, b):
  (granted, removed) = DiffBindings(a, b)
  return not granted.bindings and not removed.bindings def PatchBindings(base, diff):
  """Patches a diff list of BindingsValueListEntry to the base.

  Will remove duplicate members for any given role on a grant operation.

  Args:
    base: A list of BindingsValueListEntry instances.
    diff: A BindingsTuple instance of diff to be applied.

  Returns:
    The computed difference, as a list of
    apitools_messages.Policy.BindingsValueListEntry instances.
  """

  # Convert the list of bindings into an {r: [m]} dictionary object.
  tmp_base = BindingsToDict(base)
  tmp_diff = BindingsToDict(diff.bindings)

  # Patch the diff into base
  if diff.is_grant:
    for (role, members) in six.iteritems(tmp_diff):
      if not role:
        raise CommandException('Role must be specified for a grant request.')
      tmp_base[role].update(members)
  else:
    for role in tmp_base:
      tmp_base[role].difference_update(tmp_diff[role])
      # Drop all members with the DROP_ALL role specifed from input.
      tmp_base[role].difference_update(tmp_diff[DROP_ALL])

  # Construct the BindingsValueListEntry list
  bindings = [
      apitools_messages.Policy.BindingsValueListEntry(role=r, members=list(m))
      for (r, m) in six.iteritems(tmp_base)
      if m
  ]

  return bindings
from collections import defaultdict
from gslib.exception import CommandException
from gslib.project_id import PopulateProjectId
from gslib.tests.testcase.integration_testcase import SkipForS3
from gslib.tests.testcase.integration_testcase import SkipForXML
from gslib.tests.util import GenerationFromURI as urigen
from gslib.tests.util import SetBotoConfigForTest
from gslib.tests.util import unittest
from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages
from gslib.utils.constants import UTF8
from gslib.utils.iam_helper import BindingsToDict
from gslib.utils.iam_helper import BindingStringToTuple as bstt
from gslib.utils.iam_helper import BindingsTuple
from gslib.utils.iam_helper import DiffBindings
from gslib.utils.iam_helper import IsEqualBindings
from gslib.utils.iam_helper import PatchBindings
from gslib.utils.retry_util import Retry
import json
import gslib.tests.testcase as testcase
    old = [bvle(role='x', members=['z', 'y'])]
    new = [bvle(role='x', members=['y'])]
    expected = [bvle(role='x', members=['z'])]
    (granted, removed) = DiffBindings(old, new)
    self.assertEquals(granted.bindings, [])
    self.assertEquals(removed.bindings, expected)

  def test_diff_bindings_swap_member(self):
    """Tests expected behavior of switching a member in a role."""
    old = [bvle(role='x', members=['z'])]
    new = [bvle(role='x', members=['y'])]
    (granted, removed) = DiffBindings(old, new)
    self.assertEquals(granted.bindings, new)
    self.assertEquals(removed.bindings, old)

  def test_patch_bindings_grant(self):
    """Tests patching a grant binding."""
    base = [
        bvle(role='a', members=['user:foo@bar.com']),
        bvle(role='b', members=['user:foo@bar.com']),
        bvle(role='c', members=['user:foo@bar.com']),
    ]
    diff = [
        bvle(role='d', members=['user:foo@bar.com']),
    ]
    expected = base + diff
    res = PatchBindings(base, BindingsTuple(True, diff))
    self.assertTrue(IsEqualBindings(res, expected))

  def test_patch_bindings_remove(self):
    """Tests patching a remove binding."""
    base = [
        bvle(members=['user:foo@bar.com'], role='a'),
        bvle(members=['user:foo@bar.com'], role='b'),
        bvle(members=['user:foo@bar.com'], role='c'),
    ]
    diff = [
        bvle(members=['user:foo@bar.com'], role='a'),
    ]
    expected = [
        bvle(members=['user:foo@bar.com'], role='b'),
        bvle(members=['user:foo@bar.com'], role='c'),
    ]

    res = PatchBindings(base, BindingsTuple(False, diff))
    self.assertTrue(IsEqualBindings(res, expected))

  def test_patch_bindings_remove_all(self):
    """Tests removing all roles from a member."""
    base = [
        bvle(members=['user:foo@bar.com'], role='a'),
        bvle(members=['user:foo@bar.com'], role='b'),
        bvle(members=['user:foo@bar.com'], role='c'),
    ]
    diff = [
        bvle(members=['user:foo@bar.com'], role=''),
    ]
    res = PatchBindings(base, BindingsTuple(False, diff))
    self.assertEquals(res, [])

    diff = [
        bvle(members=['user:foo@bar.com'], role='a'),
        bvle(members=['user:foo@bar.com'], role='b'),
        bvle(members=['user:foo@bar.com'], role='c'),
    ]

    res = PatchBindings(base, BindingsTuple(False, diff))
    self.assertEquals(res, [])

  def test_patch_bindings_multiple_users(self):
    """Tests expected behavior when multiple users exist."""
    expected = [
        bvle(members=['user:fii@bar.com'], role='b'),
    ]
    base = [
        bvle(members=['user:foo@bar.com'], role='a'),
        bvle(members=['user:foo@bar.com', 'user:fii@bar.com'], role='b'),
        bvle(members=['user:foo@bar.com'], role='c'),
    ]
    diff = [
        bvle(members=['user:foo@bar.com'], role='a'),
        bvle(members=['user:foo@bar.com'], role='b'),
        bvle(members=['user:foo@bar.com'], role='c'),
    ]
    res = PatchBindings(base, BindingsTuple(False, diff))
    self.assertTrue(IsEqualBindings(res, expected))

  def test_patch_bindings_grant_all_users(self):
    """Tests a public member grant."""
    base = [
        bvle(role='a', members=['user:foo@bar.com']),
        bvle(role='b', members=['user:foo@bar.com']),
        bvle(role='c', members=['user:foo@bar.com']),
    ]
    diff = [
        bvle(role='a', members=['allUsers']),
    ]
    expected = [
        bvle(role='a', members=['allUsers', 'user:foo@bar.com']),
        bvle(role='b', members=['user:foo@bar.com']),
        bvle(role='c', members=['user:foo@bar.com']),
    ]

    res = PatchBindings(base, BindingsTuple(True, diff))
    self.assertTrue(IsEqualBindings(res, expected))

  def test_patch_bindings_public_member_overwrite(self):
    """Tests public member vs. public member interaction."""
    base = [
        bvle(role='a', members=['allUsers']),
    ]
    diff = [
        bvle(role='a', members=['allAuthenticatedUsers']),
    ]

    res = PatchBindings(base, BindingsTuple(True, diff))
    self.assertTrue(IsEqualBindings(res, base + diff))

  def test_valid_public_member_single_role(self):
    """Tests parsing single role (case insensitive)."""

    (_, bindings) = bstt(True, 'allusers:admin')