@contextmanager
def notifies_hipchat(start_msg, end_msg):
    """
    A decorator to post a notification to hipchat at the start and end of this
    function.

    The `FOO_msg` arguments define template strings that can use the
    following variables as context:

    * `deployer` The deploying user
    * `deployment_name` The deploying environment. Eg. "beta"
    * `generation` The generational target. Eg. "live", "pending"
    * `git_branch` The current git branch name.
    * `duration` The number of wall-clock seconds taken to complete the
      decorated method. Note: Only available to `end_msg`.
    """
    # Ensure we have the required configs
    hipchat_conf = {}
    for key in ['api_token', 'room']:
        hipchat_conf[key] = env.get('hipchat_%s' % key, None)
    for key, value in hipchat_conf.items():
        if value is None:
            logger.warning(
                "No hipchat_%s found. Not notifying.",
                key,
            )
            yield
            logger.warning(
                "No hipchat_%s found. Not notifying.",
                key,
            )
            return
    hipchat_conf['color'] = env.get('hipchat_color', 'green')
    hipchat_conf['from'] = env.get('hipchat_from', 'Neckbeard')

    # Get our git branchname. Fallback to a SHA if detached head
    r = git.Repo('.')
    branch_name = r.commit().hexsha[:7]
    if not r.head.is_detached:
        branch_name = r.active_branch.name

    # Build the message
    context = {
        'deployer': get_deployer(),
        'deployment_name': env.get('_deployment_name', 'unknown'),
        'generation': _get_gen_target().lower(),
        'git_branch': branch_name,
    }
    message = start_msg % context

    _send_hipchat_msg(message, hipchat_conf)

    method_start = datetime.now()
    yield
    duration = datetime.now() - method_start

    context['duration'] = duration.seconds
    message = end_msg % context

    _send_hipchat_msg(message, hipchat_conf) def _get_git_repo(path='.'):
    if git is None:
        raise ImportError('This check requires GitPython to be installed.')

    try:
        repo = git.Repo(path)
    except git.InvalidGitRepositoryError:
        logger.critical('Not in a git repository.')
        exit(1)
    return repo def _is_unchanged_from_head(repo):
    # Check that there are no uncommitted changes.
    head_commit = repo.head.commit
    if head_commit.diff(None) != []:  # Diff HEAD commit versus working copy.
        return False
    return True def _is_tagged_version(repo):
    """
    Check that we are deploying from a tagged commit, and have no uncommitted
    changes.

    Return a boolean.
    """
    # Check that the tag is correct
    head_commit = repo.head.commit
    for tag in reversed(repo.tags):
        if tag.commit == head_commit:
            return True
    return False def _push_tags(repo):
    # Push tags if there is just a single remote.
    if len(repo.remotes) == 1:
        remote = repo.remotes[0]
        logger.info("Pushing tags for current git repository.")
        remote.push(tags=True)
    else:
        logger.warning(
            "Could not push tags, there are %d remotes." % len(repo.remotes)) def _take_temporary_pagerduty(
        duration, api_key, user_id, project_subdomain, schedule_key):
    '''
    ``duration`` is the amount of seconds that the pagerduty user associated
    with the ``user_id`` will take over pagerduty at ``project_subdomain`` for
    the schedule associated with the ``schedule_key``. The ``api_key`` is
    needed for authentication and can be found at
    https://``project_subdomain``.pagerduty.com/api_keys.
    '''
    logger.info('Attempting Pagerduty override')

    # Set up headers
    headers = {
        'Content-type': 'application/json',
        'Authorization': 'Token token=%s' % api_key,
    }

    # Set up time start and end dates
    now = datetime.utcnow()
    later = now + timedelta(seconds=duration)
    # Using a trailing Z to indicate UTC time
    payload = {
        'override': {
            'user_id': '%s' % user_id,
            'start': '%sZ' % now.isoformat(),
            'end': '%sZ' % later.isoformat(),
        },
    }

    # Post to pagerduty to override whos on call.
    response = requests.post(
        PAGERDUTY_SCHEDULE_URL % {
            'project_subdomain': project_subdomain,
            'schedule_key': schedule_key,
        },
        data=json.dumps(payload),
        headers=headers,
    )

    # Alert the deployer if there is an error.
    if not response.ok:
        logger.error(
            'Pagerduty override failed status code: "%s".',
            response.status_code,
        )
        logger.error(
            'Error response output: "%s".',
            response.content,
        )
    else:
        logger.info('Pagerduty override has completed successfully.') DT_NOTIFY = True def _send_deployment_done_desktop_notification(pre_deploy_time, deployer):
    if not DT_NOTIFY:
        return
    title = "%(deployment)s %(target)s %(node_name)s deployed"
    content = "Took %(seconds)s seconds"
    time_diff = datetime.now() - pre_deploy_time
    context = {
        'deployment': env._deployment_name,
        'target': _get_gen_target().lower(),
        'node_name': deployer.node_name,
        'seconds': time_diff.seconds,
    }

    notification = pynotify.Notification(title % context, content % context)
    notification.show() def _announce_deployment():
    newrelic_conf = env._deployment_confs['conf'].get('newrelic', {})
    if newrelic_conf.get('announce_deploy', False):
        _send_deployment_end_newrelic() ACTIVE = 'ACTIVE' @contextmanager
def logs_duration(deploy_timers, timer_name='Total', output_result=False):
    """
    A decorator to output the duration of the function after completion.
    """
    start_time = datetime.now()
    try:
        yield
    finally:
        duration = datetime.now() - start_time
        deploy_timers[timer_name] = duration.seconds
        if output_result:
            time_logger.info("%02ds- %s", duration.seconds, timer_name) @contextmanager
def prompt_on_exception(msg):
    try:
        yield
    except Exception, e:
        logger.warning(msg)
        logger.warning("Exception thrown: %s", e)

        continue_choice = 'C'
        abort_choice = 'A'
        opts = [continue_choice, abort_choice]
        prompt_str = "[C]ontinue the deploy or [A]bort?"
        user_opt = None
        while not user_opt in opts:
            user_opt = prompt(prompt_str)

        if user_opt == abort_choice:
            logger.critical("Aborting deploy")
            exit(1)
        logger.warning("Continuing, despite error") class Deployment(object):
    """
    Use configuration info to classify all currently running ec2 and RDS
    instances and group them by deployment generation.
    """
    def __init__(self, deployment_name, ec2_nodes, rds_nodes, elb_nodes):
        """
        ``deployment_name`` A string uniquely identifying a deployment.
        """
        self.deployment_name = deployment_name
        self.deployment_confs = {}
        self.deployment_confs['ec2'] = ec2_nodes
        self.deployment_confs['rds'] = rds_nodes
        self.deployment_confs['elb'] = elb_nodes

        aws_credentials = self._get_valid_aws_credentials(
            self.deployment_confs,
        )

        self._pending_gen_id = None
        self._active_gen_id = None

        self.ec2conn = ec2.EC2Connection(
            aws_credentials['access_key_id'],
            aws_credentials['secret_access_key'],
        )
        self.rdsconn = rds.RDSConnection(
            aws_credentials['access_key_id'],
            aws_credentials['secret_access_key'],
        )

    def _get_valid_aws_credentials(self, deployment_confs):
        # TODO: Instead of just ensuring that all of the resources use the same
        # credentials, actually manage multiple connection
        aws_credentials = {
            'access_key_id': None,
            'secret_access_key': None,
        }
        for aws_type, configs_of_type in deployment_confs.items():
            for name, resource_config in configs_of_type.items():
                aws_config = resource_config.get('aws')
                if aws_config is None:
                    raise MissingAWSCredentials(
                        "%s resource %s has no 'aws' configs" % (
                            aws_type,
                            name,
                        ),
                    )

                # Ensure that the keys exist and that they're not different
                # from those found on other resources
                for key in ['access_key_id', 'secret_access_key']:
                    value = aws_config.get(key)

                    if value is None:
                        raise MissingAWSCredentials(
                            "%s resource %s has no '%s' config" % (
                                aws_type,
                                name,
                                key,
                            ),
                        )
                    other_resource_value = aws_credentials.get(key)
                    if other_resource_value is not None:
                        if value != other_resource_value:
                            raise NonUniformAWSCredentials(
                                (
                                    "%s resource %s has no '%s' config "
                                    "different from previously seen"
                                ) % (
                                    aws_type,
                                    name,
                                    key,
                                ),
                            )
                    aws_credentials[key] = value

        return aws_credentials

    @property
    def active_gen_id(self):
        if self._active_gen_id:
            return self._active_gen_id

        active_nodes = InfrastructureNode.objects.filter(
            deployment_name=self.deployment_name,
            is_active_generation=1)
        if len(active_nodes) == 0:
            return None

        first_active_node = active_nodes[0]
        gen_id = first_active_node.generation_id
        for active_node in active_nodes:
            if active_node.generation_id != gen_id:
                err_str = (
                    "Inconsistent generation ids in simpledb. "
                    "%s:%s and %s:%s both marked active"
                )
                context = (
                    first_active_node.aws_id,
                    first_active_node.generation_id,
                    active_node.aws_id,
                    active_node.generation_id)
                raise Exception(err_str % context)

        self._active_gen_id = gen_id

        return self._active_gen_id

    @property
    def pending_gen_id(self):
        if self._pending_gen_id:
            return self._pending_gen_id

        if self.active_gen_id:
            self._pending_gen_id = self.active_gen_id + 1
            return self._pending_gen_id

        self._pending_gen_id = 1
        return self._pending_gen_id

    def get_blank_node(self, aws_type):
        node = InfrastructureNode()
        node.set_aws_conns(self.ec2conn, self.rdsconn)
        node.aws_type = aws_type

        return node

    def get_active_node(self, aws_type, node_name):
        if self.active_gen_id:
            return self.get_node(
                aws_type, node_name, self.active_gen_id, is_running=1)
        return None

    def get_pending_node(self, aws_type, node_name):
        return self.get_node(
            aws_type, node_name, self.pending_gen_id, is_running=1)

    def get_all_active_nodes(self, is_running=None):
        if self.active_gen_id:
            return self.get_all_nodes(
                self.active_gen_id, is_running=is_running)
        return []

    def get_all_pending_nodes(self, is_running=None):
        return self.get_all_nodes(self.pending_gen_id, is_running=is_running)

    def get_all_old_nodes(self, is_running=None):
        nodes = []
        all_nodes = self.get_all_nodes()
        new_generations = [self.active_gen_id, self.pending_gen_id]
        for node in all_nodes:
            if node.generation_id not in new_generations:
                nodes.append(node)

        nodes.sort(key=lambda n: n.generation_id)

        return nodes

    def get_all_nodes(self, generation_id=None, is_running=None):
        matching_nodes = InfrastructureNode.objects.filter(
            deployment_name=self.deployment_name,
        )
        if generation_id:
            matching_nodes = matching_nodes.filter(generation_id=generation_id)
        if is_running is not None:
            matching_nodes = matching_nodes.filter(is_running=is_running)

        configured_nodes = []
        for node in matching_nodes:
            node.set_aws_conns(self.ec2conn, self.rdsconn)
            deploy_conf = self.deployment_confs[node.aws_type].get(
                node.name, None)
            if deploy_conf is None:
                # We don't have a config for this named node
                # If it's not running though, it's probably from an old
                # deployment and we've since changed our infrastructure
                # configuration, so it's not worth worrying about
                if node.is_running:
                    logger.warning(
                        "No DEPLOYMENTS configuration exists in the <%s> "
                        "deployment for the <%s> node with the name <%s>",
                        self.deployment_name,
                        node.aws_type,
                        node.name,
                    )
                    logger.info(
                        "Available configurations: %s",
                        self.deployment_confs[node.aws_type].keys(),
                    )
            else:
                node.set_deployment_info(
                    self.deployment_confs[node.aws_type][node.name])
                configured_nodes.append(node)

        return configured_nodes

    def get_node(self, aws_type, node_name, generation_id, is_running=1):
        matching_nodes = InfrastructureNode.objects.filter(
            deployment_name=self.deployment_name,
            generation_id=generation_id,
            aws_type=aws_type,
            name=node_name,
            is_running=is_running,
        )
        if len(matching_nodes) > 1:
            raise Exception('More than one matching node')
        elif len(matching_nodes) == 1:
            node = matching_nodes[0]
            node.set_aws_conns(self.ec2conn, self.rdsconn)
            node.set_deployment_info(
                self.deployment_confs[node.aws_type][node.name])
            return node
        else:
            return None

    def set_active_node(self, aws_type, node_name, boto_object):
        if self.active_gen_id:
            gen_id = self.active_gen_id
        else:
            gen_id = self.pending_gen_id

        return self.set_node(
            aws_type,
            node_name,
            gen_id,
            boto_object,
            is_active=True)

    def set_pending_node(self, aws_type, node_name, boto_object):
        return self.set_node(
            aws_type,
            node_name,
            self.pending_gen_id,
            boto_object,
            is_active=False)

    def set_node(
        self, aws_type, node_name, generation_id, boto_object, is_active,
    ):

        aws_id = boto_object.id

        matching_nodes = InfrastructureNode.objects.filter(
            aws_type=aws_type,
            aws_id=aws_id,
            deployment_name=self.deployment_name,
        )
        if len(matching_nodes) == 1:
            node = matching_nodes[0]
        else:
            node = self.get_blank_node(aws_type)

        node.generation_id = generation_id
        node.deployment_name = self.deployment_name
        node.aws_type = aws_type
        node.aws_id = aws_id
        node.name = node_name
        node.creation_date = datetime.now()
        node.is_running = 1
        node.is_active_generation = is_active

        node.save()

    def get_new_rds_label(self, node_name, version, is_active=False):
        counter = self.pending_gen_id
        if is_active:
            counter = self.active_gen_id

        rds_label = PstatRdsId(
            'pstat' + self.deployment_name, version, counter)
        return rds_label

    def verify_running_state(self, nodes):
        for node in nodes:
            node.verify_running_state()

    def verify_active_deployment_state(self):
        """
        Ensures that all node entries match nodes. If an entry is found for an
        ``is_running`` node that isn't running, that entry is updated.

        Nodes that actually exist but aren't recorded are NOT affected.
        """
        nodes = self.get_all_active_nodes()
        self.verify_running_state(nodes)

    def verify_pending_deployment_state(self):
        nodes = self.get_all_pending_nodes()
        self.verify_running_state(nodes)

    def verify_old_deployment_state(self):
        nodes = self.get_all_old_nodes()
        self.verify_running_state(nodes)

    def verify_deployment_state(self, verify_old=True):
        """
        Ensures that all node entries match nodes. If an entry is found for an
        ``is_running`` node that isn't running, that entry is updated.

        Nodes that actually exist but aren't recorded are NOT affected.
        """
        self.verify_pending_deployment_state()
        self.verify_active_deployment_state()
        if verify_old:
            self.verify_old_deployment_state()

    def get_inoperational_active_nodes(self):
        """
        Get a list of configured nodes that aren't operational.

        If no running node exists to match a configuration, a mock node is
        returned in its place with no aws_id.
        """
        inoperational = []

        # Ensure that all roles are filled exactly once with operational nodes
        for aws_type, confs in self.deployment_confs.items():
            for node_name, node_confs in confs.items():
                node = self.get_active_node(aws_type, node_name)
                if not node:
                    mock_node = self.get_blank_node(aws_type)
                    mock_node.name = node_name
                    inoperational.append(mock_node)
                    logger.info("Missing node: %s-%s" % (aws_type, node_name))
                    continue
                if not node.is_operational:
                    inoperational.append(node)

        return inoperational

    def get_unhealthy_active_nodes(self):
        """
        Get a list of configured nodes that don't exist or aren't healthy.
        """
        nodes = self.get_all_active_nodes()

        # Check that all running nodes are healthy
        unhealthy = [
            node for node in nodes
            if node.is_running and not node.is_healthy
        ]

        # Ensure that all roles are filled exactly once with healthy nodes
        for aws_type, confs in self.deployment_confs.items():
            for node_name, node_confs in confs.items():
                node = self.get_active_node(aws_type, node_name)
                if not node:
                    mock_node = self.get_blank_node(aws_type)
                    mock_node.name = node_name
                    unhealthy.append(mock_node)
                    logger.info("Missing node: %s-%s" % (aws_type, node_name))
                    continue
                if not node.is_healthy:
                    unhealthy.append(node)
                    logger.info("Node unhealthy: %s" % node)

        return unhealthy

    def get_unhealthy_pending_nodes(self):
        """
        Get a list of configured nodes that don't exist or aren't healthy.
        """
        nodes = self.get_all_pending_nodes()

        # Check that all running nodes are healthy
        unhealthy = [
            node for node in nodes
            if node.is_running and not node.is_healthy
        ]

        # Ensure that all roles are filled exactly once with healthy nodes
        for aws_type, confs in self.deployment_confs.items():
            for node_name, node_confs in confs.items():
                node = self.get_pending_node(aws_type, node_name)
                if not node:
                    mock_node = self.get_blank_node(aws_type)
                    mock_node.name = node_name
                    unhealthy.append(mock_node)
                    logger.info("Missing node: %s-%s" % (aws_type, node_name))
                    continue
                if not node.is_healthy:
                    unhealthy.append(node)
                    logger.info("Node unhealthy: %s" % node)

        return unhealthy

    def active_is_healthy(self):
        """
        Determine if the active generation is fully healthy.
        """
        unhealthy_nodes = self.get_unhealthy_active_nodes()
        if unhealthy_nodes:
            return False

        return True

    def active_is_fully_operational(self):
        inoperational_nodes = self.get_inoperational_active_nodes()
        if inoperational_nodes:
            return False
        return True

    def pending_is_healthy(self):
        """
        Determine if the pending generation is fully healthy.
        """
        unhealthy_nodes = self.get_unhealthy_pending_nodes()
        if unhealthy_nodes:
            return False
        return True

    def repair_active_generation(
        self, force_operational=False, wait_until_operational=True,
    ):
        """
        Ensure that all healthy active-generation nodes are operational.

        ``force_operational`` If True, even non-healthy active-generation nodes
        will be made operational.
        ``wait_until_operational`` If False, doesn't wait for the nodes to
        become fully operational.

        Returns any nodes that were made operational by this action.
        """
        if self.active_is_fully_operational():
            logger.info("All active nodes are operational")
            return []

        nodes = self.get_inoperational_active_nodes()

        made_operational = []
        for node in nodes:
            if not node.aws_id:
                logger.warning(
                    "Node missing: %s- %s",
                    node.aws_type,
                    node.name,
                )
                continue
            if not node.is_operational:
                if node.is_healthy or force_operational:
                    logger.info("Making node operational: %s" % node)
                    node.make_operational(force_operational=force_operational)
                    made_operational.append(node)
                else:
                    logger.warning("Node unhealthy: %s" % node.boto_instance)
            else:
                logger.debug("Node operational: %s" % node)

        fixed_nodes = copy(made_operational)
        # Wait til of these nodes are actually operational
        if len(fixed_nodes) == 0:
            logger.info("No healthy non-operational nodes available")
            return []

        if not wait_until_operational:
            return fixed_nodes

        logger.info("Waiting until all nodes are actually operational")
        time_waited = 0
        while made_operational:
            if time_waited > MAKE_OPERATIONAL_TIMEOUT:
                raise WaitTimedOut(
                    "Timed out waiting on nodes: %s" % made_operational)
            _made_operational = made_operational[:]
            for node in _made_operational:
                if node.is_operational:
                    made_operational.remove(node)
            if made_operational:
                logger.info(
                    "%s node(s) still not operational. Waiting %ss",
                    len(made_operational),
                    WAIT_TIME,
                )
                time.sleep(WAIT_TIME)
                time_waited += WAIT_TIME

        return fixed_nodes

    def uses_rds(self):
        """
        Does this deployment uses rds. (is there an RDS node in the config)
        """
        if len(self.deployment_confs['rds'].keys()) == 0:
            return False
        return True

    def has_required_redundancy(self, node):
        """
        Determine whether the given ``node`` has the required level of
        redundancy so that it can be rendered inoperational without service
        interruption to the deployment as a whole.

        Returns True if there are redundant nodes, False otherwise.

        The logic is currently crude in that only one other node of the same
        ``aws_type`` needs to be available.
        """
        all_gen_nodes = self.get_all_nodes(node.generation_id, is_running=True)
        same_aws_nodes = [
            n
            for n in all_gen_nodes
            if n.aws_type == node.aws_type
        ]

        other_operational_nodes = []
        for other_node in same_aws_nodes:
            # Don't include the node we're checking
            if other_node.name != node.name:
                if other_node.is_operational:
                    other_operational_nodes.append(other_node)

        return bool(other_operational_nodes)

    def increment_generation(self):
        """
        Node by node, make the nodes in pending generation operational.
        """
        if not self.pending_is_healthy():
            raise Exception(
                "Pending generation must be fully healthy "
                "to increment_generation",
            )

        active_nodes = self.get_all_active_nodes()
        pending_nodes = self.get_all_pending_nodes()

        # Set is_active_generation to 0 for the active and 1 for pending
        for node in active_nodes:
            node.is_active_generation = 0
            node.save()
        for node in pending_nodes:
            node.is_active_generation = 1
            node.save()
        self._active_gen_id += 1
        self._pending_gen_id += 1

        logger.info("Generation succesfully incremented.")
        logger.info("Making nodes operational")
        self.repair_active_generation() class Ec2NodeDeployment(BaseNodeDeployment):
    """
    Deployment controller for an ec2 appserver nodes. Deals with the
    AWS-specific configuration actions and delegates system-level configuration
    to the provisioner.
    """

    def __init__(self, *args, **kwargs):
        """
        ``conf[ebs]`` is a dictionary mapping out exactly where and how any EBS
        volumes should be attached, formatted and mounted, along with any
        system directories that need to be symlinked to live on the EBS volume.

        eg.

            ebs_confs = {
                'vols': {
                    'fs': {
                        'device': '/dev/sdf',
                        'filesystem': 'xfs',
                        'size': 50, # Size in GB
                        'mount_point': '/vol/fs',
                        'ebs_bound_dirs': [
                            {
                                'bind_path': '/var/log/pstat',
                                'src_path': '/vol/fs/var/log/pstat',
                            },
                            {
                                'bind_path': '/var/log/celery',
                                'src_path': '/vol/fs/var/log/celery',
                            },
                        ],
                    },
                },
            }
        """
        conf = kwargs['conf']
        self.ebs_confs = conf.get('ebs', {'vols': {}})

        super(Ec2NodeDeployment, self).__init__(*args, **kwargs)

        # A volume-label keyed dictionary with EBS seed snapshot ids
        # eg. {'logs': 'snap-xxxxxxx'}
        self.seed_ebs_snapshots = {}
        # A volume-label keyed dictionary with the attached EBS volume id
        # eg. {'fs': 'vol-xxxxxxx'}
        self.attached_volumes = {}

    def get_seed_data(self):
        """
        Gather the seed data with which to start the node.
        """
        if not self.seed_deployment:
            return
        require('aws_access_key_id')
        require('aws_secret_access_key')

        self.seed_ebs_snapshots = self._get_seed_ebs_snapshots()

    def wait_until_created(self, node):
        while not self.creation_complete(node):
            logging.info("Instance pending. Waiting %ss", LAUNCH_REFRESH)
            time.sleep(LAUNCH_REFRESH)

    def creation_complete(self, node):
        start_host_string = env.host_string
        start_host = env.host
        node.refresh_boto_instance()
        if node.boto_instance.state == 'running':
            # Try to SSH in to test that it's actually available
            timeout = 60  # 1 minute
            tries = 0
            step = LAUNCH_REFRESH
            while tries * step <= timeout:
                node.refresh_boto_instance()
                env.host_string = node.boto_instance.public_dns_name
                env.host = node.boto_instance.public_dns_name
                tries += 1
                try:
                    with hide('everything'):
                        sudo('uptime', pty=True)
                    env.host_string = start_host_string
                    env.host = start_host
                    return True
                except:
                    logger.info(
                        "%s not ready for SSH. Waiting %ss",
                        node.boto_instance.public_dns_name,
                        LAUNCH_REFRESH)
                    time.sleep(LAUNCH_REFRESH)

        env.host_string = start_host_string
        env.host = start_host
        return False

    def create_new_node(self):
        """
        Launch a new ec2 instance from the appropriate AMI and configure any
        EBS volumes and first-run-only configurations.
        """
        logger.info("Launching new ec2 instance")
        ec2_instance = self.launch()

        if self.is_active:
            self.deployment.set_active_node(
                'ec2', env.node_name, ec2_instance)
        else:
            self.deployment.set_pending_node(
                'ec2', env.node_name, ec2_instance)

        # Build EBS volumes, unless explicitly told not to.
        ebs_debug = self.ebs_confs.get('debug', {})
        if not ebs_debug.get('attach_ebs_volumes', True):
            logger.warning("Debug ebs_conf found")
            logger.warning("Not attaching configured EBS volumes.")

            return self.get_node()

        vol_confs = self.ebs_confs.get('vols', {})
        self.attached_volumes = self._attach_ebs_vols(
            ec2_instance_id=ec2_instance.id,
            vol_confs=vol_confs,
            seed_ebs_snapshots=self.seed_ebs_snapshots)

        # TODO: Find a way to move this fstab stuff to the normal deploy
        # Otherwise, we're reducing our ability to parallelize node
        # creation. If we use new fabric to actually parallelize the
        # create_new_node deploy process, then this isn't actually a big
        # deal, though.
        node = self.get_node()
        self.wait_until_created(node)

        # Ensure we're running commands against our spanking-new instance
        node.refresh_boto_instance()
        env.host_string = node.boto_instance.public_dns_name

        self._ensure_ebs_vols_mounted(ec2_instance.id, vol_confs)

        return self.get_node()

    def deploy(self, node, first_run=False):
        logger.info("Deploying to instance: %s" % node)
        self.wait_until_created(node)

        # RDS vs non-RDS configuration
        env.db_host = 'localhost'
        env.db_master_user = 'root'
        env.db_master_password = ''
        rds_node = self._get_masterdb()
        if rds_node:
            env.db_host, _ = rds_node.boto_instance.endpoint  # (Host, Port)
            env.db_master_user = rds_node.boto_instance.master_username
            env.db_master_password = env.rds_master_password

        # Set fab parameters based on the ec2 instance
        env.hosts = [node.boto_instance.public_dns_name]
        env.host_string = node.boto_instance.public_dns_name
        env.host = node.boto_instance.public_dns_name

        env.hostname = '%s-%s' % (env.get('env'), env.get('node_name'))
        env.ec2_instance_id = node.boto_instance.id

        if first_run:
            ebs_confs = self.ebs_confs
            ebs_debug = ebs_confs.get('debug', {})
            if ebs_debug.get('attach_ebs_volumes', True):
                # Create the .ec2tools configuration for snapshotting/backups
                self._configure_ec2tools(
                    self.deployment.ec2conn,
                    node.boto_instance.id,
                    self.ebs_confs['vols'])
                if ebs_confs.get('do_snapshot_backups', False):
                    self._configure_ebs_backup(
                        '/opt/pstat/versions/current/pstat',
                        '/home/policystat/env/bin/python',
                        '/vol/fs/pstat_storage',
                    )

            else:
                logger.warning(
                    "Debug ebs_conf found with 'attach_ebs_volumes' False")
                logger.warning("Not configuring ec2tools or backups.")

            # Fix file permissions
            logger.info(
                "Fixing file permissions and starting stopped services")
            self.provisioner.stop_services()
            self.provisioner.fix_folder_perms()
            self.provisioner.start_services()

        node_role_map = self._get_node_role_map()

        self.provisioner.do_update(
            node_role_map=node_role_map,
            node_roles=node._deployment_info.get('roles', []),
            first_run=not self.initial_deploy_complete,
        )

    def _get_node_role_map(self):
        """
        Get a dictionary with mappings of roles and boto ec2 instances in the
        same availability zone and all nodes with the role.

        Eg. {
            'memcached': {
                'same_az': [foo1],
                'all': [foo1, foo2]
            }
        }
        """
        def _role_map():
            return {'same_az': [], 'all': []}

        nodes_by_role = defaultdict(_role_map)

        if self.is_active:
            nodes = self.deployment.get_all_active_nodes(is_running=1)
        else:
            nodes = self.deployment.get_all_pending_nodes(is_running=1)

        deploy_node = self.get_node()
        local_az = deploy_node.boto_instance.placement

        for node in [n for n in nodes if n.aws_type == 'ec2']:
            node_confs = self.deployment.deployment_confs['ec2'][node.name]
            roles = node_confs.get('roles', [])
            for role in roles:
                nodes_by_role[role]['all'].append(node)
                if node.boto_instance.placement == local_az:
                    nodes_by_role[role]['same_az'].append(node)

        return nodes_by_role

    def _get_debug_seed_snapshot(self, vol_label):
        """
        Determine if the ebs_confs is configured in debug mode and has a seed
        snapshot for the given `vol_label`.

        Returns the snapshot_id if it exists, None otherwise.
        """
        ebs_debug = self.ebs_confs.get('debug', {})
        if not ebs_debug:
            return None

        vols_debug = ebs_debug.get('vols', {})
        vol_debug = vols_debug.get(vol_label, None)
        return vol_debug

    def _get_seed_ebs_snapshots(self):
        """
        Take snapshots of the seed EBS volumes to use as sources for new EBS
        volumes.

        Returns a dictionary of volume names and their snapshot ids.
        """
        logger.info(u"Determining seed EBS snapshots")
        # Perform EBS snapshots on the configured volumes

        seed_snapshots = {}
        for volume_name, volume_conf in self.ebs_confs['vols'].items():
            snapshot_id = self._get_debug_seed_snapshot(volume_name)
            if snapshot_id is None:
                env.host_string = self.seed_node.boto_instance.public_dns_name
                snapshot_id = self._get_ebs_snapshot(volume_name, volume_conf)
            else:
                logger.info(
                    "Using hard-coded debug snapshot_id %s for volume: %s",
                    snapshot_id,
                    volume_name
                )

            seed_snapshots[volume_name] = snapshot_id

        return seed_snapshots

    def _install_ec2_consistent_snapshot(self):
        """
        Install the ec2-consistent-snapshot utility to handle xfs freezing and
        unfreezing.

        https://github.com/alestic/ec2-consistent-snapshot
        """
        with hide(*fab_quiet):
            install_check = run('which ec2-consistent-snapshot')
        if install_check.return_code != 0:
            logger.info("Installing ec2-consistent-snapshot")

            with hide(*fab_output_hides):
                sudo('add-apt-repository ppa:alestic')
                sudo('apt-get update')
                sudo('apt-get install ec2-consistent-snapshot -y')

    def _get_ebs_snapshot(self, volume_name, volume_conf):
        """
        Take a snapshot on the current host of the EBS volume corresponding to
        ``volume_name`` and return the snapshot id.
        """
        self._install_ec2_consistent_snapshot()

        # Need to take a snapshot for use
        logger.info(u"Snapshotting EBS vols from %s", env.host_string)

        if volume_conf['filesystem'] != 'xfs':
            raise NotImplementedError("Only XFS EBS vols are supported")

        # Find the volume id of the appropriate seed EBS volume
        device = volume_conf['device']
        seed_volume = _get_attached_volume(
            self.seed_deployment.ec2conn,
            self.seed_node.boto_instance.id,
            device)

        if seed_volume is None:
            # No seed volume with matching device found
            logger.critical("No seed volume on device %s found", device)
            exit(1)

        # Build the ec2-consistent-snapshot command to take the snapshot for
        # the appropriate volume
        mount_point = volume_conf['mount_point']
        cmd_tpl = (
            'ec2-consistent-snapshot '
            '--freeze-filesystem %(mount_point)s '
            '--description '
            '"seed from %(seed_deploy)s-%(seed_node)s to %(deploy)s-%(node)s" '
            '--aws-access-key-id %(aws_access_key_id)s '
            '--aws-secret-access-key %(aws_secret_access_key)s '
            '%(volume_id)s '
        )
        context = {
            'mount_point': mount_point,
            'seed_deploy': self.seed_deployment.deployment_name,
            'seed_node': self.seed_node_name,
            'deploy': self.deployment.deployment_name,
            'node': self.node_name,
            'aws_access_key_id': env.aws_access_key_id,
            'aws_secret_access_key': env.aws_secret_access_key,
            'volume_id': seed_volume.id,
        }
        cmd = cmd_tpl % context

        with hide(*fab_output_hides):
            snapshot_result = sudo(cmd)
            if snapshot_result.return_code != 0:
                logger.critical("Error creating seed snapshot")
                logger.critical("Output: %s", snapshot_result)
                exit(1)

        # eg. snap-aaaabbbb
        assert len(str(snapshot_result)) == 13

        return str(snapshot_result)

    def _get_masterdb(self):
        """
        Get the rds masterdb node for this deployment.
        """
        if self.is_active:
            return self.deployment.get_active_node('rds', 'masterdb')
        else:
            return self.deployment.get_pending_node('rds', 'masterdb')

    def _attach_ebs_vols(self, ec2_instance_id, vol_confs, seed_ebs_snapshots):
        """
        Attach the appropriate EBS volumes to the ``ec2_instance`` and wait
        for them to complete.
        """
        require('aws_availability_zone')

        with hide(*fab_quiet):
            logger.info("Attaching EBS volumes from snapshots")
            attached_volumes = create_and_attach_ebs_vols(
                ec2conn=self.deployment.ec2conn,
                availability_zone=env.aws_availability_zone,
                instance_id=ec2_instance_id,
                vol_confs=vol_confs,
                seed_ebs_snapshots=seed_ebs_snapshots)

            env.attached_volumes = True

            return attached_volumes

    def _ensure_ebs_vols_mounted(self, ec2_instance_id, vol_confs):
        """
        Ensure that the EBS volumes are succesfully mounted and any required
        directory symlinks are in place.
        """
        self.provisioner.stop_services()

        self._update_fstab(vol_confs)

        with hide(*fab_quiet):
            for vol_label, vol_conf in vol_confs.items():
                self._ensure_ebs_vol_mounted(
                    ec2_instance_id,
                    vol_label,
                    vol_conf,
                )

    def _update_fstab(self, vol_configs):
        """
        Update /etc/fstab to include the configured EBS volumes and all of
        their bound directories.
        """
        # Load the /etc/fstab template and the ec2 defaults
        fstab_entries = []
        fstab_entries.extend(EC2_FSTAB_DEFAULTS)

        # Add entries for all attached volumes
        for vol_label, vol_conf in vol_configs.items():
            logger.info("Configuring fstab entries for %s", vol_label)
            # Configure the mount point for the whole volume
            fstab_entries.append(
                fstabEntry(
                    device_name=vol_conf['device'],
                    mount_point=vol_conf['mount_point'],
                    fs_type=vol_conf['filesystem'],
                    options='noatime',
                    dump_freq=0,
                    pass_num=0,
                )
            )

            # Configure all of the bound directories
            for ebs_bind_confs in vol_conf['ebs_bound_dirs']:
                # Add a directory bind line
                # eg. /vol/fs/var/log/pstat /var/log/pstat none bind 0 0
                fstab_entries.append(
                    fstabEntry(
                        device_name=ebs_bind_confs['src_path'],
                        mount_point=ebs_bind_confs['bind_path'],
                        fs_type='none',
                        options='bind',
                        dump_freq=0,
                        pass_num=0,
                    )
                )

        # Push the /etc/fstab file to the server
        with NamedTemporaryFile() as fstab_tpl_f:
            fstab_tpl_f.write(EC2_FSTAB_TPL)
            fstab_tpl_f.flush()
            context = {
                'fstab_entries': fstab_entries,
            }
            tpl_dir, tpl_name = os.path.split(fstab_tpl_f.name)
            with hide(*fab_output_hides):
                upload_template(
                    tpl_name,
                    '/etc/fstab',
                    context=context,
                    use_jinja=True,
                    template_dir=tpl_dir,
                    mode=644,
                    use_sudo=True,
                )

    def _ensure_ebs_vol_mounted(self, ec2_instance_id, vol_label, vol_config):
        """
        Ensure a single EBS volume is succesfully mounted and any required
        fstab directory binds are in place.
        """
        with hide(*fab_quiet):
            # Wait for the volume attachment to complete
            logger.info(
                "Waiting for EBS volume %s to attach to device %s",
                vol_label,
                vol_config['device']
            )

            # ``stat`` on the device until it exists
            stat_result = sudo('stat %s' % vol_config['device'])
            while stat_result.return_code != 0:
                logger.info(
                    "Attachment pending. Waiting %s seconds.",
                    LAUNCH_REFRESH,
                )
                time.sleep(LAUNCH_REFRESH)
                stat_result = sudo('stat %s' % vol_config['device'])

            # Ensure all of the EBS fstab binds are configured
            logger.info("Binding/Mounting EBS directories")

            if vol_config['filesystem'] != 'xfs':
                raise NotImplementedError(
                    "Only the xfs filesystem is implemented")

            # Mount the EBS volume
            sudo("mkdir %s --parent" % vol_config['mount_point'])
            with hide(*fab_quiet):
                sudo("mount %s" % vol_config['mount_point'])

            # Mount all of the bound folders
            for ebs_bind_confs in vol_config['ebs_bound_dirs']:
                src_path = ebs_bind_confs['src_path']
                bind_path = ebs_bind_confs['bind_path']
                sudo('mkdir %s --parent' % bind_path)
                sudo('mkdir %s --parent' % src_path)
                logger.info(
                    "Mounting bound dir %s to %s",
                    src_path,
                    bind_path,
                )
                mount_result = sudo('mount %s' % bind_path)
                if mount_result.failed:
                    logger.warning(
                        "Error mounting %s. Msg: %s",
                        bind_path,
                        mount_result)
                else:
                    # If the mount succeeded, optionally fix directory
                    # permissions and ownership
                    uid = ebs_bind_confs.get('uid', '')
                    gid = ebs_bind_confs.get('gid', '')
                    mode = ebs_bind_confs.get('mode', '')
                    if mode:
                        logger.info(
                            "Fixing directory mode on %s to %s",
                            bind_path,
                            mode,
                        )
                        sudo('chmod %s %s' % (mode, bind_path))
                    if uid or gid:
                        ownership = "%s:%s" % (uid, gid)
                        logger.info(
                            "Changing ownership of %s to %s",
                            bind_path,
                            ownership,
                        )
                        sudo('chown %s %s' % (ownership, bind_path))

            if vol_config['filesystem'] == 'xfs':
                # Grow the file systems if needed
                logger.info("Growing the file system")
                with hide(*fab_quiet):
                    sudo('xfs_growfs %s' % vol_config['mount_point'])

    def _configure_ec2tools(self, ec2conn, ec2_instance_id, vol_confs):
        """
        Configure the django-ec2tools configuration file for backing up
        EBS volumes.
        """
        logger.info("Configuring /root/.ec2tools.ini")
        config = ConfigParser.ConfigParser()
        config.add_section('volume_aliases')
        for vol_label in vol_confs.keys():
            config.set('volume_aliases', vol_label, vol_label)

        for vol_label, vol_conf in vol_confs.items():
            attached_vol = _get_attached_volume(
                ec2conn, ec2_instance_id, vol_conf['device'])

            config.add_section(vol_label)
            config.set(vol_label, 'volume_id', attached_vol.id)
            config.set(vol_label, 'mountpoint', vol_conf['mount_point'])

        with NamedTemporaryFile() as ec2_file:
            config.write(ec2_file)
            ec2_file.file.flush()
            with hide(*fab_output_hides):
                put(ec2_file.name, '/tmp/.ec2tools.ini')
                sudo('mv /tmp/.ec2tools.ini /root/')

    def _configure_ebs_backup(self, manage_dir, python_file, storage_dir):
        """
        Configure cronjob to perform EBS backup for the configured volumes
        using django-ec2tools.
        """
        logger.info("Configuring EBS snapshotting via django-ec2tools")
        minutes = ['10']
        user = 'root'

        cmd_dict = {
            'manage_dir': manage_dir,
            'python_file': python_file,
            'storage_dir': storage_dir,
            'log_dir': LOG_DIR,
        }
        cmd_tpl = 'cd %(manage_dir)s && \
        %(python_file)s manage.py ec2_take_snapshot \
        --config-file=/root/.ec2tools.ini fs >> \
        %(storage_dir)s/snapshot_backup.log 2>&1'
        cmd = cmd_tpl % cmd_dict

        cron_tpl = '%(time)s %(user)s %(cmd)s'
        time_tpl = '%(minutes)s %(hours)s * * %(dow)s'

        # Prime time backups
        hours = [str(n) for n in range(8, 18)]
        dow = ['mon', 'tue', 'wed', 'thu', 'fri']

        time_dict = {
            'minutes': ','.join(minutes),
            'hours': ','.join(hours),
            'dow': ','.join(dow),
        }
        time_conf = time_tpl % time_dict

        cron = cron_tpl % {'time': time_conf, 'user': user, 'cmd': cmd}
        fname = 'ebs_backup_prime'

        sudo('rm /etc/cron.d/%s' % fname)
        sudo('echo "%s" > /etc/cron.d/%s' % (cron, fname))

        # Off hour backups
        hours = ['4']
        dow = ['*']

        time_dict = {
            'minutes': ','.join(minutes),
            'hours': ','.join(hours),
            'dow': ','.join(dow)
        }
        time_conf = time_tpl % time_dict

        cron = cron_tpl % {'time': time_conf, 'user': user, 'cmd': cmd}
        fname = 'ebs_backup_off'

        sudo('rm /etc/cron.d/%s' % fname)
        sudo('echo "%s" > /etc/cron.d/%s' % (cron, fname))

        # Configure backup pruning
        cmd_tpl = (
            'cd %(manage_dir)s && '
            '%(python_file)s manage.py ec2_prune_snapshots '
            '--config-file=/root/.ec2tools.ini '
            '--all '
            '--strategy twoday_rolling_parent '
            '>> %(log_dir)s/snapshot_pruning.log 2>&1'
        )
        cmd = cmd_tpl % cmd_dict

        time_dict = {
            'minutes': 15,
            'hours': 3,
            'dow': '*',
        }
        time_conf = time_tpl % time_dict

        cron = cron_tpl % {'time': time_conf, 'user': user, 'cmd': cmd}
        fname = 'ebs_snapshot_pruning'

        sudo('rm /etc/cron.d/%s' % fname)
        sudo('echo "%s" > /etc/cron.d/%s' % (cron, fname))

        # Configure snapshot limit monitoring
        cmd_tpl = 'cd %(manage_dir)s && \
        %(python_file)s manage.py ec2_check_snapshot_limit >> \
        %(log_dir)s/snapshot_checker.log 2>&1'
        cmd = cmd_tpl % cmd_dict

        time_dict = {
            'minutes': 20,
            'hours': 3,
            'dow': '*',
        }
        time_conf = time_tpl % time_dict

        cron = cron_tpl % {'time': time_conf, 'user': user, 'cmd': cmd}
        fname = 'ebs_snapshot_checker'

        sudo('rm /etc/cron.d/%s' % fname)
        sudo('echo "%s" > /etc/cron.d/%s' % (cron, fname))

    def launch(self):
        """
        Launch a new EC2 instance

        Returns the boto ec2 instance.
        """
        require('aws_access_key_id')
        require('aws_secret_access_key')
        require('ami_id')
        require('aws_availability_zone')
        require('config_bucket')
        require('ec2_instance_type')

        ec2conn = self.deployment.ec2conn

        # Start a new ec2 instance and get its instance id
        reservation = ec2conn.run_instances(
            env.ami_id,
            key_name=self.config['aws']['keypair'],
            placement=env.aws_availability_zone,
            instance_type=env.ec2_instance_type,
            security_groups=env.aws_security_groups,
        )

        instance = reservation.instances[0]

        logging.info("Instance launching with id: %s", instance.id)

        return instance class RdsNodeDeployment(BaseNodeDeployment):
    """
    Deployment controller for an RDS database node.
    """

    def __init__(self, *args, **kwargs):
        super(RdsNodeDeployment, self).__init__(*args, **kwargs)

        self.seed_snapshot_id = None
        self.seed_master_password = None
        if self.seed_node:
            conf = self.seed_node._deployment_info['conf']
            self.seed_master_password = conf['rds_master_password']

        self._local_ip = None

    def get_ip(self):
        if self._local_ip:
            return self._local_ip

        max_retries = 10
        retries = 0
        ip = None
        while not ip:
            try:
                response = urllib.urlopen(IP_SERVICE_URL).read()
                match = re.search('\d+\.\d+\.\d+\.\d+', response)
                if match:
                    ip = match.group(0)
                else:
                    logger.error(
                        'Error getting IP from %s. Aborting' % IP_SERVICE_URL)
                    exit(1)
                logger.info("Your IP: %s", ip)
            except:
                if retries > max_retries:
                    raise
                retries += 1
                logger.warning(
                    "Error attempting to determine IP from %s",
                    IP_SERVICE_URL)
                time.sleep(5)

        self._local_ip = ip

        return self._local_ip

    def verify_seed_data(self, node, retries_remaining=3):
        """
        Ensure that the seed data you received is correct.
        """
        pass

    def get_seed_data(self):
        """
        Ensure that the seed node has a recent latest restorable time,
        otherwise allow the user to create a snapshot to restore from.
        """
        if self.seed_node and self.seed_verification:
            # TODO: Hook to do maintenance announcements, etc
            self._create_snapshot()
            return

        if self.seed_node:
            restoration_lag = self._get_restorable_lag()
            if restoration_lag > MAX_RESTORABLE_LAG:
                logger.critical("DB restoration lag: %s", restoration_lag)
                logger.critical("DB restoration lag too high.")
            else:
                return

        opts = ['F', 'S', 'E']
        action = None
        while action not in opts:
            action = prompt(
                "Create (S)napshot, use (E)xisting snapshot or (F)ail?")

        if action == 'S':
            self._create_snapshot()
        elif action == 'E':
            self.seed_snapshot_id = prompt("Enter snapshot id:")
        else:
            logger.critical("FAIL. DB restoration lag too high")
            exit(1)

    def _create_snapshot(self):
        instance = self.seed_node.boto_instance
        now = datetime.now()
        nowstr = now.strftime('%Y%m%d-%H%M%S')
        label = '%s-%sseed%s' % (
            self.deployment.deployment_name,
            self.node_name,
            nowstr,
        )
        restoration_snapshot = instance.snapshot(label)
        self.seed_snapshot_id = restoration_snapshot.id

    def _get_restorable_lag(self):
        """
        Get a timedelta representing the lag between now and the latest
        restorable time from the seed node.
        """
        seed_instance = self.seed_node.boto_instance
        latest_restorable_time = seed_instance.latest_restorable_time
        # Parse the time from the ISO 8601 string
        latest_restorable_time = dateutil.parser.parse(latest_restorable_time)

        restoration_lag = datetime.now(
            tzlocal()
        ) - latest_restorable_time

        return restoration_lag

    def create_new_node(self):
        """
        Launches and configures a new rds instance.
        """
        # Start up and connect to the Amazon RDS instance
        rds_label = str(self.deployment.get_new_rds_label(
            self.node_name, DEPLOYED_APP_VERSION))

        rds_instance = self.launch(rds_label)

        if self.is_active:
            self.deployment.set_active_node(
                'rds', self.node_name, rds_instance)
        else:
            self.deployment.set_pending_node(
                'rds', self.node_name, rds_instance)

        # Wait for node registration to complete
        time.sleep(2)

        # Doing this at the end to give the node-setting time to propogate
        return self.get_node()

    def launch(self, rds_label):
        """
        Launches an RDS instance with the given ``rds_label``. Takes in to
        account the seed node settings to start the database with the
        appropriate data.

        Returns the launched boto rds instance.
        """
        conf = self.deployment.deployment_confs['rds'][self.node_name]['conf']

        rds_label = str(rds_label)

        if self.seed_snapshot_id:
            logger.info(
                "Creating RDS instance: %s from snapshot: %s",
                rds_label,
                self.seed_snapshot_id,
            )

            # Wait for the snapshot to complete
            snapshot = self.deployment.rdsconn.get_all_dbsnapshots(
                snapshot_id=self.seed_snapshot_id)[0]
            while snapshot.status != 'available':
                logger.info(
                    "RDS Snapshot pending. Waiting %ss",
                    LAUNCH_REFRESH)
                time.sleep(LAUNCH_REFRESH)
                snapshot = self.deployment.rdsconn.get_all_dbsnapshots(
                    snapshot_id=self.seed_snapshot_id)[0]

            rdsconn = self.deployment.rdsconn
            db_instance = rdsconn.restore_dbinstance_from_dbsnapshot(
                identifier=snapshot.id,
                instance_id=rds_label,
                instance_class=conf['rds_instance_class'],
                availability_zone=conf['rds_availability_zone'])
        else:
            if self.seed_node:
                seed_instance = self.seed_node.boto_instance
                logger.info(
                    "Creating RDS instance: %s using PiT restore from: %s",
                    rds_label,
                    seed_instance.id)

                # Create using rdsconn.restore_db_instance_from_dbsnapshot
                rdsconn = self.deployment.rdsconn
                db_instance = rdsconn.restore_dbinstance_from_point_in_time(
                    source_instance_id=seed_instance.id,
                    target_instance_id=rds_label,
                    use_latest=True,
                    dbinstance_class=conf['rds_instance_class'],
                    availability_zone=conf['rds_availability_zone'],
                )

            else:
                # Creating a new, blank, DB
                logger.info("Creating new blank RDS instance: %s" % rds_label)
                db_instance = self.deployment.rdsconn.create_dbinstance(
                    id=rds_label,
                    allocated_storage=conf['rds_allocated_storage'],
                    instance_class=conf['rds_instance_class'],
                    master_username=conf['rds_master_username'],
                    master_password=conf['rds_master_password'],
                    security_groups=conf['rds_security_groups'],
                    availability_zone=conf['rds_availability_zone'],
                    preferred_maintenance_window=conf['rds_preferred_maintenance_window'],  # NOQA
                    backup_retention_period=conf['rds_backup_retention_period'],  # NOQA
                    preferred_backup_window=conf['rds_preferred_backup_window'],  # NOQA
                    multi_az=conf['rds_multi_az'])

        # Wait for the RDS instance request to actually appear
        return db_instance

    def _modify_db_config(self, node, conf, apply_immediately=False):
        """
        Modify the db parameters based on ``conf``.

        Returns ``True`` if a database restart was required.
        """
        node.refresh_boto_instance()

        logger.info("Modifying RDS DB parameters")
        # Need to modify the db to make sure all of the properties are set
        self.deployment.rdsconn.modify_dbinstance(
            id=node.boto_instance.id,
            allocated_storage=conf['rds_allocated_storage'],
            instance_class=conf['rds_instance_class'],
            master_password=conf['rds_master_password'],
            security_groups=conf['rds_security_groups'],
            preferred_maintenance_window=conf['rds_preferred_maintenance_window'],  # NOQA
            backup_retention_period=conf['rds_backup_retention_period'],
            preferred_backup_window=conf['rds_preferred_backup_window'],
            param_group=conf['rds_parameter_group'],
            multi_az=conf['rds_multi_az'],
            apply_immediately=apply_immediately)

        # Small sleep so that our changes have time to register
        time.sleep(3)
        # Wait for db modifications to complete
        self.wait_until_created(node)

        node.refresh_boto_instance()
        if node.boto_instance.pending_modified_values \
           and node.boto_instance.status != 'modifying':
            # Some modifications are still pending, do a hard restart since
            # that's required for MultiAZ
            logger.info(
                "DB has pending modified values and state of %s, a restart "
                "might be required",
                node.boto_instance.status)
            if self.is_active:
                # Don't allow restarting active nodes
                logger.warning("---")
                logger.warning(
                    "The DB node is live, and a DB restart might be required. "
                    "Please manually verify the database parameters"
                )
                logger.warning("---")
                return False

            node.boto_instance.reboot()
            return True

        return False

    def deploy(self, node, first_run=False):
        self.wait_until_created(node)
        conf = self.deployment.deployment_confs['rds'][self.node_name]['conf']

        # First need to make sure the parameter group is configured properly
        self.configure_parameter_group(
            conf['rds_parameter_group'],
            DB_PARAMETER_GROUPS[conf['rds_parameter_group']])
        self.wait_until_created(node)

        if first_run or self._parameters_differ(node, conf):
            # Only need to modify the instance on the first run
            required_restart = self._modify_db_config(
                node, conf, apply_immediately=first_run)

            if required_restart:
                # Small sleep so that our changes have time to register
                time.sleep(3)
                # Wait for db modifications to complete
                self.wait_until_created(node)

    def _parameters_differ(self, node, conf):
        """
        Determine whether the current database configuration differs from the
        configuration defined in the ``conf`` dictionary.
        """
        node.refresh_boto_instance()

        checked_params = [
            'rds_allocated_storage',
            'rds_instance_class',
            'rds_preferred_maintenance_window',
            'rds_backup_retention_period',
            'rds_preferred_backup_window',
            'rds_multi_az',
        ]

        groups_changed = False
        current_security_group = node.boto_instance.security_group.name
        if current_security_group not in conf['rds_security_groups']:
            logger.info(
                "param %s defined: %s actual: %s",
                'rds_security_groups',
                conf['rds_security_groups'],
                getattr(node.boto_instance, 'security_group').name)
            groups_changed = True
        current_parameter_group = node.boto_instance.parameter_group.name
        if current_parameter_group != conf['rds_parameter_group']:
            logger.info(
                "param %s defined: %s actual: %s",
                'rds_parameter_group',
                conf['rds_parameter_group'],
                getattr(node.boto_instance, 'parameter_group'))
            groups_changed = True

        diff_params = []
        for checked_param in checked_params:
            rds_param = checked_param.replace('rds_', '')
            defined_val = str(
                conf[checked_param]
            ).lower().strip()
            actual_val = str(
                getattr(node.boto_instance, rds_param)
            ).lower().strip()
            if defined_val != actual_val:
                diff_params.append(checked_param)

        if diff_params or groups_changed:
            for diff_param in diff_params:
                rds_param = diff_param.replace('rds_', '')
                defined_val = str(conf[diff_param]).lower().strip()
                actual_val = str(
                    getattr(node.boto_instance, rds_param)
                ).lower().strip()
                logger.info(
                    "param %s defined: %s actual: %s",
                    diff_param,
                    defined_val,
                    actual_val)

            return True

        logger.info("DB Parameters Already Synced")
        return False

    def wait_until_created(self, node):
        start_time = time.time()
        waited = False
        while not self.creation_complete(node):
            waited = True
            logger.info(
                "RDS DB still pending. Status: %s",
                node.boto_instance.status,
            )
            time.sleep(LAUNCH_REFRESH)
        if waited:
            logger.info("Waited %.1fs for DB", time.time() - start_time)

    def creation_complete(self, node):
        node.refresh_boto_instance()
        return node.boto_instance.status == 'available'

    def _get_all_db_parameters(self, group_name):
        """
        Create the named parameter group (if needed)
        and return a list of all the db parameters, accounting for request
        pagination.
        """
        rdsconn = self.deployment.rdsconn
        all_parameters = []
        try:
            all_parameters = rdsconn.get_all_dbparameters(
                groupname=group_name,
                max_records=100,
            )
        except BotoServerError:
            logger.info(
                "RDS Parameter Group %s doesn't exist. Creating it.",
                group_name,
            )
            # TODO: Detect the message `Invalid parameter group name:` Message
            # to ensure that this isn't some other kind of error
            rdsconn.create_parameter_group(name=group_name)
            all_parameters = rdsconn.get_all_dbparameters(
                groupname=group_name,
                max_records=100,
            )

        def _get_pagination_marker(parameter_group):
            # Determine if we got paginated results
            pagination_marker = None
            if hasattr(parameter_group, 'Marker'):
                pagination_marker = parameter_group.Marker

            return pagination_marker

        pagination_marker = _get_pagination_marker(all_parameters)

        while pagination_marker:
            additional_parameters = rdsconn.get_all_dbparameters(
                groupname=group_name,
                max_records=100,
                marker=pagination_marker,
            )
            all_parameters.update(additional_parameters)

            pagination_marker = _get_pagination_marker(additional_parameters)

        return all_parameters

    def _validate_desired_parameter_group_configuration(
        self,
        group_name,
        confs,
    ):
        """
        Ensure that we're using proper types for our desired parameters.
        """
        valid_types = rds.parametergroup.Parameter.ValidTypes
        for name, desired_value in confs.items():
            is_valid = False
            for type_method in valid_types.values():
                if isinstance(desired_value, type_method):
                    is_valid = True
                    break
            if not is_valid:
                logger.critical(
                    "Desired DB Parameter config error for group: %s",
                    group_name,
                )
                logger.critical(
                    "Desired value of '%s' for '%s' is an invalid type",
                )
                valid_type_names = ', '.join(valid_types.keys())
                logger.critical("Should be one of: %s", valid_type_names)
                exit(1)

    def configure_parameter_group(self, group_name, confs):
        """
        Configure the RDS parameter group given by ``group_name`` with a
        dict of ``confs`` with key => values for the parameters to modify.
        """
        self._validate_desired_parameter_group_configuration(
            group_name=group_name,
            confs=confs,
        )
        current_parameters = self._get_all_db_parameters(group_name)

        parameters_to_modify = []
        for parameter_name, desired_value in confs.items():
            try:
                current_parameter = current_parameters[parameter_name]
            except KeyError:
                logger.warning(
                    "RDS Parameter '%s' does not currently exist",
                    parameter_name,
                )
                current_parameter = rds.parametergroup.Parameter(
                    group=current_parameters,
                    name=parameter_name,
                )
                # Set the expected data type for validation
                # The default is string
                if isinstance(desired_value, int):
                    current_parameter.type = 'integer'
                elif isinstance(desired_value, bool):
                    current_parameter.type = 'boolean'
                elif isinstance(desired_value, str):
                    current_parameter.type = 'string'

            if current_parameter.type == str:
                # Work-around for a boto 1.9 Parameter.type initialization bug
                # Remove this when we drop support for boto 1.9
                current_parameter.type = 'string'
            if current_parameter.value != desired_value:
                logger.info(
                    "RDS Parameter Group %s requires change for %s",
                    group_name,
                    parameter_name,
                )
                logger.info("Current value: %s", current_parameter.value)
                logger.info("Desired value: %s", desired_value)
                current_parameter.value = desired_value
                current_parameter.apply_method = 'immediate'
                parameters_to_modify.append(current_parameter)

        if parameters_to_modify:
            logger.info("Modifying RDS Parameter Group: %s", group_name)
            self.deployment.rdsconn.modify_parameter_group(
                name=group_name,
                parameters=parameters_to_modify,
            )
import logging
import time
from datetime import datetime
from decorator import contextmanager
from fabric.api import env, task, prompt
from neckbeard.actions.contrib_hooks import (
    notifies_hipchat,
    _get_git_repo,
    _is_unchanged_from_head,
    _is_tagged_version,
    _push_tags,
    _take_temporary_pagerduty,
    DT_NOTIFY,
    _send_deployment_done_desktop_notification,
    _announce_deployment,
)
from neckbeard.actions.utils import (
    ACTIVE,
    logs_duration,
    prompt_on_exception,
)
from neckbeard.environment_manager import Deployment
from neckbeard.cloud_provisioners.aws import (
    Ec2NodeDeployment,
    RdsNodeDeployment,
)
            # Optionally require that we deploy from a tagged commit.
            if git_conf.get('require_tag', False):
                logger.info("Enforcing git tag requirement")
                if not _is_unchanged_from_head(repo):
                    logger.critical(
                        "Refusing to deploy, uncommitted changes exist.")
                    exit(1)
                if not _is_tagged_version(repo):
                    logger.critical(
                        "Refusing to deploy from an untagged commit.",
                    )
                    exit(1)
                _push_tags(repo)

        # TODO: Make this an optional hook that can be registered
        pagerduty_conf = {}
        if pagerduty_conf.get('temporarily_become_oncall', False):
            logger.info("Taking Pagerduty, temporarily")
            _take_temporary_pagerduty(
                duration=pagerduty_conf.get('temporary_oncall_duration'),
                api_key=pagerduty_conf.get('api_key'),
                user_id=pagerduty_conf.get('user_id'),
                project_subdomain=pagerduty_conf.get('project_subdomain'),
                schedule_key=pagerduty_conf.get('schedule_key'),
            )

    logger.info("Gathering deployment state")
    with logs_duration(timer, timer_name='gather deployment state'):
        environment_config = configuration_manager.get_environment_config(
            environment_name,
        )
        deployment = Deployment(
            environment_name,
            environment_config.get('ec2', {}),
            environment_config.get('rds', {}),
            environment_config.get('elb', {}),
        )
        # up never deals with old nodes, so just verify pending and active to
        # save HTTP round trips
        deployment.verify_deployment_state(verify_old=False)

    # Gather all of the configurations for each node, including their
    # seed deployment information
    logger.info("Gathering seed deployment state")
    with logs_duration(timer, timer_name='seed_deployment_state'):
        # If this environment has a seed environment, build that environment
        # manager
        seed_deployment = None
        seed_deployment_name = configuration_manager.get_seed_environment_name(
            environment_name,
        )
        if seed_deployment_name:
            seed_config = configuration_manager.get_environment_config(
                seed_deployment_name,
            )
            seed_deployment = Deployment(
                seed_deployment_name,
                seed_config.get('ec2', {}),
                seed_config.get('rds', {}),
                seed_config.get('elb', {}),
            )
            logger.info("Verifying seed deployment state")
            seed_deployment.verify_deployment_state(verify_old=False)

    # Build all of the deployment objects
    logger.info("Building Node deployers")
    with logs_duration(timer, timer_name='build deployers'):
        ec2_deployers = []
        rds_deployers = []

        # All rds and ec2 nodes, rds nodes first
        dep_confs = [
            (
                'rds',
                environment_config.get('ec2', {}),
            ),
            (
                'ec2',
                environment_config.get('rds', {}),
            ),
        ]

        for aws_type, node_confs in dep_confs:
            for node_name, conf in node_confs.items():
                # Get the seed deployment new instances will be copied from
                seed_node_name = None
                if seed_deployment and 'seed' in conf:
                    seed_node_name = conf['seed']['unique_id']
                    verify_seed_data = conf['seed_node'].get('verify', False)
                else:
                    logger.info("No seed node configured")
                    seed_node_name = None
                    verify_seed_data = False

                if aws_type == 'ec2':
                    klass = Ec2NodeDeployment
                elif aws_type == 'rds':
                    klass = RdsNodeDeployment

                deployer = klass(
                    deployment=deployment,
                    seed_deployment=seed_deployment,
                    is_active=env._active_gen,
                    aws_type=aws_type,
                    node_name=node_name,
                    seed_node_name=seed_node_name,
                    seed_verification=verify_seed_data,
                    brain_wrinkles=conf.get('brain_wrinkles', {}),
                    conf=conf,
                )

                if aws_type == 'ec2':
                    ec2_deployers.append(deployer)
                elif aws_type == 'rds':
                    rds_deployers.append(deployer)

    # We don't actually want to do deployments until we have tests
    assert False

    # Provision the RDS nodes

    with logs_duration(timer, timer_name='initial provision'):