class ChartBuilder(object):
    '''
    This class handles taking chart intentions as a parameter and turning those
    into proper ``protoc`` Helm charts that can be pushed to Tiller.
    '''

    def __init__(self, chart):
        '''Initialize the :class:`ChartBuilder` class.

        :param dict chart: The document containing all intentions to pass to
                           Tiller.
        '''

        # cache for generated protoc chart object
        self._helm_chart = None

        # store chart schema
        self.chart = chart

        # extract, pull, whatever the chart from its source
        self.source_directory = self.get_source_path()

        # load ignored files from .helmignore if present
        self.ignored_files = self.get_ignored_files()

    def get_source_path(self):
        '''Return the joined path of the source directory and subpath.

        Returns "<source directory>/<subpath>" taken from the "source_dir"
        property from the chart, or else "" if the property isn't a 2-tuple.
        '''
        source_dir = self.chart.get('source_dir')
        return (
            os.path.join(*source_dir)
            if (source_dir and
                isinstance(source_dir, (list, tuple)) and
                len(source_dir) == 2)
            else ""
        )

    def get_ignored_files(self):
        '''Load files to ignore from .helmignore if present.'''
        try:
            ignored_files = []
            if os.path.exists(os.path.join(self.source_directory,
                                           '.helmignore')):
                with open(os.path.join(self.source_directory,
                                       '.helmignore')) as f:
                    ignored_files = f.readlines()
            return [filename.strip() for filename in ignored_files]
        except Exception:
            raise chartbuilder_exceptions.IgnoredFilesLoadException()

    def ignore_file(self, filename):
        '''Returns whether a given ``filename`` should be ignored.

        :param filename: Filename to compare against list of ignored files.
        :returns: True if file matches an ignored file wildcard or exact name,
                  False otherwise.
        '''
        for ignored_file in self.ignored_files:
            if (ignored_file.startswith('*') and
                    filename.endswith(ignored_file.strip('*'))):
                return True
            elif ignored_file == filename:
                return True
        return False

    def get_metadata(self):
        '''Extract metadata from Chart.yaml to construct an instance of
        :class:`hapi.chart.metadata_pb2.Metadata`.
        '''
        try:
            with open(os.path.join(self.source_directory, 'Chart.yaml')) as f:
                chart_yaml = yaml.safe_load(f.read().encode('utf-8'))

        except Exception:
            raise chartbuilder_exceptions.MetadataLoadException()

        # Construct Metadata object.
        return Metadata(
            description=chart_yaml.get('description'),
            name=chart_yaml.get('name'),
            version=chart_yaml.get('version'))

    def get_files(self):
        '''
        Return (non-template) files in this chart.

        Non-template files include all files *except* Chart.yaml, values.yaml,
        values.toml, and any file nested under charts/ or templates/. The only
        exception to this rule is charts/.prov

        The class :class:`google.protobuf.any_pb2.Any` is wrapped around
        each file as that is what Helm uses.

        For more information, see:
        https://github.com/kubernetes/helm/blob/fa06dd176dbbc247b40950e38c09f978efecaecc/pkg/chartutil/load.go

        :returns: List of non-template files.
        :rtype: List[:class:`google.protobuf.any_pb2.Any`]
        '''

        files_to_ignore = ['Chart.yaml', 'values.yaml', 'values.toml']
        non_template_files = []

        def _append_file_to_result(root, rel_folder_path, file):
            abspath = os.path.abspath(os.path.join(root, file))
            relpath = os.path.join(rel_folder_path, file)

            encodings = ('utf-8', 'latin1')
            unicode_errors = []

            for encoding in encodings:
                try:
                    with open(abspath, 'r') as f:
                        file_contents = f.read().encode(encoding)
                except OSError as e:
                    LOG.debug('Failed to open and read file %s in the helm '
                              'chart directory.', abspath)
                    raise chartbuilder_exceptions.FilesLoadException(
                        file=abspath, details=e)
                except UnicodeError as e:
                    LOG.debug('Attempting to read %s using encoding %s.',
                              abspath, encoding)
                    msg = "(encoding=%s) %s" % (encoding, str(e))
                    unicode_errors.append(msg)
                else:
                    break

            if len(unicode_errors) == 2:
                LOG.debug('Failed to read file %s in the helm chart directory.'
                          ' Ensure that it is encoded using utf-8.', abspath)
                raise chartbuilder_exceptions.FilesLoadException(
                    file=abspath, clazz=unicode_errors[0].__class__.__name__,
                    details='\n'.join(e for e in unicode_errors))

            non_template_files.append(
                Any(type_url=relpath,
                    value=file_contents))

        for root, dirs, files in os.walk(self.source_directory):
            relfolder = os.path.split(root)[-1]
            rel_folder_path = os.path.relpath(root, self.source_directory)

            if not any(root.startswith(os.path.join(self.source_directory, x))
                       for x in ['templates', 'charts']):
                for file in files:
                    if (file not in files_to_ignore and
                            file not in non_template_files):
                        _append_file_to_result(root, rel_folder_path, file)
            elif relfolder == 'charts' and '.prov' in files:
                _append_file_to_result(root, rel_folder_path, '.prov')

        return non_template_files

    def get_values(self):
        '''Return the chart's (default) values.'''

        # create config object representing unmarshaled values.yaml
        if os.path.exists(os.path.join(self.source_directory, 'values.yaml')):
            with open(os.path.join(self.source_directory, 'values.yaml')) as f:
                raw_values = f.read()
        else:
            LOG.warn("No values.yaml in %s, using empty values",
                     self.source_directory)
            raw_values = ''

        return Config(raw=raw_values)

    def get_templates(self):
        '''Return all the chart templates.

        Process all files in templates/ as a template to attach to the chart,
        building a :class:`hapi.chart.template_pb2.Template` object.
        '''
        chart_name = self.chart.get('chart_name')
        templates = []
        if not os.path.exists(
                os.path.join(self.source_directory, 'templates')):
            LOG.warn("Chart %s has no templates directory. "
                     "No templates will be deployed", chart_name)
        for root, _, files in os.walk(
                os.path.join(self.source_directory, 'templates'),
                topdown=True):
            for tpl_file in files:
                tname = os.path.relpath(
                    os.path.join(root, tpl_file),
                    os.path.join(self.source_directory, 'templates'))
                if self.ignore_file(tname):
                    LOG.debug('Ignoring file %s', tname)
                    continue

                with open(os.path.join(root, tpl_file)) as f:
                    templates.append(
                        Template(name=tname, data=f.read().encode()))

        return templates

    def get_helm_chart(self):
        '''Return a Helm chart object.

        Constructs a :class:`hapi.chart.chart_pb2.Chart` object from the
        ``chart`` intentions, including all dependencies.
        '''
        if self._helm_chart:
            return self._helm_chart

        dependencies = []
        chart_dependencies = self.chart.get('dependencies', [])
        chart_name = self.chart.get('chart_name', None)
        chart_release = self.chart.get('release', None)
        for dep in chart_dependencies:
            dep_chart = dep.get('chart', {})
            dep_chart_name = dep_chart.get('chart_name', None)
            LOG.info("Building dependency chart %s for release %s.",
                     dep_chart_name, chart_release)
            try:
                dependencies.append(ChartBuilder(dep_chart).get_helm_chart())
            except Exception:
                raise chartbuilder_exceptions.DependencyException(chart_name)

        try:
            helm_chart = Chart(
                metadata=self.get_metadata(),
                templates=self.get_templates(),
                dependencies=dependencies,
                values=self.get_values(),
                files=self.get_files())
        except Exception as e:
            raise chartbuilder_exceptions.HelmChartBuildException(
                chart_name, details=e)

        self._helm_chart = helm_chart
        return helm_chart

    def dump(self):
        '''Dumps a chart object as a serialized string so that we can perform a
        diff.

        It recurses into dependencies.
        '''
        return self.get_helm_chart().SerializeToString() class Manifest(object):

    def __init__(self, documents, target_manifest=None):
        """Instantiates a Manifest object.

        An Armada Manifest expects that at least one of each of the following
        be included in ``documents``:

        * A document with schema "armada/Chart/v1"
        * A document with schema "armada/ChartGroup/v1"

        And only one document of the following is allowed:

        * A document with schema "armada/Manifest/v1"

        If multiple documents with schema "armada/Manifest/v1" are provided,
        specify ``target_manifest`` to select the target one.

        :param List[dict] documents: Documents out of which to build the
            Armada Manifest.
        :param str target_manifest: The target manifest to use when multiple
            documents with "armada/Manifest/v1" are contained in
            ``documents``. Default is None.
        :raises ManifestException: If the expected number of document types
            are not found or if the document types are missing required
            properties.
        """
        self.documents = deepcopy(documents)
        self.charts, self.groups, manifests = self._find_documents(
            target_manifest)

        if len(manifests) > 1:
            error = ('Multiple manifests are not supported. Ensure that the '
                     '`target_manifest` option is set to specify the target '
                     'manifest')
            LOG.error(error)
            raise exceptions.ManifestException(details=error)
        else:
            self.manifest = manifests[0] if manifests else None

        if not all([self.charts, self.groups, self.manifest]):
            expected_schemas = [const.DOCUMENT_CHART, const.DOCUMENT_GROUP]
            error = ('Documents must be a list of documents with at least one '
                     'of each of the following schemas: %s and only one '
                     'manifest' % expected_schemas)
            LOG.error(error)
            raise exceptions.ManifestException(details=error)

    def _find_documents(self, target_manifest=None):
        """Returns the chart documents, chart group documents,
        and Armada manifest

        If multiple documents with schema "armada/Manifest/v1" are provided,
        specify ``target_manifest`` to select the target one.

        :param str target_manifest: The target manifest to use when multiple
            documents with "armada/Manifest/v1" are contained in
            ``documents``. Default is None.
        :returns: Tuple of chart documents, chart groups, and manifests
            found in ``self.documents``
        :rtype: tuple
        """
        charts = []
        groups = []
        manifests = []
        for document in self.documents:
            if document.get('schema') == const.DOCUMENT_CHART:
                charts.append(document)
            if document.get('schema') == const.DOCUMENT_GROUP:
                groups.append(document)
            if document.get('schema') == const.DOCUMENT_MANIFEST:
                manifest_name = document.get('metadata', {}).get('name')
                if target_manifest:
                    if manifest_name == target_manifest:
                        manifests.append(document)
                else:
                    manifests.append(document)
        return charts, groups, manifests

    def find_chart_document(self, name):
        """Returns a chart document with the specified name

        :param str name: name of the desired chart document
        :returns: The requested chart document
        :rtype: dict
        :raises ManifestException: If a chart document with the
            specified name is not found
        """
        for chart in self.charts:
            if chart.get('metadata', {}).get('name') == name:
                return chart
        raise exceptions.ManifestException(
            details='Could not find a {} named "{}"'.format(
                const.DOCUMENT_CHART, name))

    def find_chart_group_document(self, name):
        """Returns a chart group document with the specified name

        :param str name: name of the desired chart group document
        :returns: The requested chart group document
        :rtype: dict
        :raises ManifestException: If a chart
            group document with the specified name is not found
        """
        for group in self.groups:
            if group.get('metadata', {}).get('name') == name:
                return group
        raise exceptions.ManifestException(
            details='Could not find a {} named "{}"'.format(
                const.DOCUMENT_GROUP, name))

    def build_charts_deps(self):
        """Build chart dependencies for every ``chart``.

        :returns: None
        """
        for chart in self.charts:
            self.build_chart_deps(chart)

    def build_chart_groups(self):
        """Build chart dependencies for every ``chart_group``.

        :returns: None
        """
        for chart_group in self.groups:
            self.build_chart_group(chart_group)

    def build_chart_deps(self, chart):
        """Recursively build chart dependencies for ``chart``.

        :param dict chart: The chart whose dependencies will be recursively
            built.
        :returns: The chart with all dependencies.
        :rtype: dict
        :raises ManifestException: If a chart for a dependency name listed
            under ``chart['data']['dependencies']`` could not be found.
        """
        try:
            dep = None
            chart_dependencies = chart.get('data', {}).get('dependencies', [])
            for iter, dep in enumerate(chart_dependencies):
                if isinstance(dep, dict):
                    continue
                chart_dep = self.find_chart_document(dep)
                self.build_chart_deps(chart_dep)
                chart['data']['dependencies'][iter] = {
                    'chart': chart_dep.get('data', {})
                }
        except Exception:
            raise exceptions.ManifestException(
                details="Could not find dependency chart {} in {}".format(
                    dep, const.DOCUMENT_CHART))
        else:
            return chart

    def build_chart_group(self, chart_group):
        """Builds the chart dependencies for`charts`chart group``.

        :param dict chart_group: The chart_group whose dependencies
            will be built.
        :returns: The chart_group with all dependencies.
        :rtype: dict
        :raises ManifestException: If a chart for a dependency name listed
            under ``chart_group['data']['chart_group']`` could not be found.
        """
        try:
            chart = None
            for iter, chart in enumerate(chart_group.get('data', {}).get(
                    'chart_group', [])):
                if isinstance(chart, dict):
                    continue
                chart_dep = self.find_chart_document(chart)
                chart_group['data']['chart_group'][iter] = {
                    'chart': chart_dep.get('data', {})
                }
        except Exception:
            raise exceptions.ManifestException(
                details="Could not find chart {} in {}".format(
                    chart, const.DOCUMENT_GROUP))
        else:
            return chart_group

    def build_armada_manifest(self):
        """Builds the Armada manifest while pulling out data
        from the chart_group.

        :returns: The Armada manifest with the data of the chart groups.
        :rtype: dict
        :raises ManifestException: If a chart group's data listed
            under ``chart_group['data']`` could not be found.
        """
        try:
            group = None
            for iter, group in enumerate(self.manifest.get('data', {}).get(
                    'chart_groups', [])):
                if isinstance(group, dict):
                    continue
                chart_grp = self.find_chart_group_document(group)

                # Add name to chart group
                ch_grp_data = chart_grp.get('data', {})
                ch_grp_data['name'] = chart_grp.get('metadata', {}).get('name')

                self.manifest['data']['chart_groups'][iter] = ch_grp_data
        except Exception:
            raise exceptions.ManifestException(
                "Could not find chart group {} in {}".format(
                    group, const.DOCUMENT_MANIFEST))
        else:
            return self.manifest

    def get_manifest(self):
        """Builds all of the documents including the dependencies of the
        chart documents, the charts in the chart_groups, and the
        Armada manifest

        :returns: The Armada manifest.
        :rtype: dict
        """
        self.build_charts_deps()
        self.build_chart_groups()
        self.build_armada_manifest()

        return {
            'armada': self.manifest.get('data', {})
        } class Override(object):
    def __init__(self, documents, overrides=None, values=None):
        self.documents = documents
        self.overrides = overrides
        self.values = values

    def _load_yaml_file(self, doc):
        '''
        Retrieve yaml file as a dictionary.
        '''
        try:
            with open(doc) as f:
                return list(yaml.safe_load_all(f.read()))
        except IOError:
            raise override_exceptions.InvalidOverrideFileException(doc)

    def update(self, d, u):
        for k, v in u.items():
            if isinstance(v, collections.Mapping):
                r = self.update(d.get(k, {}), v)
                d[k] = r
            elif isinstance(v, str) and isinstance(d.get(k), (list, tuple)):
                d[k] = [x.strip() for x in v.split(',')]
            else:
                d[k] = u[k]
        return d

    def find_document_type(self, alias):
        if alias == 'chart_group':
            return const.DOCUMENT_GROUP
        if alias == 'chart':
            return const.DOCUMENT_CHART
        if alias == 'manifest':
            return const.DOCUMENT_MANIFEST
        else:
            raise ValueError("Could not find {} document".format(alias))

    def find_manifest_document(self, doc_path):
        for doc in self.documents:
            if doc.get('schema') == self.find_document_type(
                doc_path[0]) and doc.get('metadata', {}).get(
                    'name') == doc_path[1]:
                return doc

        raise override_exceptions.UnknownDocumentOverrideException(
            doc_path[0], doc_path[1])

    def array_to_dict(self, data_path, new_value):
        # TODO(fmontei): Handle `json.decoder.JSONDecodeError` getting thrown
        # better.
        def convert(data):
            if isinstance(data, str):
                return str(data)
            elif isinstance(data, collections.Mapping):
                return dict(map(convert, data.items()))
            elif isinstance(data, collections.Iterable):
                return type(data)(map(convert, data))
            else:
                return data

        if not new_value:
            return

        if not data_path:
            return

        tree = {}

        t = tree
        for part in data_path:
            if part == data_path[-1]:
                t.setdefault(part, None)
                continue
            t = t.setdefault(part, {})

        string = json.dumps(tree).replace('null', '"{}"'.format(new_value))
        data_obj = convert(json.loads(string, encoding='utf-8'))

        return data_obj

    def override_manifest_value(self, doc_path, data_path, new_value):
        document = self.find_manifest_document(doc_path)
        new_data = self.array_to_dict(data_path, new_value)
        self.update(document.get('data', {}), new_data)

    def update_document(self, merging_values):
        for doc in merging_values:
            if doc.get('schema') == const.DOCUMENT_CHART:
                self.update_chart_document(doc)
            if doc.get('schema') == const.DOCUMENT_GROUP:
                self.update_chart_group_document(doc)
            if doc.get('schema') == const.DOCUMENT_MANIFEST:
                self.update_armada_manifest(doc)

    def update_chart_document(self, ovr):
        for doc in self.documents:
            if doc.get('schema') == const.DOCUMENT_CHART and doc.get(
                    'metadata', {}).get('name') == ovr.get('metadata', {}).get(
                        'name'):
                self.update(doc.get('data', {}), ovr.get('data', {}))
                return

    def update_chart_group_document(self, ovr):
        for doc in self.documents:
            if doc.get('schema') == const.DOCUMENT_GROUP and doc.get(
                    'metadata', {}).get('name') == ovr.get('metadata', {}).get(
                        'name'):
                self.update(doc.get('data', {}), ovr.get('data', {}))
                return

    def update_armada_manifest(self, ovr):
        for doc in self.documents:
            if doc.get('schema') == const.DOCUMENT_MANIFEST and doc.get(
                    'metadata', {}).get('name') == ovr.get('metadata', {}).get(
                        'name'):
                self.update(doc.get('data', {}), ovr.get('data', {}))
                return

    def update_manifests(self):

        if self.values:
            for value in self.values:
                merging_values = self._load_yaml_file(value)
                self.update_document(merging_values)
        if self.overrides:
            for override in self.overrides:
                new_value = override.split('=')[1]
                doc_path = override.split('=')[0].split(":")
                data_path = doc_path.pop().split('.')

                self.override_manifest_value(doc_path, data_path, new_value)

        try:
            validate.validate_armada_documents(self.documents)
        except Exception:
            raise override_exceptions.InvalidOverrideValueException(
                self.overrides)

        return self.documents class Tiller(object):
    '''
    The Tiller class supports communication and requests to the Tiller Helm
    service over gRPC
    '''

    def __init__(self, tiller_host=None, tiller_port=None,
                 tiller_namespace=None):
        self.tiller_host = tiller_host
        self.tiller_port = tiller_port or CONF.tiller_port
        self.tiller_namespace = tiller_namespace or CONF.tiller_namespace
        # init k8s connectivity
        self.k8s = K8s()

        # init Tiller channel
        self.channel = self.get_channel()

        # init timeout for all requests
        # and assume eventually this will
        # be fed at runtime as an override
        self.timeout = TILLER_TIMEOUT

        LOG.debug('Armada is using Tiller at: %s:%s, namespace=%s, timeout=%s',
                  self.tiller_host, self.tiller_port, self.tiller_namespace,
                  self.timeout)

    @property
    def metadata(self):
        '''
        Return Tiller metadata for requests
        '''
        return [(b'x-helm-api-client', TILLER_VERSION)]

    def get_channel(self):
        '''
        Return a Tiller channel
        '''
        tiller_ip = self._get_tiller_ip()
        tiller_port = self._get_tiller_port()
        try:
            LOG.debug('Tiller getting gRPC insecure channel at %s:%s '
                      'with options: [grpc.max_send_message_length=%s, '
                      'grpc.max_receive_message_length=%s]',
                      tiller_ip, tiller_port,
                      MAX_MESSAGE_LENGTH, MAX_MESSAGE_LENGTH)
            return grpc.insecure_channel(
                '%s:%s' % (tiller_ip, tiller_port),
                options=[
                    ('grpc.max_send_message_length', MAX_MESSAGE_LENGTH),
                    ('grpc.max_receive_message_length', MAX_MESSAGE_LENGTH)
                ]
            )
        except Exception:
            raise ex.ChannelException()

    def _get_tiller_pod(self):
        '''
        Returns Tiller pod using the Tiller pod labels specified in the Armada
        config
        '''
        pods = None
        namespace = self._get_tiller_namespace()
        pods = self.k8s.get_namespace_pod(namespace,
                                          CONF.tiller_pod_labels).items
        # No Tiller pods found
        if not pods:
            raise ex.TillerPodNotFoundException(CONF.tiller_pod_labels)

        # Return first Tiller pod in running state
        for pod in pods:
            if pod.status.phase == 'Running':
                LOG.debug('Found at least one Running Tiller pod.')
                return pod

        # No Tiller pod found in running state
        raise ex.TillerPodNotRunningException()

    def _get_tiller_ip(self):
        '''
        Returns the Tiller pod's IP address by searching all namespaces
        '''
        if self.tiller_host:
            LOG.debug('Using Tiller host IP: %s', self.tiller_host)
            return self.tiller_host
        else:
            pod = self._get_tiller_pod()
            LOG.debug('Using Tiller pod IP: %s', pod.status.pod_ip)
            return pod.status.pod_ip

    def _get_tiller_port(self):
        '''Stub method to support arbitrary ports in the future'''
        LOG.debug('Using Tiller host port: %s', self.tiller_port)
        return self.tiller_port

    def _get_tiller_namespace(self):
        LOG.debug('Using Tiller namespace: %s', self.tiller_namespace)
        return self.tiller_namespace

    def tiller_status(self):
        '''
        return if Tiller exist or not
        '''
        if self._get_tiller_ip():
            LOG.debug('Getting Tiller Status: Tiller exists')
            return True

        LOG.debug('Getting Tiller Status: Tiller does not exist')
        return False

    def list_releases(self):
        '''
        List Helm Releases
        '''
        # TODO(MarshM possibly combine list_releases() with list_charts()
        # since they do the same thing, grouping output differently
        releases = []
        stub = ReleaseServiceStub(self.channel)
        # TODO(mark-burnett): Since we're limiting page size, we need to
        # iterate through all the pages when collecting this list.
        # NOTE(MarshM): `Helm List` defaults to returning Deployed and Failed,
        # but this might not be a desireable ListReleasesRequest default.
        req = ListReleasesRequest(limit=RELEASE_LIMIT,
                                  status_codes=[STATUS_DEPLOYED,
                                                STATUS_FAILED],
                                  sort_by='LAST_RELEASED',
                                  sort_order='DESC')

        LOG.debug('Tiller ListReleases() with timeout=%s', self.timeout)
        release_list = stub.ListReleases(req, self.timeout,
                                         metadata=self.metadata)

        for y in release_list:
            # TODO(MarshM) this log is too noisy, fix later
            # LOG.debug('Found release: %s', y.releases
            releases.extend(y.releases)

        return releases

    def get_chart_templates(self, template_name, name, release_name, namespace,
                            chart, disable_hooks, values):
        # returns some info

        LOG.info("Template( %s ) : %s ", template_name, name)

        stub = ReleaseServiceStub(self.channel)
        release_request = InstallReleaseRequest(
            chart=chart,
            dry_run=True,
            values=values,
            name=name,
            namespace=namespace,
            wait=False)

        templates = stub.InstallRelease(
            release_request, self.timeout, metadata=self.metadata)

        for template in yaml.load_all(
                getattr(templates.release, 'manifest', [])):
            if template_name == template.get('metadata', None).get(
                    'name', None):
                LOG.info(template_name)
                return template

    def _pre_update_actions(self, actions, release_name, namespace, chart,
                            disable_hooks, values, timeout):
        '''
        :params actions - array of items actions
        :params namespace - name of pod for actions
        '''

        try:
            for action in actions.get('update', []):
                name = action.get('name')
                LOG.info('Updating %s ', name)
                action_type = action.get('type')
                labels = action.get('labels')

                self.rolling_upgrade_pod_deployment(
                    name, release_name, namespace, labels,
                    action_type, chart, disable_hooks, values, timeout)
        except Exception:
            LOG.warn("Pre: Could not update anything, please check yaml")
            raise ex.PreUpdateJobDeleteException(name, namespace)

        try:
            for action in actions.get('delete', []):
                name = action.get('name')
                action_type = action.get('type')
                labels = action.get('labels', None)

                self.delete_resources(release_name, name, action_type,
                                      labels, namespace, timeout)
        except Exception:
            LOG.warn("PRE: Could not delete anything, please check yaml")
            raise ex.PreUpdateJobDeleteException(name, namespace)

        try:
            for action in actions.get('create', []):
                name = action.get("name")
                action_type = action.get("type")
                if "job" in action_type:
                    LOG.info("Creating %s in namespace: %s", name, namespace)
                    # TODO(MarshM) create_job_action does nothing but LOG.debug
                    self.k8s.create_job_action(name, action_type)
                    continue
        except Exception:
            LOG.warn("PRE: Could not create anything, please check yaml")
            raise ex.PreUpdateJobCreateException(name, namespace)

    def _post_update_actions(self, actions, namespace):
        try:
            for action in actions.get('create', []):
                name = action.get("name")
                action_type = action.get("type")
                if "job" in action_type:
                    LOG.info("Creating %s in namespace: %s", name, namespace)
                    # TODO(MarshM) create_job_action does nothing but LOG.debug
                    self.k8s.create_job_action(name, action_type)
                    continue
        except Exception:
            LOG.warn("POST: Could not create anything, please check yaml")
            raise ex.PreUpdateJobCreateException(name, namespace)

    def list_charts(self):
        '''
        List Helm Charts from Latest Releases

        Returns a list of tuples in the form:
        (name, version, chart, values, status)
        '''
        LOG.debug('Getting known releases from Tiller...')
        charts = []
        for latest_release in self.list_releases():
            try:
                release = (
                    latest_release.name, latest_release.version,
                    latest_release.chart, latest_release.config.raw,
                    latest_release.info.status.Code.Name(
                        latest_release.info.status.code))
                charts.append(release)
                LOG.debug('Found release %s, version %s, status: %s',
                          release[0], release[1], release[4])
            except (AttributeError, IndexError) as e:
                LOG.debug('%s while getting releases: %s, ex=%s',
                          e.__class__.__name__, latest_release, e)
                continue
        return charts

    def update_release(self, chart, release, namespace,
                       dry_run=False,
                       pre_actions=None,
                       post_actions=None,
                       disable_hooks=False,
                       values=None,
                       wait=False,
                       timeout=None):
        '''
        Update a Helm Release
        '''

        rel_timeout = self.timeout if not timeout else timeout

        LOG.debug('Helm update release%s: wait=%s, timeout=%s',
                  (' (dry run)' if dry_run else ''),
                  wait, timeout)

        if values is None:
            values = Config(raw='')
        else:
            values = Config(raw=values)

        self._pre_update_actions(pre_actions, release, namespace, chart,
                                 disable_hooks, values, timeout)

        # build release install request
        try:
            stub = ReleaseServiceStub(self.channel)
            release_request = UpdateReleaseRequest(
                chart=chart,
                dry_run=dry_run,
                disable_hooks=disable_hooks,
                values=values,
                name=release,
                wait=wait,
                timeout=timeout)

            update_msg = stub.UpdateRelease(
                release_request, rel_timeout + GRPC_EPSILON,
                metadata=self.metadata)

            tiller_result = TillerResult(
                update_msg.release.name,
                update_msg.release.namespace,
                update_msg.release.info.status.Code.Name(
                    update_msg.release.info.status.code),
                update_msg.release.info.Description,
                update_msg.release.version)

            return tiller_result
        except Exception:
            LOG.exception('Error while updating release %s', release)
            status = self.get_release_status(release)
            raise ex.ReleaseException(release, status, 'Upgrade')

        self._post_update_actions(post_actions, namespace)

    def install_release(self, chart, release, namespace,
                        dry_run=False,
                        values=None,
                        wait=False,
                        timeout=None):
        '''
        Create a Helm Release
        '''

        rel_timeout = self.timeout if not timeout else timeout

        LOG.debug('Helm install release%s: wait=%s, timeout=%s',
                  (' (dry run)' if dry_run else ''),
                  wait, timeout)

        if values is None:
            values = Config(raw='')
        else:
            values = Config(raw=values)

        # build release install request
        try:
            stub = ReleaseServiceStub(self.channel)
            release_request = InstallReleaseRequest(
                chart=chart,
                dry_run=dry_run,
                values=values,
                name=release,
                namespace=namespace,
                wait=wait,
                timeout=timeout)

            install_msg = stub.InstallRelease(
                release_request, rel_timeout + GRPC_EPSILON,
                metadata=self.metadata)

            tiller_result = TillerResult(
                install_msg.release.name,
                install_msg.release.namespace,
                install_msg.release.info.status.Code.Name(
                    install_msg.release.info.status.code),
                install_msg.release.info.Description,
                install_msg.release.version)

            return tiller_result
        except Exception:
            LOG.exception('Error while installing release %s', release)
            status = self.get_release_status(release)
            raise ex.ReleaseException(release, status, 'Install')

    def testing_release(self, release, timeout=300, cleanup=True):
        '''
        :param release - name of release to test
        :param timeout - runtime before exiting
        :param cleanup - removes testing pod created

        :returns - results of test pod
        '''

        LOG.debug("Helm test release %s, timeout=%s", release, timeout)

        try:

            stub = ReleaseServiceStub(self.channel)

            release_request = TestReleaseRequest(
                name=release, timeout=timeout, cleanup=cleanup)

            content = self.get_release_content(release)

            if not len(content.release.hooks):
                LOG.info('No test found')
                return False

            if content.release.hooks[0].events[0] == RUNTEST_SUCCESS:
                test = stub.RunReleaseTest(
                    release_request, self.timeout, metadata=self.metadata)

                if test.running():
                    self.k8s.wait_get_completed_podphase(release)

                test.cancel()

                return self.get_release_status(release)

        except Exception:
            LOG.exception('Error while testing release %s', release)
            status = self.get_release_status(release)
            raise ex.ReleaseException(release, status, 'Test')

    def get_release_status(self, release, version=0):
        '''
        :param release - name of release to test
        :param version - version of release status
        '''

        LOG.debug('Helm getting release status for release=%s, version=%s',
                  release, version)
        try:
            stub = ReleaseServiceStub(self.channel)
            status_request = GetReleaseStatusRequest(
                name=release, version=version)

            release_status = stub.GetReleaseStatus(
                status_request, self.timeout, metadata=self.metadata)
            LOG.debug('GetReleaseStatus= %s', release_status)
            return release_status

        except Exception:
            raise ex.GetReleaseStatusException(release, version)

    def get_release_content(self, release, version=0):
        '''
        :param release - name of release to test
        :param version - version of release status
        '''

        LOG.debug('Helm getting release content for release=%s, version=%s',
                  release, version)
        try:
            stub = ReleaseServiceStub(self.channel)
            status_request = GetReleaseContentRequest(
                name=release, version=version)

            release_content = stub.GetReleaseContent(
                status_request, self.timeout, metadata=self.metadata)
            LOG.debug('GetReleaseContent= %s', release_content)
            return release_content

        except Exception:
            raise ex.GetReleaseContentException(release, version)

    def tiller_version(self):
        '''
        :returns - Tiller version
        '''
        try:
            stub = ReleaseServiceStub(self.channel)
            release_request = GetVersionRequest()

            LOG.debug('Getting Tiller version, with timeout=%s', self.timeout)
            tiller_version = stub.GetVersion(
                release_request, self.timeout, metadata=self.metadata)

            tiller_version = getattr(tiller_version.Version, 'sem_ver', None)
            LOG.debug('Got Tiller version %s', tiller_version)
            return tiller_version

        except Exception:
            LOG.debug('Failed to get Tiller version')
            raise ex.TillerVersionException()

    def uninstall_release(self, release, disable_hooks=False, purge=True):
        '''
        :params - release - Helm chart release name
        :params - purge - deep delete of chart

        deletes a Helm chart from Tiller
        '''

        # build release install request
        try:
            stub = ReleaseServiceStub(self.channel)
            LOG.info("Uninstall %s release with disable_hooks=%s, "
                     "purge=%s flags", release, disable_hooks, purge)
            release_request = UninstallReleaseRequest(
                name=release, disable_hooks=disable_hooks, purge=purge)

            return stub.UninstallRelease(
                release_request, self.timeout, metadata=self.metadata)

        except Exception:
            LOG.exception('Error while uninstalling release %s', release)
            status = self.get_release_status(release)
            raise ex.ReleaseException(release, status, 'Delete')

    def chart_cleanup(self, prefix, charts):
        '''
        :params charts - list of yaml charts
        :params known_release - list of releases in tiller

        :result - will remove any chart that is not present in yaml
        '''

        valid_charts = []
        for gchart in charts:
            for chart in gchart.get('chart_group'):
                valid_charts.append(release_prefix(
                    prefix, chart.get('chart').get('name')))

        actual_charts = [x.name for x in self.list_releases()]
        chart_diff = list(set(actual_charts) - set(valid_charts))

        for chart in chart_diff:
            if chart.startswith(prefix):
                LOG.debug("Release: %s will be removed", chart)
                self.uninstall_release(chart)

    def delete_resources(self, release_name, resource_name, resource_type,
                         resource_labels, namespace, wait=False,
                         timeout=TILLER_TIMEOUT):
        '''
        :params release_name - release name the specified resource is under
        :params resource_name - name of specific resource
        :params resource_type - type of resource e.g. job, pod, etc.
        :params resource_labels - labels by which to identify the resource
        :params namespace - namespace of the resource

        Apply deletion logic based on type of resource
        '''

        label_selector = ''
        if resource_labels is not None:
            label_selector = label_selectors(resource_labels)
        LOG.debug("Deleting resources in namespace %s matching "
                  "selectors %s.", namespace, label_selector)

        handled = False
        if resource_type == 'job':
            get_jobs = self.k8s.get_namespace_job(namespace, label_selector)
            for jb in get_jobs.items:
                jb_name = jb.metadata.name
                LOG.info("Deleting job %s in namespace: %s",
                         jb_name, namespace)
                self.k8s.delete_job_action(jb_name, namespace, timeout=timeout)
            handled = True

        if resource_type == 'cronjob' or resource_type == 'job':
            get_jobs = self.k8s.get_namespace_cron_job(
                namespace, label_selector)
            for jb in get_jobs.items:
                jb_name = jb.metadata.name
                LOG.info("Deleting cron job %s in namespace: %s",
                         jb_name, namespace)
                if resource_type == 'job':
                    # TODO: Eventually disallow this, allowing initially since
                    #       some existing clients were expecting this behavior.
                    LOG.warning("Deleting cron jobs via `type: job` is "
                                "deprecated, use `type: cronjob` instead")
                self.k8s.delete_cron_job_action(jb_name, namespace)
            handled = True

        if resource_type == 'pod':
            release_pods = self.k8s.get_namespace_pod(
                namespace, label_selector)

            for pod in release_pods.items:
                pod_name = pod.metadata.name
                LOG.info("Deleting pod %s in namespace: %s",
                         pod_name, namespace)
                self.k8s.delete_namespace_pod(pod_name, namespace)
                if wait:
                    self.k8s.wait_for_pod_redeployment(pod_name, namespace)
            handled = True

        if not handled:
            LOG.error("Unable to execute name: %s type: %s ",
                      resource_name, resource_type)

    def rolling_upgrade_pod_deployment(self, name, release_name, namespace,
                                       resource_labels, action_type, chart,
                                       disable_hooks, values,
                                       timeout=TILLER_TIMEOUT):
        '''
        update statefullsets (daemon, stateful)
        '''

        if action_type == 'daemonset':

            LOG.info('Updating: %s', action_type)

            label_selector = ''

            if resource_labels is not None:
                label_selector = label_selectors(resource_labels)

            get_daemonset = self.k8s.get_namespace_daemonset(
                namespace=namespace, label=label_selector)

            for ds in get_daemonset.items:
                ds_name = ds.metadata.name
                ds_labels = ds.metadata.labels
                if ds_name == name:
                    LOG.info("Deleting %s : %s in %s", action_type, ds_name,
                             namespace)
                    self.k8s.delete_daemon_action(ds_name, namespace)

                    # update the daemonset yaml
                    template = self.get_chart_templates(
                        ds_name, name, release_name, namespace, chart,
                        disable_hooks, values)
                    template['metadata']['labels'] = ds_labels
                    template['spec']['template']['metadata'][
                        'labels'] = ds_labels

                    self.k8s.create_daemon_action(
                        namespace=namespace, template=template)

                    # delete pods
                    self.delete_resources(
                        release_name, name, 'pod', resource_labels, namespace,
                        wait=True, timeout=timeout)

        else:
            LOG.error("Unable to exectue name: % type: %s", name, action_type) class ArmadaTimeoutException(ArmadaException):
    '''Exception that occurs when Armada times out while processing.'''

    def __init__(self, reason):
        self._message = 'Armada timed out waiting on: %s' % (reason)
        super(ArmadaTimeoutException, self).__init__(self._message) class SourceException(base_exception.ArmadaBaseException):
class GitException(SourceException):
class GitAuthException(SourceException):
class GitProxyException(SourceException):
class GitSSHException(SourceException):
class SourceCleanupException(SourceException):
class TarballDownloadException(SourceException):
class TarballExtractException(SourceException):
class InvalidPathException(SourceException):
class ChartSourceException(SourceException):
    def __init__(self, location):
    def __init__(self, repo_url, ssh_key_path):
    def __init__(self, location):
    def __init__(self, ssh_key_path):
    def __init__(self, target_dir):
    def __init__(self, tarball_url):
    def __init__(self, tarball_dir):
    def __init__(self, path):
    def __init__(self, chart_name, source_type): class ValidateException(base_exception.ArmadaBaseException):
class InvalidManifestException(ValidateException):
class InvalidChartNameException(ValidateException):
class InvalidChartDefinitionException(ValidateException):
class InvalidReleaseException(ValidateException):
class InvalidArmadaObjectException(ValidateException): class TillerException(ex):
class TillerServicesUnavailableException(TillerException):
class ChartCleanupException(TillerException):
class ListChartsException(TillerException):
class PostUpdateJobDeleteException(TillerException):
class PostUpdateJobCreateException(TillerException):
class PreUpdateJobDeleteException(TillerException):
class PreUpdateJobCreateException(TillerException):
class ReleaseException(TillerException):
class ChannelException(TillerException):
class GetReleaseStatusException(TillerException):
class GetReleaseContentException(TillerException):
class TillerPodNotFoundException(TillerException):
class TillerPodNotRunningException(TillerException):
class TillerVersionException(TillerException):
    def __init__(self, chart_name):
    def __init__(self, name, namespace):
    def __init__(self, name, namespace):
    def __init__(self, name, namespace):
    def __init__(self, name, namespace):
    def __init__(self, name, status, action):
    def __init__(self, release, version):
    def __init__(self, release, version):
    def __init__(self, labels): def release_prefix(prefix, chart):
    '''
    how to attach prefix to chart
    '''
    return "{}-{}".format(prefix, chart) CONF = cfg.CONF
LOG = logging.getLogger(__name__)
def git_clone(repo_url, ref='master', proxy_server=None, auth_method=None):
def get_tarball(tarball_url, verify=False):
def download_tarball(tarball_url, verify=False):
def extract_tarball(tarball_path):
def source_cleanup(git_path): LOG = logging.getLogger(__name__)
SCHEMAS = {}
def _get_schema_dir():
def _load_schemas():
def _validate_armada_manifest(manifest):
def validate_armada_manifests(documents):
def validate_armada_document(document):
def validate_armada_documents(documents):
def validate_manifest_url(value):
def validate_manifest_filepath(value): DEFAULT_CHART_TIMEOUT = 3600 KEYWORD_ARMADA = 'armada' KEYWORD_CHARTS = 'chart_group' KEYWORD_GROUPS = 'chart_groups' KEYWORD_PREFIX = 'release_prefix' STATUS_FAILED = 'FAILED'
import difflib
import time
import yaml
from oslo_config import cfg
from oslo_log import log as logging
from armada.handlers.chartbuilder import ChartBuilder
from armada.handlers.manifest import Manifest
from armada.handlers.override import Override
from armada.handlers.tiller import Tiller
from armada.exceptions.armada_exceptions import ArmadaTimeoutException
from armada.exceptions import source_exceptions
from armada.exceptions import validate_exceptions
from armada.exceptions import tiller_exceptions
from armada.utils.release import release_prefix
from armada.utils import source
from armada.utils import validate
from armada.const import DEFAULT_CHART_TIMEOUT
from armada.const import KEYWORD_ARMADA
from armada.const import KEYWORD_CHARTS
from armada.const import KEYWORD_GROUPS
from armada.const import KEYWORD_PREFIX
from armada.const import STATUS_FAILED
                                 'before deployment', release[0])
                        self.tiller.uninstall_release(release[0])

        # Clone the chart sources
        #
        # We only support a git source type right now, which can also
        # handle git:// local paths as well
        repos = {}
        for group in manifest_data.get(KEYWORD_GROUPS, []):
            for ch in group.get(KEYWORD_CHARTS, []):
                self.tag_cloned_repo(ch, repos)

                for dep in ch.get('chart', {}).get('dependencies', []):
                    self.tag_cloned_repo(dep, repos)

    def tag_cloned_repo(self, ch, repos):
        chart = ch.get('chart', {})
        chart_source = chart.get('source', {})
        location = chart_source.get('location')
        ct_type = chart_source.get('type')
        subpath = chart_source.get('subpath', '.')

        if ct_type == 'local':
            chart['source_dir'] = (location, subpath)
        elif ct_type == 'tar':
            LOG.info('Downloading tarball from: %s', location)

            if not CONF.certs:
                LOG.warn(
                    'Disabling server validation certs to extract charts')
                tarball_dir = source.get_tarball(location, verify=False)
            else:
                tarball_dir = source.get_tarball(location, verify=CONF.cert)

            chart['source_dir'] = (tarball_dir, subpath)
        elif ct_type == 'git':
            reference = chart_source.get('reference', 'master')
            repo_branch = (location, reference)

            if repo_branch not in repos:
                auth_method = chart_source.get('auth_method')
                proxy_server = chart_source.get('proxy_server')

                logstr = 'Cloning repo: {} from branch: {}'.format(
                    *repo_branch)
                if proxy_server:
                    logstr += ' proxy: {}'.format(proxy_server)
                if auth_method:
                    logstr += ' auth method: {}'.format(auth_method)
                LOG.info(logstr)

                repo_dir = source.git_clone(*repo_branch,
                                            proxy_server=proxy_server,
                                            auth_method=auth_method)
                repos[repo_branch] = repo_dir

                chart['source_dir'] = (repo_dir, subpath)
            else:
                chart['source_dir'] = (repos.get(repo_branch), subpath)
        else:
            chart_name = chart.get('chart_name')
            raise source_exceptions.ChartSourceException(ct_type, chart_name)

    def get_releases_by_status(self, status):
        '''
        :params status - status string to filter releases on

        Return a list of current releases with a specified status
        '''
        filtered_releases = []
        known_releases = self.tiller.list_charts()
        for release in known_releases:
            if release[4] == status:
                filtered_releases.append(release)

        return filtered_releases

    def sync(self):
        '''
        Synchronize Helm with the Armada Config(s)
        '''

        msg = {'install': [], 'upgrade': [], 'diff': []}

        # TODO: (gardlt) we need to break up this func into
        # a more cleaner format
        self.pre_flight_ops()

        # extract known charts on tiller right now
        known_releases = self.tiller.list_charts()
        manifest_data = self.manifest.get(KEYWORD_ARMADA, {})
        prefix = manifest_data.get(KEYWORD_PREFIX, '')

        for chartgroup in manifest_data.get(KEYWORD_GROUPS, []):
            cg_name = chartgroup.get('name', '<missing name>')
            cg_desc = chartgroup.get('description', '<missing description>')
            LOG.info('Processing ChartGroup: %s (%s)', cg_name, cg_desc)

            cg_sequenced = chartgroup.get('sequenced', False)
            cg_test_all_charts = chartgroup.get('test_charts', False)

            namespaces_seen = set()
            tests_to_run = []

            cg_charts = chartgroup.get(KEYWORD_CHARTS, [])

            # Track largest Chart timeout to stop the ChartGroup at the end
            cg_max_timeout = 0

            for chart_entry in cg_charts:
                chart = chart_entry.get('chart', {})
                namespace = chart.get('namespace')
                release = chart.get('release')
                values = chart.get('values', {})
                pre_actions = {}
                post_actions = {}

                wait_timeout = self.timeout
                wait_labels = {}


                release_name = release_prefix(prefix, release)