<reponame>oponcea/ceph-uprev-stx-config
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# All Rights Reserved.
#

""" System Inventory Kubernetes Application Operator."""

import docker
import grp
import os
import pwd
import re
import shutil
import subprocess
import threading
import time
import yaml

from collections import namedtuple
from eventlet import greenpool
from eventlet import greenthread
from eventlet import queue
from eventlet import Timeout
from oslo_config import cfg
from oslo_log import log as logging
from sysinv.common import constants
from sysinv.common import exception
from sysinv.common import kubernetes
from sysinv.common import utils as cutils
from sysinv.helm import common
from sysinv.helm import helm


# Log and config
LOG = logging.getLogger(__name__)
kube_app_opts = [
    cfg.StrOpt('armada_image_tag',
               default=('quay.io/airshipit/armada:'
                        'f807c3a1ec727c883c772ffc618f084d960ed5c9'),
               help='Docker image tag of Armada.'),
                ]
CONF = cfg.CONF
CONF.register_opts(kube_app_opts)


# Constants
APPLY_SEARCH_PATTERN = 'Processing Chart,'
ARMADA_CONTAINER_NAME = 'armada_service'
ARMADA_MANIFEST_APPLY_SUCCESS_MSG = 'Done applying manifest'
CONTAINER_ABNORMAL_EXIT_CODE = 137
DELETE_SEARCH_PATTERN = 'Deleting release'
INSTALLATION_TIMEOUT = 3600
MAX_DOWNLOAD_THREAD = 20
TARFILE_DOWNLOAD_CONNECTION_TIMEOUT = 60
TARFILE_TRANSFER_CHUNK_SIZE = 1024 * 512


# Helper functions
def generate_armada_manifest_filename(app_name, manifest_filename):
    return os.path.join('/manifests', app_name + '-' + manifest_filename)


def generate_armada_manifest_filename_abs(app_name, manifest_filename):
    return os.path.join(constants.APP_SYNCED_DATA_PATH,
                        app_name + '-' + manifest_filename)


def generate_manifest_filename_abs(app_name, manifest_filename):
    return os.path.join(constants.APP_INSTALL_PATH,
                        app_name, manifest_filename)


def generate_images_filename_abs(app_name):
    return os.path.join(constants.APP_SYNCED_DATA_PATH,
                        app_name + '-images.yaml')


def create_app_path(path):
    uid = pwd.getpwnam(constants.SYSINV_USERNAME).pw_uid
    gid = os.getgid()

    if not os.path.exists(constants.APP_INSTALL_PATH):
        os.makedirs(constants.APP_INSTALL_PATH)
        os.chown(constants.APP_INSTALL_PATH, uid, gid)

    os.makedirs(path)
    os.chown(path, uid, gid)


def get_app_install_root_path_ownership():
    uid = os.stat(constants.APP_INSTALL_ROOT_PATH).st_uid
    gid = os.stat(constants.APP_INSTALL_ROOT_PATH).st_gid
    return (uid, gid)


Chart = namedtuple('Chart', 'name namespace')


class AppOperator(object):
    """Class to encapsulate Kubernetes App operations for System Inventory"""

    def __init__(self, dbapi):
        self._dbapi = dbapi
        self._docker = DockerHelper()
        self._helm = helm.HelmOperator(self._dbapi)
        self._kube = kubernetes.KubeOperator(self._dbapi)
        self._lock = threading.Lock()

    def _cleanup(self, app):
        """" Remove application directories and override files """
        try:
            if (app.status != constants.APP_UPLOAD_FAILURE and
                    os.path.exists(os.path.join(app.path, 'metadata.yaml'))):
                self._process_node_labels(app, op=constants.LABEL_REMOVE_OP)
            if app.system_app and app.status != constants.APP_UPLOAD_FAILURE:
                self._remove_chart_overrides(app.armada_mfile_abs)

            if os.path.exists(app.armada_mfile_abs):
                os.unlink(app.armada_mfile_abs)
            if os.path.exists(app.imgfile_abs):
                os.unlink(app.imgfile_abs)

            if os.path.exists(app.path):
                shutil.rmtree(app.path)
        except OSError as e:
            LOG.error(e)

    def _update_app_status(self, app, new_status=None, new_progress=None):
        """ Persist new app status """

        if new_status is None:
            new_status = app.status
        elif (new_status in [constants.APP_UPLOAD_SUCCESS,
                             constants.APP_APPLY_SUCCESS]):
            new_progress = constants.APP_PROGRESS_COMPLETED

        with self._lock:
            app.update_status(new_status, new_progress)

    def _abort_operation(self, app, operation,
                         progress=constants.APP_PROGRESS_ABORTED):
        if (app.status == constants.APP_UPLOAD_IN_PROGRESS):
            self._update_app_status(app, constants.APP_UPLOAD_FAILURE,
                                    progress)
        elif (app.status == constants.APP_APPLY_IN_PROGRESS):
            self._update_app_status(app, constants.APP_APPLY_FAILURE,
                                    progress)
        elif (app.status == constants.APP_REMOVE_IN_PROGRESS):
            self._update_app_status(app, constants.APP_REMOVE_FAILURE,
                                    progress)
        LOG.error("Application %s aborted!." % operation)

    def _download_tarfile(self, app):
        from six.moves.urllib.request import urlopen
        from six.moves.urllib.error import HTTPError
        from six.moves.urllib.error import URLError
        from socket import timeout as socket_timeout
        from six.moves.urllib.parse import urlparse

        def _handle_download_failure(reason):
            raise exception.KubeAppUploadFailure(
                name=app.name,
                reason=reason)

        try:
            remote_file = urlopen(
                app.tarfile, timeout=TARFILE_DOWNLOAD_CONNECTION_TIMEOUT)
            try:
                remote_filename = remote_file.info()['Content-Disposition']
            except KeyError:
                remote_filename = os.path.basename(
                    urlparse.urlsplit(remote_file.url).path)

            filename_avail = True if (remote_filename is None or
                                      remote_filename == '') else False

            if filename_avail:
                if (not remote_filename.endswith('.tgz') and
                        not remote_filename.endswith('.tar.gz')):
                    reason = app.tarfile + ' has unrecognizable tar file ' + \
                        'extension. Supported extensions are: .tgz and .tar.gz.'
                    _handle_download_failure(reason)
                    return None

                filename = '/tmp/' + remote_filename
            else:
                filename = '/tmp/' + app.name + '.tgz'

            with open(filename, 'wb') as dest:
                shutil.copyfileobj(remote_file, dest, TARFILE_TRANSFER_CHUNK_SIZE)
            return filename

        except HTTPError as err:
            LOG.error(err)
            reason = 'failed to download tarfile ' + app.tarfile + \
                     ', error code = ' + str(err.code)
            _handle_download_failure(reason)
        except URLError as err:
            LOG.error(err)
            reason = app.tarfile + ' is unreachable.'
            _handle_download_failure(reason)
        except shutil.Error as err:
            LOG.error(err)
            err_file = os.path.basename(filename) if filename_avail else app.tarfile
            reason = 'failed to process tarfile ' + err_file
            _handle_download_failure(reason)
        except socket_timeout as e:
            LOG.error(e)
            reason = 'failed to download tarfile ' + app.tarfile + \
                     ', connection timed out.'
            _handle_download_failure(reason)

    def _extract_tarfile(self, app):
        def _handle_extract_failure(
                reason='failed to extract tarfile content.'):
            raise exception.KubeAppUploadFailure(
                name=app.name,
                reason=reason)

        def _find_manifest_file(app_path):
            mfiles = cutils.find_manifest_file(app_path)

            if mfiles is None:
                _handle_extract_failure('manifest file is corrupted.')

            if mfiles:
                if len(mfiles) == 1:
                    return mfiles[0]
                else:
                    _handle_extract_failure(
                        'tarfile contains more than one manifest file.')
            else:
                _handle_extract_failure('manifest file is missing.')

        orig_uid, orig_gid = get_app_install_root_path_ownership()

        try:
            # One time set up of Armada manifest path for the system
            if not os.path.isdir(constants.APP_SYNCED_DATA_PATH):
                os.makedirs(constants.APP_SYNCED_DATA_PATH)

            if not os.path.isdir(app.path):
                create_app_path(app.path)

            # Temporarily change /scratch group ownership to wrs_protected
            os.chown(constants.APP_INSTALL_ROOT_PATH, orig_uid,
                     grp.getgrnam(constants.SYSINV_WRS_GRPNAME).gr_gid)

            # Extract the tarfile as sysinv user
            if not cutils.extract_tarfile(app.path, app.tarfile, demote_user=True):
                _handle_extract_failure()

            if app.downloaded_tarfile:
                if not cutils.verify_checksum(app.path):
                    _handle_extract_failure('checksum validation failed.')
                mname, mfile = _find_manifest_file(app.path)
                # Save the official manifest file info. They will be persisted
                # in the next status update
                app.regenerate_manifest_filename(mname, os.path.basename(mfile))

            if os.path.isdir(app.charts_dir):
                if len(os.listdir(app.charts_dir)) == 0:
                    _handle_extract_failure('tarfile contains no Helm charts.')

                tar_filelist = cutils.get_files_matching(app.charts_dir,
                                                         '.tgz')
                if not tar_filelist:
                    reason = 'tarfile contains no Helm charts of expected ' + \
                             'file extension (.tgz).'
                    _handle_extract_failure(reason)

                for p, f in tar_filelist:
                    if not cutils.extract_tarfile(
                            p, os.path.join(p, f), demote_user=True):
                        _handle_extract_failure()
        except OSError as e:
            LOG.error(e)
            _handle_extract_failure()
        finally:
            os.chown(constants.APP_INSTALL_ROOT_PATH, orig_uid, orig_gid)

    def _get_image_tags_by_path(self, path):
        """ Mine the image tags from values.yaml files in the chart directory,
            intended for custom apps.

            TODO(awang): Support custom apps to pull images from local registry
        """

        image_tags = []
        ids = []
        for r, f in cutils.get_files_matching(path, 'values.yaml'):
            with open(os.path.join(r, f), 'r') as file:
                try:
                    y = yaml.load(file)
                    ids = y["images"]["tags"].values()
                except (TypeError, KeyError):
                    pass
            image_tags.extend(ids)
        return list(set(image_tags))

    def _get_image_tags_by_charts(self, app_path, charts):
        """ Mine the image tags from the chart paths. Add the converted
            image tags to the overrides if the image tags from the chart
            paths do not exist. Intended for system app.

            The image tagging conversion(local docker registry address prepended):
            ${LOCAL_DOCKER_REGISTRY_IP}:${REGISTRY_PORT}/<image-name>
            (ie..192.168.204.2:9001/docker.io/mariadb:10.2.13)
        """

        local_docker_registry_ip = self._dbapi.address_get_by_name(
            cutils.format_address_name(constants.CONTROLLER_HOSTNAME,
                                       constants.NETWORK_TYPE_MGMT)
        ).address

        image_tags = []
        for chart in charts:
            images_charts = {}
            images_overrides = {}
            overrides = chart.namespace + '-' + chart.name + '.yaml'
            overrides_file = os.path.join(common.HELM_OVERRIDES_PATH,
                                          overrides)
            chart_name = os.path.join(app_path, chart.name)
            chart_path = os.path.join(chart_name, 'values.yaml')

            # Get the image tags from the chart path
            if os.path.exists(chart_path):
                with open(chart_path, 'r') as file:
                    try:
                        doc = yaml.load(file)
                        images_charts = doc["images"]["tags"]
                    except (TypeError, KeyError):
                        pass

            # Get the image tags from the overrides file
            if os.path.exists(overrides_file):
                with open(overrides_file, 'r') as file:
                    try:
                        y = yaml.load(file)
                        images_overrides = y["data"]["values"]["images"]["tags"]
                    except (TypeError, KeyError):
                        LOG.info("Overrides file %s has no img tags" %
                                  overrides_file)
                        pass

            # Add the converted image tags to the overrides if the images from
            # the chart path do not exist in the overrides
            tags_updated = False
            for key, image_tag in images_charts.items():
                if (key not in images_overrides and
                        not image_tag.startswith(local_docker_registry_ip)):
                    images_overrides.update(
                        {key: '{}:{}/{}'.format(local_docker_registry_ip, common.REGISTRY_PORT, image_tag)})
                    tags_updated = True

            if tags_updated:
                with open(overrides_file, 'w') as file:
                    try:
                        if "images" not in y["data"]["values"]:
                            file.seek(0)
                            file.truncate()
                            y["data"]["values"]["images"] = {"tags": images_overrides}
                        else:
                            y["data"]["values"]["images"]["tags"] = images_overrides

                        yaml.safe_dump(y, file, explicit_start=True,
                                       default_flow_style=False)
                        LOG.info("Overrides file %s updated with new image tags" %
                                  overrides_file)
                    except (TypeError, KeyError):
                        LOG.error("Overrides file %s fails to update" %
                                   overrides_file)

            if images_overrides:
                image_tags.extend(images_overrides.values())

        return list(set(image_tags))

    def _register_embedded_images(self, app):
        """
        TODO(tngo): When we're ready to support air-gap scenario and private
        images, the following need to be done:
            a. load the embedded images
            b. tag and push them to the docker registery on the controller
            c. find image tag IDs in each chart and replace their values with
               new tags. Alternatively, document the image tagging convention
               ${MGMT_FLOATING_IP}:${REGISTRY_PORT}/<image-name>
               (e.g. 192.168.204.2:9001/prom/mysqld-exporter)
               to be referenced in the application Helm charts.
        """
        raise exception.KubeAppApplyFailure(
            name=app.name,
            reason="embedded images are not yet supported.")

    def _save_images_list(self, app):
        # Extract the list of images from the charts and overrides where
        # applicable. Save the list to the same location as the armada manifest
        # so it can be sync'ed.
        if app.system_app:
            LOG.info("Generating application overrides...")
            self._helm.generate_helm_application_overrides(
                app.name, cnamespace=None, armada_format=True, combined=True)
            app.charts = self._get_list_of_charts(app.armada_mfile_abs)
            # Get the list of images from the updated images overrides
            images_to_download = self._get_image_tags_by_charts(
                app.charts_dir, app.charts)
        else:
            # For custom apps, mine image tags from application path
            images_to_download = self._get_image_tags_by_path(app.path)

        if not images_to_download:
            # TODO(tngo): We may want to support the deployment of apps that
            # set up resources only in the future. In which case, generate
            # an info log and let it advance to the next step.
            raise exception.KubeAppUploadFailure(
                name=app.name,
                reason="charts specify no docker images.")

        with open(app.imgfile_abs, 'wb') as f:
            yaml.safe_dump(images_to_download, f, explicit_start=True,
                           default_flow_style=False)

    def _retrieve_images_list(self, app_images_file):
        with open(app_images_file, 'rb') as f:
            images_list = yaml.load(f)
        return images_list

    def _download_images(self, app):
        if os.path.isdir(app.images_dir):
            return self._register_embedded_images(app)

        if app.system_app:
            # Some images could have been overwritten via user overrides
            # between upload and apply, or between applies. Refresh the
            # saved images list.
            saved_images_list = self._retrieve_images_list(app.imgfile_abs)
            combined_images_list = list(saved_images_list)
            combined_images_list.extend(
                self._get_image_tags_by_charts(app.charts_dir, app.charts))
            images_to_download = list(set(combined_images_list))
            if saved_images_list != images_to_download:
                with open(app.imgfile_abs, 'wb') as f:
                    yaml.safe_dump(images_to_download, f, explicit_start=True,
                                   default_flow_style=False)
        else:
            images_to_download = self._retrieve_images_list(app.imgfile_abs)

        total_count = len(images_to_download)
        threads = min(MAX_DOWNLOAD_THREAD, total_count)
        failed_downloads = []

        start = time.time()
        pool = greenpool.GreenPool(size=threads)
        for tag, rc in pool.imap(self._docker.download_an_image,
                                images_to_download):
            if not rc:
                failed_downloads.append(tag)
        elapsed = time.time() - start
        failed_count = len(failed_downloads)
        if failed_count > 0:
            raise exception.KubeAppApplyFailure(
                name=app.name,
                reason="failed to download one or more image(s).")
        else:
            LOG.info("All docker images for application %s were successfully "
                     "downloaded in %d seconds" % (app.name, elapsed))

    def _validate_helm_charts(self, app):
        failed_charts = []
        for r, f in cutils.get_files_matching(app.charts_dir, 'Chart.yaml'):
            # Eliminate redundant validation for system app
            if app.system_app and '/charts/helm-toolkit' in r:
                continue
            try:
                output = subprocess.check_output(['helm', 'lint', r])
                if "no failures" in output:
                    LOG.info("Helm chart %s validated" % os.path.basename(r))
                else:
                    LOG.error("Validation failed for helm chart %s" %
                              os.path.basename(r))
                    failed_charts.append(r)
            except Exception as e:
                raise exception.KubeAppUploadFailure(
                    name=app.name, reason=str(e))

        if len(failed_charts) > 0:
            raise exception.KubeAppUploadFailure(
                name=app.name, reason="one or more charts failed validation.")

    def _upload_helm_charts(self, app):
        # Set env path for helm-upload execution
        env = os.environ.copy()
        env['PATH'] = '/usr/local/sbin:' + env['PATH']
        charts = [os.path.join(r, f)
                  for r, f in cutils.get_files_matching(app.charts_dir, '.tgz')]

        orig_uid, orig_gid = get_app_install_root_path_ownership()
        try:
            # Temporarily change /scratch group ownership to wrs_protected
            os.chown(constants.APP_INSTALL_ROOT_PATH, orig_uid,
                     grp.getgrnam(constants.SYSINV_WRS_GRPNAME).gr_gid)
            with open(os.devnull, "w") as fnull:
                for chart in charts:
                    subprocess.check_call(['helm-upload', chart], env=env,
                                          stdout=fnull, stderr=fnull)
                    LOG.info("Helm chart %s uploaded" % os.path.basename(chart))
        except Exception as e:
            raise exception.KubeAppUploadFailure(
                name=app.name, reason=str(e))
        finally:
            os.chown(constants.APP_INSTALL_ROOT_PATH, orig_uid, orig_gid)

    def _validate_labels(self, labels):
        expr = re.compile(r'[a-z0-9]([-a-z0-9]*[a-z0-9])')
        for label in labels:
            if not expr.match(label):
                return False
        return True

    def _update_kubernetes_labels(self, hostname, label_dict):
        body = {
            'metadata': {
                'labels': {}
            }
        }
        body['metadata']['labels'].update(label_dict)
        try:
            self._kube.kube_patch_node(hostname, body)
        except exception.K8sNodeNotFound:
            pass

    def _assign_host_labels(self, hosts, labels):
        for host in hosts:
            if host.administrative != constants.ADMIN_LOCKED:
                continue
            for label_str in labels:
                k, v = label_str.split('=')
                try:
                    self._dbapi.label_create(
                        host.id, {'host_id': host.id,
                                  'label_key': k,
                                  'label_value': v})
                except exception.HostLabelAlreadyExists:
                    pass
            label_dict = {k: v for k, v in (i.split('=') for i in labels)}
            self._update_kubernetes_labels(host.hostname, label_dict)

    def _find_label(self, host_uuid, label_str):
        host_labels = self._dbapi.label_get_by_host(host_uuid)
        for label_obj in host_labels:
            if label_str == label_obj.label_key + '=' + label_obj.label_value:
                return label_obj
        return None

    def _remove_host_labels(self, hosts, labels):
        for host in hosts:
            if host.administrative != constants.ADMIN_LOCKED:
                continue
            null_labels = {}
            for label_str in labels:
                lbl_obj = self._find_label(host.uuid, label_str)
                if lbl_obj:
                    self._dbapi.label_destroy(lbl_obj.uuid)
                    key = lbl_obj.label_key
                    null_labels[key] = None
            if null_labels:
                self._update_kubernetes_labels(host.hostname, null_labels)

    def _process_node_labels(self, app, op=constants.LABEL_ASSIGN_OP):
        # Node labels are host personality based and are defined in
        # metadata.yaml file in the following format:
        # labels:
        #   controller: '<label-key1>=<value>, <label-key2>=<value>, ...'
        #   compute: '<label-key1>=<value>, <label-key2>=<value>, ...'

        lfile = os.path.join(app.path, 'metadata.yaml')
        controller_labels = []
        compute_labels = []
        controller_l = compute_l = None
        controller_labels_set = set()
        compute_labels_set = set()

        if os.path.exists(lfile) and os.path.getsize(lfile) > 0:
            with open(lfile, 'r') as f:
                try:
                    y = yaml.load(f)
                    labels = y['labels']
                except KeyError:
                    raise exception.KubeAppUploadFailure(
                        name=app.name,
                        reason="labels file contains no labels.")
            for key, value in labels.items():
                if key == constants.CONTROLLER:
                    controller_l = value
                elif key == constants.WORKER:
                    compute_l = value
        else:
            if not app.system_app:
                LOG.info("Application %s does not require specific node "
                         "labeling." % app.name)
                return

        if controller_l:
            controller_labels =\
                controller_l.replace(',', ' ').split()
            controller_labels_set = set(controller_labels)
            if not self._validate_labels(controller_labels_set):
                raise exception.KubeAppUploadFailure(
                    name=app.name,
                    reason="controller labels are malformed.")

        if compute_l:
            compute_labels =\
                compute_l.replace(',', ' ').split()
            compute_labels_set = set(compute_labels)
            if not self._validate_labels(compute_labels_set):
                raise exception.KubeAppUploadFailure(
                    name=app.name,
                    reason="compute labels are malformed.")

        # Add the default labels for system app. They must exist for
        # the app manifest to be applied successfully. If the nodes have
        # been assigned these labels manually before, these
        # reassignments are simply ignored.
        if app.system_app:
            controller_labels_set.add(constants.CONTROL_PLANE_LABEL)
            compute_labels_set.add(constants.COMPUTE_NODE_LABEL)
            compute_labels_set.add(constants.OPENVSWITCH_LABEL)
            compute_labels_set.add(constants.SRIOV_LABEL)

        # Get controller host(s)
        controller_hosts =\
            self._dbapi.ihost_get_by_personality(constants.CONTROLLER)
        if constants.WORKER in controller_hosts[0].subfunctions:
            # AIO system
            labels = controller_labels_set.union(compute_labels_set)
            if op == constants.LABEL_ASSIGN_OP:
                self._assign_host_labels(controller_hosts, labels)
            elif op == constants.LABEL_REMOVE_OP:
                self._remove_host_labels(controller_hosts, labels)
        else:
            # Standard system
            compute_hosts =\
                self._dbapi.ihost_get_by_personality(constants.WORKER)
            if op == constants.LABEL_ASSIGN_OP:
                self._assign_host_labels(controller_hosts, controller_labels_set)
                self._assign_host_labels(compute_hosts, compute_labels_set)
            elif op == constants.LABEL_REMOVE_OP:
                self._remove_host_labels(controller_hosts, controller_labels_set)
                self._remove_host_labels(compute_hosts, compute_labels_set)

    def _get_list_of_charts(self, manifest_file):
        charts = []
        with open(manifest_file, 'r') as f:
            docs = yaml.load_all(f)
            for doc in docs:
                try:
                    if "armada/Chart/" in doc['schema']:
                        charts.append(Chart(
                            name=doc['data']['chart_name'],
                            namespace=doc['data']['namespace']))
                except KeyError:
                    pass
        return charts

    def _get_overrides_files(self, charts):
        """Returns list of override files or None, used in
           application-install and application-delete."""

        missing_overrides = []
        available_overrides = []
        excluded = ['helm-toolkit']

        for chart in charts:
            overrides = chart.namespace + '-' + chart.name + '.yaml'
            if chart.name in excluded:
                LOG.debug("Skipping overrides %s " % overrides)
                continue
            overrides_file = os.path.join(common.HELM_OVERRIDES_PATH,
                                          overrides)
            if not os.path.exists(overrides_file):
                missing_overrides.append(overrides_file)
            else:
                available_overrides.append(overrides_file)

                # Now handle any meta-overrides files.  These can affect
                # sections of the chart schema other than "values, and can
                # affect the chartgroup or even the manifest.
                if self._helm.generate_meta_overrides(
                        chart.name, chart.namespace):
                    overrides = chart.namespace + '-' + chart.name + \
                                '-meta' + '.yaml'
                    overrides_file = os.path.join(common.HELM_OVERRIDES_PATH,
                                                  overrides)
                    if not os.path.exists(overrides_file):
                        missing_overrides.append(overrides_file)
                    else:
                        available_overrides.append(overrides_file)

        if missing_overrides:
            LOG.error("Missing the following overrides: %s" % missing_overrides)
            return None
        return available_overrides

    def _generate_armada_overrides_str(self, overrides_files):
        return " ".join([' --values /overrides/{0}'.format(os.path.basename(i))
                        for i in overrides_files])

    def _remove_chart_overrides(self, manifest_file):
        charts = self._get_list_of_charts(manifest_file)
        for chart in charts:
            if chart.name in constants.SUPPORTED_HELM_CHARTS:
                self._helm.remove_helm_chart_overrides(chart.name,
                                                       chart.namespace)

    def _make_armada_request_with_monitor(self, app, request, overrides_str=None):
        """Initiate armada request with monitoring

        This method delegates the armada request to docker helper and starts
        a monitoring thread to persist status and progress along the way.

        :param app: application data object
        :param request: type of request (apply or delete)
        :param overrides_str: list of overrides in string format to be applied
        """

        def _get_armada_log_stats(pattern, logfile):
            """
            TODO(tngo): In the absence of an Armada API that provides the current
            status of an apply/delete manifest operation, the progress is derived
            from specific log entries extracted from the execution logs. This
            inner method is to be replaced with an official API call when
            it becomes available.
            """
            p1 = subprocess.Popen(['docker', 'exec', ARMADA_CONTAINER_NAME,
                                   'grep', pattern, logfile],
                                   stdout=subprocess.PIPE)
            p2 = subprocess.Popen(['awk', '{print $NF}'], stdin=p1.stdout,
                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            p1.stdout.close()
            result, err = p2.communicate()
            if result:
                # Strip out ANSI color code that might be in the text stream
                r = re.compile("\x1b\[[0-9;]*m")
                result = r.sub('', result)
                matches = result.split()
                num_chart_processed = len(matches)
                last_chart_processed = matches[num_chart_processed - 1]
                if '=' in last_chart_processed:
                    last_chart_processed = last_chart_processed.split('=')[1]
                return last_chart_processed, num_chart_processed

            return None, None

        def _check_progress(monitor_flag, app, pattern, logfile):
            """ Progress monitoring task, to be run in a separate thread """
            LOG.info("Starting progress monitoring thread for app %s" % app.name)
            try:
                with Timeout(INSTALLATION_TIMEOUT,
                             exception.KubeAppProgressMonitorTimeout()):
                    while True:
                        try:
                            monitor_flag.get_nowait()
                            LOG.debug("Received monitor stop signal for %s" % app.name)
                            monitor_flag.task_done()
                            break
                        except queue.Empty:
                            last, num = _get_armada_log_stats(pattern, logfile)
                            if last:
                                if app.system_app:
                                    # helm-toolkit doesn't count
                                    percent = \
                                        round(float(num) / (len(app.charts) - 1) * 100)
                                else:
                                    percent = round(float(num) / len(app.charts) * 100)
                                progress_str = 'processing chart: ' + last +\
                                    ', overall completion: ' + str(percent) + '%'
                                if app.progress != progress_str:
                                    LOG.info("%s" % progress_str)
                                    self._update_app_status(
                                        app, new_progress=progress_str)
                            greenthread.sleep(1)
            except Exception as e:
                # timeout or subprocess error
                LOG.exception(e)
            finally:
                LOG.info("Exiting progress monitoring thread for app %s" % app.name)

        # Body of the outer method
        mqueue = queue.Queue()
        rc = True
        logfile = app.name + '-' + request + '.log'
        if request == constants.APP_APPLY_OP:
            pattern = APPLY_SEARCH_PATTERN
        else:
            pattern = DELETE_SEARCH_PATTERN

        monitor = greenthread.spawn_after(1, _check_progress, mqueue, app,
                                          pattern, logfile)
        rc = self._docker.make_armada_request(request, app.armada_mfile,
                                              overrides_str, logfile)
        mqueue.put('done')
        monitor.kill()
        return rc

    def perform_app_upload(self, rpc_app, tarfile):
        """Process application upload request

        This method validates the application manifest. If Helm charts are
        included, they are validated and uploaded to local Helm repo. It also
        downloads the required docker images for custom apps during upload
        stage.

        :param rpc_app: application object in the RPC request
        :param tarfile: location of application tarfile
        """

        app = AppOperator.Application(rpc_app)
        LOG.info("Application (%s) upload started." % app.name)

        try:
            app.tarfile = tarfile

            if cutils.is_url(app.tarfile):
                self._update_app_status(
                    app, new_progress=constants.APP_PROGRESS_TARFILE_DOWNLOAD)
                downloaded_tarfile = self._download_tarfile(app)

                if downloaded_tarfile is None:
                    self._abort_operation(app, constants.APP_UPLOAD_OP)
                else:
                    app.tarfile = downloaded_tarfile

                app.downloaded_tarfile = True

            # Full extraction of application tarball at /scratch/apps.
            # Manifest file is placed under /opt/platform/armada
            # which is managed by drbd-sync and visible to Armada.
            self._update_app_status(
                app, new_progress=constants.APP_PROGRESS_EXTRACT_TARFILE)

            with self._lock:
                self._extract_tarfile(app)
            shutil.copy(app.mfile_abs, app.armada_mfile_abs)

            if not self._docker.make_armada_request('validate', app.armada_mfile):
                return self._abort_operation(app, constants.APP_UPLOAD_OP)

            self._update_app_status(
                app, new_progress=constants.APP_PROGRESS_VALIDATE_UPLOAD_CHARTS)
            if os.path.isdir(app.charts_dir):
                self._validate_helm_charts(app)
                with self._lock:
                    self._upload_helm_charts(app)

            self._save_images_list(app)
            self._update_app_status(app, constants.APP_UPLOAD_SUCCESS)
            LOG.info("Application (%s) upload completed." % app.name)
        except exception.KubeAppUploadFailure as e:
            LOG.exception(e)
            self._abort_operation(app, constants.APP_UPLOAD_OP, str(e))
        except Exception as e:
            LOG.exception(e)
            self._abort_operation(app, constants.APP_UPLOAD_OP)

    def perform_app_apply(self, rpc_app):
        """Process application install request

        This method processes node labels per configuration and invokes
        Armada to apply the application manifest.

        For OpenStack app (system app), the method generates combined
        overrides (a merge between system and user overrides if available)
        for the charts that comprise the app before downloading docker images
        and applying the manifest.

        Usage: the method can be invoked at initial install or after the
               user has either made some manual configuration changes or
               or applied (new) user overrides to some Helm chart(s) to
               correct/update a previous manifest apply.

        :param rpc_app: application object in the RPC request
        :return boolean: whether application apply was successful
        """

        app = AppOperator.Application(rpc_app)
        LOG.info("Application (%s) apply started." % app.name)

        self._process_node_labels(app)

        overrides_str = ''
        ready = True
        try:
            app.charts = self._get_list_of_charts(app.armada_mfile_abs)
            if app.system_app:
                self._update_app_status(
                    app, new_progress=constants.APP_PROGRESS_GENERATE_OVERRIDES)
                LOG.info("Generating application overrides...")
                self._helm.generate_helm_application_overrides(
                    app.name, cnamespace=None, armada_format=True,
                    combined=True)
                overrides_files = self._get_overrides_files(app.charts)
                if overrides_files:
                    LOG.info("Application overrides generated.")
                    # Ensure all chart overrides are readable by Armada
                    for file in overrides_files:
                        os.chmod(file, 0o644)
                    overrides_str =\
                        self._generate_armada_overrides_str(overrides_files)
                    self._update_app_status(
                        app, new_progress=constants.APP_PROGRESS_DOWNLOAD_IMAGES)
                    self._download_images(app)
                else:
                    ready = False
            else:
                # No support for custom app overrides at this point, just
                # download the needed images.
                self._update_app_status(
                    app, new_progress=constants.APP_PROGRESS_DOWNLOAD_IMAGES)
                self._download_images(app)

            if ready:
                self._update_app_status(
                    app, new_progress=constants.APP_PROGRESS_APPLY_MANIFEST)
                if self._make_armada_request_with_monitor(app,
                                                          constants.APP_APPLY_OP,
                                                          overrides_str):
                    self._update_app_status(app,
                                            constants.APP_APPLY_SUCCESS)
                    LOG.info("Application (%s) apply completed." % app.name)
                    return True
        except Exception as e:
            LOG.exception(e)

        # If it gets here, something went wrong
        self._abort_operation(app, constants.APP_APPLY_OP)
        return False

    def perform_app_remove(self, rpc_app):
        """Process application remove request

        This method invokes Armada to delete the application manifest.
        For system app, it also cleans up old test pods.

        :param rpc_app: application object in the RPC request
        :return boolean: whether application remove was successful
        """

        app = AppOperator.Application(rpc_app)
        LOG.info("Application (%s) remove started." % app.name)

        app.charts = self._get_list_of_charts(app.armada_mfile_abs)
        self._update_app_status(
            app, new_progress=constants.APP_PROGRESS_DELETE_MANIFEST)

        if self._make_armada_request_with_monitor(app, constants.APP_DELETE_OP):
            if app.system_app:

                # TODO convert these kubectl commands to use the k8s api
                p1 = subprocess.Popen(
                    ['kubectl', '--kubeconfig=/etc/kubernetes/admin.conf',
                     'get', 'pvc', '--no-headers', '-n', 'openstack'],
                    stdout=subprocess.PIPE)
                p2 = subprocess.Popen(['awk', '{print $3}'],
                                      stdin=p1.stdout,
                                      stdout=subprocess.PIPE)
                p3 = subprocess.Popen(
                    ['xargs', '-i', 'kubectl',
                     '--kubeconfig=/etc/kubernetes/admin.conf', 'delete',
                     'pv', '{}', '--wait=false'],
                    stdin=p2.stdout,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE)
                timer = threading.Timer(10, p3.kill)
                try:
                    timer.start()
                    p1.stdout.close()
                    p2.stdout.close()
                    out, err = p3.communicate()
                    if out and not err:
                        LOG.info("Persistent Volumes marked for deletion.")
                    else:
                        self._abort_operation(app, constants.APP_REMOVE_OP)
                        LOG.error("Failed to clean up PVs after app removal.")
                except Exception as e:
                    self._abort_operation(app, constants.APP_REMOVE_OP)
                    LOG.exception("Failed to clean up PVs after app "
                                  "removal: %s" % e)
                finally:
                    timer.cancel()

                p4 = subprocess.Popen(
                    ['kubectl', '--kubeconfig=/etc/kubernetes/admin.conf',
                     'delete', 'namespace', 'openstack'],
                    stdout=subprocess.PIPE)
                timer2 = threading.Timer(10, p4.kill)
                try:
                    timer2.start()
                    out, err = p4.communicate()
                    if out and not err:
                        LOG.info("Openstack namespace delete completed.")
                    else:
                        self._abort_operation(app, constants.APP_REMOVE_OP)
                        LOG.error("Failed to clean up openstack namespace"
                                  " after app removal.")
                except Exception as e:
                    self._abort_operation(app, constants.APP_REMOVE_OP)
                    LOG.exception("Failed to clean up openstack namespace "
                                  "after app removal: %s" % e)
                finally:
                    timer2.cancel()

            self._update_app_status(app, constants.APP_UPLOAD_SUCCESS)
            LOG.info("Application (%s) remove completed." % app.name)
            return True
        else:
            self._abort_operation(app, constants.APP_REMOVE_OP)

        return False

    def perform_app_delete(self, rpc_app):
        """Process application remove request

        This method removes the application entry from the database and
        performs cleanup which entails removing node labels where applicable
        and purge all application files from the system.

        :param rpc_app: application object in the RPC request
        """

        app = AppOperator.Application(rpc_app)
        try:
            self._dbapi.kube_app_destroy(app.name)
            self._cleanup(app)
            LOG.info("Application (%s) has been purged from the system." %
                     app.name)
            msg = None
        except Exception as e:
            # Possible exceptions are KubeAppDeleteFailure,
            # OSError and unexpectedly KubeAppNotFound
            LOG.exception(e)
            msg = str(e)
        return msg

    class Application(object):
        """ Data object to encapsulate all data required to
            support application related operations.
        """

        def __init__(self, rpc_app):
            self._kube_app = rpc_app
            self.path = os.path.join(constants.APP_INSTALL_PATH,
                                     self._kube_app.get('name'))
            self.charts_dir = os.path.join(self.path, 'charts')
            self.images_dir = os.path.join(self.path, 'images')
            self.tarfile = None
            self.downloaded_tarfile = False
            self.system_app =\
                (self._kube_app.get('name') == constants.HELM_APP_OPENSTACK)

            self.armada_mfile = generate_armada_manifest_filename(
                self._kube_app.get('name'),
                self._kube_app.get('manifest_file'))
            self.armada_mfile_abs = generate_armada_manifest_filename_abs(
                self._kube_app.get('name'),
                self._kube_app.get('manifest_file'))
            self.mfile_abs = generate_manifest_filename_abs(
                self._kube_app.get('name'),
                self._kube_app.get('manifest_file'))
            self.imgfile_abs = generate_images_filename_abs(
                self._kube_app.get('name'))

            self.charts = []

        @property
        def name(self):
            return self._kube_app.get('name')

        @property
        def status(self):
            return self._kube_app.get('status')

        @property
        def progress(self):
            return self._kube_app.get('progress')

        def update_status(self, new_status, new_progress):
            self._kube_app.status = new_status
            if new_progress:
                self._kube_app.progress = new_progress
            self._kube_app.save()

        def regenerate_manifest_filename(self, new_mname, new_mfile):
            self._kube_app.manifest_name = new_mname
            self._kube_app.manifest_file = new_mfile
            self.armada_mfile = generate_armada_manifest_filename(
                self.name, new_mfile)
            self.armada_mfile_abs = generate_armada_manifest_filename_abs(
                self.name, new_mfile)
            self.mfile_abs = generate_manifest_filename_abs(
                self.name, new_mfile)


class DockerHelper(object):
    """ Utility class to encapsulate Docker related operations """

    def _start_armada_service(self, client):
        try:
            container = client.containers.get(ARMADA_CONTAINER_NAME)
            if container.status != 'running':
                LOG.info("Restarting Armada service...")
                container.restart()
            return container
        except Exception:
            LOG.info("Starting Armada service...")
            try:
                # First make kubernetes config accessible to Armada. This
                # is a work around the permission issue in Armada container.
                kube_config = os.path.join(constants.APP_SYNCED_DATA_PATH,
                                           'admin.conf')
                shutil.copy('/etc/kubernetes/admin.conf', kube_config)
                os.chown(kube_config, 1000, grp.getgrnam("wrs").gr_gid)

                overrides_dir = common.HELM_OVERRIDES_PATH
                manifests_dir = constants.APP_SYNCED_DATA_PATH
                LOG.info("kube_config=%s, manifests_dir=%s, "
                         "overrides_dir=%s." % (kube_config, manifests_dir,
                                                overrides_dir))
                binds = {
                    kube_config: {'bind': '/armada/.kube/config', 'mode': 'ro'},
                    manifests_dir: {'bind': '/manifests', 'mode': 'ro'},
                    overrides_dir: {'bind': '/overrides', 'mode': 'ro'}}

                container = client.containers.run(
                    CONF.armada_image_tag,
                    name=ARMADA_CONTAINER_NAME,
                    detach=True,
                    volumes=binds,
                    restart_policy={'Name': 'always'},
                    command=None)
                LOG.info("Armada service started!")
                return container
            except OSError as oe:
                LOG.error("Unable to make kubernetes config accessible to "
                          "armada: %s" % oe)
            except Exception as e:
                # Possible docker exceptions are: RuntimeError, ContainerError,
                # ImageNotFound and APIError
                LOG.error("Docker error while launching Armada container: %s", e)
                os.unlink(kube_config)
            return None

    def make_armada_request(self, request, manifest_file, overrides_str='',
                            logfile=None):

        if logfile is None:
            logfile = request + '.log'

        rc = True

        try:
            client = docker.from_env(timeout=INSTALLATION_TIMEOUT)
            armada_svc = self._start_armada_service(client)
            if armada_svc:
                if request == 'validate':
                    cmd = 'armada validate ' + manifest_file
                    (exit_code, exec_logs) = armada_svc.exec_run(cmd)
                    if exit_code == 0:
                        LOG.info("Manifest file %s was successfully validated." %
                                 manifest_file)
                    else:
                        rc = False
                        if exit_code == CONTAINER_ABNORMAL_EXIT_CODE:
                            LOG.error("Failed to validate application manifest %s. "
                                      "Armada service has exited abnormally." %
                                      manifest_file)
                        else:
                            LOG.error("Failed to validate application manifest "
                                      "%s: %s." % (manifest_file, exec_logs))
                elif request == constants.APP_APPLY_OP:
                    cmd = "/bin/bash -c 'armada apply --debug " + manifest_file +\
                          overrides_str + " | tee " + logfile + "'"
                    LOG.info("Armada apply command = %s" % cmd)
                    (exit_code, exec_logs) = armada_svc.exec_run(cmd)
                    if exit_code == 0:
                        if ARMADA_MANIFEST_APPLY_SUCCESS_MSG in exec_logs:
                            LOG.info("Application manifest %s was successfully "
                                     "applied/re-applied." % manifest_file)
                        else:
                            rc = False
                            LOG.error("Received a false positive response from "
                                      "Docker/Armada. Failed to apply application "
                                      "manifest %s: %s." % (manifest_file, exec_logs))
                    else:
                        rc = False
                        if exit_code == CONTAINER_ABNORMAL_EXIT_CODE:
                            LOG.error("Failed to apply application manifest %s. "
                                      "Armada service has exited abnormally." %
                                      manifest_file)
                        else:
                            LOG.error("Failed to apply application manifest %s: "
                                      "%s." % (manifest_file, exec_logs))
                elif request == constants.APP_DELETE_OP:
                    cmd = "/bin/bash -c 'armada delete --debug --manifest " +\
                          manifest_file + " | tee " + logfile + "'"
                    (exit_code, exec_logs) = armada_svc.exec_run(cmd)
                    if exit_code == 0:
                        LOG.info("Application charts were successfully "
                                 "deleted.")
                    else:
                        rc = False
                        if exit_code == CONTAINER_ABNORMAL_EXIT_CODE:
                            LOG.error("Failed to delete application manifest %s. "
                                      "Armada service has exited abnormally." %
                                      manifest_file)
                        else:
                            LOG.error("Failed to delete application manifest %s: "
                                      "%s" % (manifest_file, exec_logs))
                else:
                    rc = False
                    LOG.error("Unsupported armada request: %s." % request)
            else:
                # Armada sevice failed to start/restart
                rc = False
        except Exception as e:
            # Failed to get a docker client
            rc = False
            LOG.error("Armada request %s for manifest %s failed: %s " %
                      (request, manifest_file, e))
        return rc

    def download_an_image(self, loc_img_tag):

        rc = True
        start = time.time()
        try:
            # Pull image from local docker registry
            LOG.info("Image %s download started from local registry" % loc_img_tag)
            client = docker.APIClient(timeout=INSTALLATION_TIMEOUT)
            client.pull(loc_img_tag)
        except docker.errors.NotFound:
            try:
                # Image is not available in local docker registry, get the image
                # from the public registry and push to the local registry
                LOG.info("Image %s is not available in local registry, "
                         "download started from public registry" % loc_img_tag)
                pub_img_tag = loc_img_tag[1 + loc_img_tag.find('/'):]
                client.pull(pub_img_tag)
                client.tag(pub_img_tag, loc_img_tag)
                client.push(loc_img_tag)
            except Exception as e:
                rc = False
                LOG.error("Image %s download failed from public registry: %s" % (pub_img_tag, e))
        except Exception as e:
            rc = False
            LOG.error("Image %s download failed from local registry: %s" % (loc_img_tag, e))
        elapsed_time = time.time() - start

        if rc:
            LOG.info("Image %s download succeeded in %d seconds" %
                     (loc_img_tag, elapsed_time))
        return loc_img_tag, rc

# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (c) 2018 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# All Rights Reserved.
#

""" System Inventory Kubernetes Application Operator."""

import docker
import grp
import os
import pwd
import re
import shutil
import subprocess
import threading
import time
import yaml

from collections import namedtuple
from eventlet import greenpool
from eventlet import greenthread
from eventlet import queue
from eventlet import Timeout
from oslo_config import cfg
from oslo_log import log as logging
from sysinv.common import constants
from sysinv.common import exception
from sysinv.common import kubernetes
from sysinv.common import utils as cutils
from sysinv.helm import common
from sysinv.helm import helm


# Log and config
LOG = logging.getLogger(__name__)
kube_app_opts = [
    cfg.StrOpt('armada_image_tag',
               default=('quay.io/airshipit/armada:'
                        'f807c3a1ec727c883c772ffc618f084d960ed5c9'),
               help='Docker image tag of Armada.'),
                ]
CONF = cfg.CONF
CONF.register_opts(kube_app_opts)


# Constants
APPLY_SEARCH_PATTERN = 'Processing Chart,'
ARMADA_CONTAINER_NAME = 'armada_service'
ARMADA_MANIFEST_APPLY_SUCCESS_MSG = 'Done applying manifest'
CONTAINER_ABNORMAL_EXIT_CODE = 137
DELETE_SEARCH_PATTERN = 'Deleting release'
INSTALLATION_TIMEOUT = 3600
MAX_DOWNLOAD_THREAD = 20
TARFILE_DOWNLOAD_CONNECTION_TIMEOUT = 60
TARFILE_TRANSFER_CHUNK_SIZE = 1024 * 512


# Helper functions
def generate_armada_manifest_filename(app_name, manifest_filename):
    return os.path.join('/manifests', app_name + '-' + manifest_filename)


def generate_armada_manifest_filename_abs(app_name, manifest_filename):
    return os.path.join(constants.APP_SYNCED_DATA_PATH,
                        app_name + '-' + manifest_filename)


def generate_manifest_filename_abs(app_name, manifest_filename):
    return os.path.join(constants.APP_INSTALL_PATH,
                        app_name, manifest_filename)


def generate_images_filename_abs(app_name):
    return os.path.join(constants.APP_SYNCED_DATA_PATH,
                        app_name + '-images.yaml')


def create_app_path(path):
    uid = pwd.getpwnam(constants.SYSINV_USERNAME).pw_uid
    gid = os.getgid()

    if not os.path.exists(constants.APP_INSTALL_PATH):
        os.makedirs(constants.APP_INSTALL_PATH)
        os.chown(constants.APP_INSTALL_PATH, uid, gid)

    os.makedirs(path)
    os.chown(path, uid, gid)


def get_app_install_root_path_ownership():
    uid = os.stat(constants.APP_INSTALL_ROOT_PATH).st_uid
    gid = os.stat(constants.APP_INSTALL_ROOT_PATH).st_gid
    return (uid, gid)


Chart = namedtuple('Chart', 'name namespace')


class AppOperator(object):
    """Class to encapsulate Kubernetes App operations for System Inventory"""

    def __init__(self, dbapi):
        self._dbapi = dbapi
        self._docker = DockerHelper()
        self._helm = helm.HelmOperator(self._dbapi)
        self._kube = kubernetes.KubeOperator(self._dbapi)
        self._lock = threading.Lock()

    def _cleanup(self, app):
        """" Remove application directories and override files """
        try:
            if (app.status != constants.APP_UPLOAD_FAILURE and
                    os.path.exists(os.path.join(app.path, 'metadata.yaml'))):
                self._process_node_labels(app, op=constants.LABEL_REMOVE_OP)
            if app.system_app and app.status != constants.APP_UPLOAD_FAILURE:
                self._remove_chart_overrides(app.armada_mfile_abs)

            if os.path.exists(app.armada_mfile_abs):
                os.unlink(app.armada_mfile_abs)
            if os.path.exists(app.imgfile_abs):
                os.unlink(app.imgfile_abs)

            if os.path.exists(app.path):
                shutil.rmtree(app.path)
        except OSError as e:
            LOG.error(e)

    def _update_app_status(self, app, new_status=None, new_progress=None):
        """ Persist new app status """

        if new_status is None:
            new_status = app.status
        elif (new_status in [constants.APP_UPLOAD_SUCCESS,
                             constants.APP_APPLY_SUCCESS]):
            new_progress = constants.APP_PROGRESS_COMPLETED

        with self._lock:
            app.update_status(new_status, new_progress)

    def _abort_operation(self, app, operation,
                         progress=constants.APP_PROGRESS_ABORTED):
        if (app.status == constants.APP_UPLOAD_IN_PROGRESS):
            self._update_app_status(app, constants.APP_UPLOAD_FAILURE,
                                    progress)
        elif (app.status == constants.APP_APPLY_IN_PROGRESS):
            self._update_app_status(app, constants.APP_APPLY_FAILURE,
                                    progress)
        elif (app.status == constants.APP_REMOVE_IN_PROGRESS):
            self._update_app_status(app, constants.APP_REMOVE_FAILURE,
                                    progress)
        LOG.error("Application %s aborted!." % operation)

    def _download_tarfile(self, app):
        from six.moves.urllib.request import urlopen
        from six.moves.urllib.error import HTTPError
        from six.moves.urllib.error import URLError
        from socket import timeout as socket_timeout
        from six.moves.urllib.parse import urlparse

        def _handle_download_failure(reason):
            raise exception.KubeAppUploadFailure(
                name=app.name,
                reason=reason)

        try:
            remote_file = urlopen(
                app.tarfile, timeout=TARFILE_DOWNLOAD_CONNECTION_TIMEOUT)
            try:
                remote_filename = remote_file.info()['Content-Disposition']
            except KeyError:
                remote_filename = os.path.basename(
                    urlparse.urlsplit(remote_file.url).path)

            filename_avail = True if (remote_filename is None or
                                      remote_filename == '') else False

            if filename_avail:
                if (not remote_filename.endswith('.tgz') and
                        not remote_filename.endswith('.tar.gz')):
                    reason = app.tarfile + ' has unrecognizable tar file ' + \
                        'extension. Supported extensions are: .tgz and .tar.gz.'
                    _handle_download_failure(reason)
                    return None

                filename = '/tmp/' + remote_filename
            else:
                filename = '/tmp/' + app.name + '.tgz'

            with open(filename, 'wb') as dest:
                shutil.copyfileobj(remote_file, dest, TARFILE_TRANSFER_CHUNK_SIZE)
            return filename

        except HTTPError as err:
            LOG.error(err)
            reason = 'failed to download tarfile ' + app.tarfile + \
                     ', error code = ' + str(err.code)
            _handle_download_failure(reason)
        except URLError as err:
            LOG.error(err)
            reason = app.tarfile + ' is unreachable.'
            _handle_download_failure(reason)
        except shutil.Error as err:
            LOG.error(err)
            err_file = os.path.basename(filename) if filename_avail else app.tarfile
            reason = 'failed to process tarfile ' + err_file
            _handle_download_failure(reason)
        except socket_timeout as e:
            LOG.error(e)
            reason = 'failed to download tarfile ' + app.tarfile + \
                     ', connection timed out.'
            _handle_download_failure(reason)

    def _extract_tarfile(self, app):
        def _handle_extract_failure(
                reason='failed to extract tarfile content.'):
            raise exception.KubeAppUploadFailure(
                name=app.name,
                reason=reason)

        def _find_manifest_file(app_path):
            mfiles = cutils.find_manifest_file(app_path)

            if mfiles is None:
                _handle_extract_failure('manifest file is corrupted.')

            if mfiles:
                if len(mfiles) == 1:
                    return mfiles[0]
                else:
                    _handle_extract_failure(
                        'tarfile contains more than one manifest file.')
            else:
                _handle_extract_failure('manifest file is missing.')

        orig_uid, orig_gid = get_app_install_root_path_ownership()

        try:
            # One time set up of Armada manifest path for the system
            if not os.path.isdir(constants.APP_SYNCED_DATA_PATH):
                os.makedirs(constants.APP_SYNCED_DATA_PATH)

            if not os.path.isdir(app.path):
                create_app_path(app.path)

            # Temporarily change /scratch group ownership to wrs_protected
            os.chown(constants.APP_INSTALL_ROOT_PATH, orig_uid,
                     grp.getgrnam(constants.SYSINV_WRS_GRPNAME).gr_gid)

            # Extract the tarfile as sysinv user
            if not cutils.extract_tarfile(app.path, app.tarfile, demote_user=True):
                _handle_extract_failure()

            if app.downloaded_tarfile:
                if not cutils.verify_checksum(app.path):
                    _handle_extract_failure('checksum validation failed.')
                mname, mfile = _find_manifest_file(app.path)
                # Save the official manifest file info. They will be persisted
                # in the next status update
                app.regenerate_manifest_filename(mname, os.path.basename(mfile))

            if os.path.isdir(app.charts_dir):
                if len(os.listdir(app.charts_dir)) == 0:
                    _handle_extract_failure('tarfile contains no Helm charts.')

                tar_filelist = cutils.get_files_matching(app.charts_dir,
                                                         '.tgz')
                if not tar_filelist:
                    reason = 'tarfile contains no Helm charts of expected ' + \
                             'file extension (.tgz).'
                    _handle_extract_failure(reason)

                for p, f in tar_filelist:
                    if not cutils.extract_tarfile(
                            p, os.path.join(p, f), demote_user=True):
                        _handle_extract_failure()
        except OSError as e:
            LOG.error(e)
            _handle_extract_failure()
        finally:
            os.chown(constants.APP_INSTALL_ROOT_PATH, orig_uid, orig_gid)

    def _get_image_tags_by_path(self, path):
        """ Mine the image tags from values.yaml files in the chart directory,
            intended for custom apps.

            TODO(awang): Support custom apps to pull images from local registry
        """

        image_tags = []
        ids = []
        for r, f in cutils.get_files_matching(path, 'values.yaml'):
            with open(os.path.join(r, f), 'r') as file:
                try:
                    y = yaml.load(file)
                    ids = y["images"]["tags"].values()
                except (TypeError, KeyError):
                    pass
            image_tags.extend(ids)
        return list(set(image_tags))

    def _get_image_tags_by_charts(self, app_path, charts):
        """ Mine the image tags from the chart paths. Add the converted
            image tags to the overrides if the image tags from the chart
            paths do not exist. Intended for system app.

            The image tagging conversion(local docker registry address prepended):
            ${LOCAL_DOCKER_REGISTRY_IP}:${REGISTRY_PORT}/<image-name>
            (ie..192.168.204.2:9001/docker.io/mariadb:10.2.13)
        """

        local_docker_registry_ip = self._dbapi.address_get_by_name(
            cutils.format_address_name(constants.CONTROLLER_HOSTNAME,
                                       constants.NETWORK_TYPE_MGMT)
        ).address

        image_tags = []
        for chart in charts:
            images_charts = {}
            images_overrides = {}
            overrides = chart.namespace + '-' + chart.name + '.yaml'
            overrides_file = os.path.join(common.HELM_OVERRIDES_PATH,
                                          overrides)
            chart_name = os.path.join(app_path, chart.name)
            chart_path = os.path.join(chart_name, 'values.yaml')

            # Get the image tags from the chart path
            if os.path.exists(chart_path):
                with open(chart_path, 'r') as file:
                    try:
                        doc = yaml.load(file)
                        images_charts = doc["images"]["tags"]
                    except (TypeError, KeyError):
                        pass

            # Get the image tags from the overrides file
            if os.path.exists(overrides_file):
                with open(overrides_file, 'r') as file:
                    try:
                        y = yaml.load(file)
                        images_overrides = y["data"]["values"]["images"]["tags"]
                    except (TypeError, KeyError):
                        LOG.info("Overrides file %s has no img tags" %
                                  overrides_file)
                        pass

            # Add the converted image tags to the overrides if the images from
            # the chart path do not exist in the overrides
            tags_updated = False
            for key, image_tag in images_charts.items():
                if (key not in images_overrides and
                        not image_tag.startswith(local_docker_registry_ip)):
                    images_overrides.update(
                        {key: '{}:{}/{}'.format(local_docker_registry_ip, common.REGISTRY_PORT, image_tag)})
                    tags_updated = True

            if tags_updated:
                with open(overrides_file, 'w') as file:
                    try:
                        if "images" not in y["data"]["values"]:
                            file.seek(0)
                            file.truncate()
                            y["data"]["values"]["images"] = {"tags": images_overrides}
                        else:
                            y["data"]["values"]["images"]["tags"] = images_overrides

                        yaml.safe_dump(y, file, explicit_start=True,
                                       default_flow_style=False)
                        LOG.info("Overrides file %s updated with new image tags" %
                                  overrides_file)
                    except (TypeError, KeyError):
                        LOG.error("Overrides file %s fails to update" %
                                   overrides_file)

            if images_overrides:
                image_tags.extend(images_overrides.values())

        return list(set(image_tags))

    def _register_embedded_images(self, app):
        """
        TODO(tngo): When we're ready to support air-gap scenario and private
        images, the following need to be done:
            a. load the embedded images
            b. tag and push them to the docker registery on the controller
            c. find image tag IDs in each chart and replace their values with
               new tags. Alternatively, document the image tagging convention
               ${MGMT_FLOATING_IP}:${REGISTRY_PORT}/<image-name>
               (e.g. 192.168.204.2:9001/prom/mysqld-exporter)
               to be referenced in the application Helm charts.
        """
        raise exception.KubeAppApplyFailure(
            name=app.name,
            reason="embedded images are not yet supported.")

    def _save_images_list(self, app):
        # Extract the list of images from the charts and overrides where
        # applicable. Save the list to the same location as the armada manifest
        # so it can be sync'ed.
        if app.system_app:
            LOG.info("Generating application overrides...")
            self._helm.generate_helm_application_overrides(
                app.name, cnamespace=None, armada_format=True, combined=True)
            app.charts = self._get_list_of_charts(app.armada_mfile_abs)
            # Get the list of images from the updated images overrides
            images_to_download = self._get_image_tags_by_charts(
                app.charts_dir, app.charts)
        else:
            # For custom apps, mine image tags from application path
            images_to_download = self._get_image_tags_by_path(app.path)

        if not images_to_download:
            # TODO(tngo): We may want to support the deployment of apps that
            # set up resources only in the future. In which case, generate
            # an info log and let it advance to the next step.
            raise exception.KubeAppUploadFailure(
                name=app.name,
                reason="charts specify no docker images.")

        with open(app.imgfile_abs, 'wb') as f:
            yaml.safe_dump(images_to_download, f, explicit_start=True,
                           default_flow_style=False)

    def _retrieve_images_list(self, app_images_file):
        with open(app_images_file, 'rb') as f:
            images_list = yaml.load(f)
        return images_list

    def _download_images(self, app):
        if os.path.isdir(app.images_dir):
            return self._register_embedded_images(app)

        if app.system_app:
            # Some images could have been overwritten via user overrides
            # between upload and apply, or between applies. Refresh the
            # saved images list.
            saved_images_list = self._retrieve_images_list(app.imgfile_abs)
            combined_images_list = list(saved_images_list)
            combined_images_list.extend(
                self._get_image_tags_by_charts(app.charts_dir, app.charts))
            images_to_download = list(set(combined_images_list))
            if saved_images_list != images_to_download:
                with open(app.imgfile_abs, 'wb') as f:
                    yaml.safe_dump(images_to_download, f, explicit_start=True,
                                   default_flow_style=False)
        else:
            images_to_download = self._retrieve_images_list(app.imgfile_abs)

        total_count = len(images_to_download)
        threads = min(MAX_DOWNLOAD_THREAD, total_count)
        failed_downloads = []

        start = time.time()
        pool = greenpool.GreenPool(size=threads)
        for tag, rc in pool.imap(self._docker.download_an_image,
                                images_to_download):
            if not rc:
                failed_downloads.append(tag)
        elapsed = time.time() - start
        failed_count = len(failed_downloads)
        if failed_count > 0:
            raise exception.KubeAppApplyFailure(
                name=app.name,
                reason="failed to download one or more image(s).")
        else:
            LOG.info("All docker images for application %s were successfully "
                     "downloaded in %d seconds" % (app.name, elapsed))

    def _validate_helm_charts(self, app):
        failed_charts = []
        for r, f in cutils.get_files_matching(app.charts_dir, 'Chart.yaml'):
            # Eliminate redundant validation for system app
            if app.system_app and '/charts/helm-toolkit' in r:
                continue
            try:
                output = subprocess.check_output(['helm', 'lint', r])
                if "no failures" in output:
                    LOG.info("Helm chart %s validated" % os.path.basename(r))
                else:
                    LOG.error("Validation failed for helm chart %s" %
                              os.path.basename(r))
                    failed_charts.append(r)
            except Exception as e:
                raise exception.KubeAppUploadFailure(
                    name=app.name, reason=str(e))

        if len(failed_charts) > 0:
            raise exception.KubeAppUploadFailure(
                name=app.name, reason="one or more charts failed validation.")

    def _upload_helm_charts(self, app):
        # Set env path for helm-upload execution
        env = os.environ.copy()
        env['PATH'] = '/usr/local/sbin:' + env['PATH']
        charts = [os.path.join(r, f)
                  for r, f in cutils.get_files_matching(app.charts_dir, '.tgz')]

        orig_uid, orig_gid = get_app_install_root_path_ownership()
        try:
            # Temporarily change /scratch group ownership to wrs_protected
            os.chown(constants.APP_INSTALL_ROOT_PATH, orig_uid,
                     grp.getgrnam(constants.SYSINV_WRS_GRPNAME).gr_gid)
            with open(os.devnull, "w") as fnull:
                for chart in charts:
                    subprocess.check_call(['helm-upload', chart], env=env,
                                          stdout=fnull, stderr=fnull)
                    LOG.info("Helm chart %s uploaded" % os.path.basename(chart))
        except Exception as e:
            raise exception.KubeAppUploadFailure(
                name=app.name, reason=str(e))
        finally:
            os.chown(constants.APP_INSTALL_ROOT_PATH, orig_uid, orig_gid)

    def _validate_labels(self, labels):
        expr = re.compile(r'[a-z0-9]([-a-z0-9]*[a-z0-9])')
        for label in labels:
            if not expr.match(label):
                return False
        return True

    def _update_kubernetes_labels(self, hostname, label_dict):
        body = {
            'metadata': {
                'labels': {}
            }
        }
        body['metadata']['labels'].update(label_dict)
        try:
            self._kube.kube_patch_node(hostname, body)
        except exception.K8sNodeNotFound:
            pass

    def _assign_host_labels(self, hosts, labels):
        for host in hosts:
            if host.administrative != constants.ADMIN_LOCKED:
                continue
            for label_str in labels:
                k, v = label_str.split('=')
                try:
                    self._dbapi.label_create(
                        host.id, {'host_id': host.id,
                                  'label_key': k,
                                  'label_value': v})
                except exception.HostLabelAlreadyExists:
                    pass
            label_dict = {k: v for k, v in (i.split('=') for i in labels)}
            self._update_kubernetes_labels(host.hostname, label_dict)

    def _find_label(self, host_uuid, label_str):
        host_labels = self._dbapi.label_get_by_host(host_uuid)
        for label_obj in host_labels:
            if label_str == label_obj.label_key + '=' + label_obj.label_value:
                return label_obj
        return None

    def _remove_host_labels(self, hosts, labels):
        for host in hosts:
            if host.administrative != constants.ADMIN_LOCKED:
                continue
            null_labels = {}
            for label_str in labels:
                lbl_obj = self._find_label(host.uuid, label_str)
                if lbl_obj:
                    self._dbapi.label_destroy(lbl_obj.uuid)
                    key = lbl_obj.label_key
                    null_labels[key] = None
            if null_labels:
                self._update_kubernetes_labels(host.hostname, null_labels)

    def _process_node_labels(self, app, op=constants.LABEL_ASSIGN_OP):
        # Node labels are host personality based and are defined in
        # metadata.yaml file in the following format:
        # labels:
        #   controller: '<label-key1>=<value>, <label-key2>=<value>, ...'
        #   compute: '<label-key1>=<value>, <label-key2>=<value>, ...'

        lfile = os.path.join(app.path, 'metadata.yaml')
        controller_labels = []
        compute_labels = []
        controller_l = compute_l = None
        controller_labels_set = set()
        compute_labels_set = set()

        if os.path.exists(lfile) and os.path.getsize(lfile) > 0:
            with open(lfile, 'r') as f:
                try:
                    y = yaml.load(f)
                    labels = y['labels']
                except KeyError:
                    raise exception.KubeAppUploadFailure(
                        name=app.name,
                        reason="labels file contains no labels.")
            for key, value in labels.items():
                if key == constants.CONTROLLER:
                    controller_l = value
                elif key == constants.WORKER:
                    compute_l = value
        else:
            if not app.system_app:
                LOG.info("Application %s does not require specific node "
                         "labeling." % app.name)
                return

        if controller_l:
            controller_labels =\
                controller_l.replace(',', ' ').split()
            controller_labels_set = set(controller_labels)
            if not self._validate_labels(controller_labels_set):
                raise exception.KubeAppUploadFailure(
                    name=app.name,
                    reason="controller labels are malformed.")

        if compute_l:
            compute_labels =\
                compute_l.replace(',', ' ').split()
            compute_labels_set = set(compute_labels)
            if not self._validate_labels(compute_labels_set):
                raise exception.KubeAppUploadFailure(
                    name=app.name,
                    reason="compute labels are malformed.")

        # Add the default labels for system app. They must exist for
        # the app manifest to be applied successfully. If the nodes have
        # been assigned these labels manually before, these
        # reassignments are simply ignored.
        if app.system_app:
            controller_labels_set.add(constants.CONTROL_PLANE_LABEL)
            compute_labels_set.add(constants.COMPUTE_NODE_LABEL)
            compute_labels_set.add(constants.OPENVSWITCH_LABEL)
            compute_labels_set.add(constants.SRIOV_LABEL)

        # Get controller host(s)
        controller_hosts =\
            self._dbapi.ihost_get_by_personality(constants.CONTROLLER)
        if constants.WORKER in controller_hosts[0].subfunctions:
            # AIO system
            labels = controller_labels_set.union(compute_labels_set)
            if op == constants.LABEL_ASSIGN_OP:
                self._assign_host_labels(controller_hosts, labels)
            elif op == constants.LABEL_REMOVE_OP:
                self._remove_host_labels(controller_hosts, labels)
        else:
            # Standard system
            compute_hosts =\
                self._dbapi.ihost_get_by_personality(constants.WORKER)
            if op == constants.LABEL_ASSIGN_OP:
                self._assign_host_labels(controller_hosts, controller_labels_set)
                self._assign_host_labels(compute_hosts, compute_labels_set)
            elif op == constants.LABEL_REMOVE_OP:
                self._remove_host_labels(controller_hosts, controller_labels_set)
                self._remove_host_labels(compute_hosts, compute_labels_set)

    def _get_list_of_charts(self, manifest_file):
        charts = []
        with open(manifest_file, 'r') as f:
            docs = yaml.load_all(f)
            for doc in docs:
                try:
                    if "armada/Chart/" in doc['schema']:
                        charts.append(Chart(
                            name=doc['data']['chart_name'],
                            namespace=doc['data']['namespace']))
                except KeyError:
                    pass
        return charts

    def _get_overrides_files(self, charts):
        """Returns list of override files or None, used in
           application-install and application-delete."""

        missing_overrides = []
        available_overrides = []
        excluded = ['helm-toolkit']

        for chart in charts:
            overrides = chart.namespace + '-' + chart.name + '.yaml'
            if chart.name in excluded:
                LOG.debug("Skipping overrides %s " % overrides)
                continue
            overrides_file = os.path.join(common.HELM_OVERRIDES_PATH,
                                          overrides)
            if not os.path.exists(overrides_file):
                missing_overrides.append(overrides_file)
            else:
                available_overrides.append(overrides_file)

                # Now handle any meta-overrides files.  These can affect
                # sections of the chart schema other than "values, and can
                # affect the chartgroup or even the manifest.
                if self._helm.generate_meta_overrides(
                        chart.name, chart.namespace):
                    overrides = chart.namespace + '-' + chart.name + \
                                '-meta' + '.yaml'
                    overrides_file = os.path.join(common.HELM_OVERRIDES_PATH,
                                                  overrides)
                    if not os.path.exists(overrides_file):
                        missing_overrides.append(overrides_file)
                    else:
                        available_overrides.append(overrides_file)

        if missing_overrides:
            LOG.error("Missing the following overrides: %s" % missing_overrides)
            return None
        return available_overrides

    def _generate_armada_overrides_str(self, overrides_files):
        return " ".join([' --values /overrides/{0}'.format(os.path.basename(i))
                        for i in overrides_files])

    def _remove_chart_overrides(self, manifest_file):
        charts = self._get_list_of_charts(manifest_file)
        for chart in charts:
            if chart.name in constants.SUPPORTED_HELM_CHARTS:
                self._helm.remove_helm_chart_overrides(chart.name,
                                                       chart.namespace)

    def _make_armada_request_with_monitor(self, app, request, overrides_str=None):
        """Initiate armada request with monitoring

        This method delegates the armada request to docker helper and starts
        a monitoring thread to persist status and progress along the way.

        :param app: application data object
        :param request: type of request (apply or delete)
        :param overrides_str: list of overrides in string format to be applied
        """

        def _get_armada_log_stats(pattern, logfile):
            """
            TODO(tngo): In the absence of an Armada API that provides the current
            status of an apply/delete manifest operation, the progress is derived
            from specific log entries extracted from the execution logs. This
            inner method is to be replaced with an official API call when
            it becomes available.
            """
            p1 = subprocess.Popen(['docker', 'exec', ARMADA_CONTAINER_NAME,
                                   'grep', pattern, logfile],
                                   stdout=subprocess.PIPE)
            p2 = subprocess.Popen(['awk', '{print $NF}'], stdin=p1.stdout,
                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            p1.stdout.close()
            result, err = p2.communicate()
            if result:
                # Strip out ANSI color code that might be in the text stream
                r = re.compile("\x1b\[[0-9;]*m")
                result = r.sub('', result)
                matches = result.split()
                num_chart_processed = len(matches)
                last_chart_processed = matches[num_chart_processed - 1]
                if '=' in last_chart_processed:
                    last_chart_processed = last_chart_processed.split('=')[1]
                return last_chart_processed, num_chart_processed

            return None, None

        def _check_progress(monitor_flag, app, pattern, logfile):
            """ Progress monitoring task, to be run in a separate thread """
            LOG.info("Starting progress monitoring thread for app %s" % app.name)
            try:
                with Timeout(INSTALLATION_TIMEOUT,
                             exception.KubeAppProgressMonitorTimeout()):
                    while True:
                        try:
                            monitor_flag.get_nowait()
                            LOG.debug("Received monitor stop signal for %s" % app.name)
                            monitor_flag.task_done()
                            break
                        except queue.Empty:
                            last, num = _get_armada_log_stats(pattern, logfile)
                            if last:
                                if app.system_app:
                                    # helm-toolkit doesn't count
                                    percent = \
                                        round(float(num) / (len(app.charts) - 1) * 100)
                                else:
                                    percent = round(float(num) / len(app.charts) * 100)
                                progress_str = 'processing chart: ' + last +\
                                    ', overall completion: ' + str(percent) + '%'
                                if app.progress != progress_str:
                                    LOG.info("%s" % progress_str)
                                    self._update_app_status(
                                        app, new_progress=progress_str)
                            greenthread.sleep(1)
            except Exception as e:
                # timeout or subprocess error
                LOG.exception(e)
            finally:
                LOG.info("Exiting progress monitoring thread for app %s" % app.name)

        # Body of the outer method
        mqueue = queue.Queue()
        rc = True
        logfile = app.name + '-' + request + '.log'
        if request == constants.APP_APPLY_OP:
            pattern = APPLY_SEARCH_PATTERN
        else:
            pattern = DELETE_SEARCH_PATTERN

        monitor = greenthread.spawn_after(1, _check_progress, mqueue, app,
                                          pattern, logfile)
        rc = self._docker.make_armada_request(request, app.armada_mfile,
                                              overrides_str, logfile)
        mqueue.put('done')
        monitor.kill()
        return rc

    def perform_app_upload(self, rpc_app, tarfile):
        """Process application upload request

        This method validates the application manifest. If Helm charts are
        included, they are validated and uploaded to local Helm repo. It also
        downloads the required docker images for custom apps during upload
        stage.

        :param rpc_app: application object in the RPC request
        :param tarfile: location of application tarfile
        """

        app = AppOperator.Application(rpc_app)
        LOG.info("Application (%s) upload started." % app.name)

        try:
            app.tarfile = tarfile

            if cutils.is_url(app.tarfile):
                self._update_app_status(
                    app, new_progress=constants.APP_PROGRESS_TARFILE_DOWNLOAD)
                downloaded_tarfile = self._download_tarfile(app)

                if downloaded_tarfile is None:
                    self._abort_operation(app, constants.APP_UPLOAD_OP)
                else:
                    app.tarfile = downloaded_tarfile

                app.downloaded_tarfile = True

            # Full extraction of application tarball at /scratch/apps.
            # Manifest file is placed under /opt/platform/armada
            # which is managed by drbd-sync and visible to Armada.
            self._update_app_status(
                app, new_progress=constants.APP_PROGRESS_EXTRACT_TARFILE)

            with self._lock:
                self._extract_tarfile(app)
            shutil.copy(app.mfile_abs, app.armada_mfile_abs)

            if not self._docker.make_armada_request('validate', app.armada_mfile):
                return self._abort_operation(app, constants.APP_UPLOAD_OP)

            self._update_app_status(
                app, new_progress=constants.APP_PROGRESS_VALIDATE_UPLOAD_CHARTS)
            if os.path.isdir(app.charts_dir):
                self._validate_helm_charts(app)
                with self._lock:
                    self._upload_helm_charts(app)

            self._save_images_list(app)
            self._update_app_status(app, constants.APP_UPLOAD_SUCCESS)
            LOG.info("Application (%s) upload completed." % app.name)
        except exception.KubeAppUploadFailure as e:
            LOG.exception(e)
            self._abort_operation(app, constants.APP_UPLOAD_OP, str(e))
        except Exception as e:
            LOG.exception(e)
            self._abort_operation(app, constants.APP_UPLOAD_OP)

    def perform_app_apply(self, rpc_app):
        """Process application install request

        This method processes node labels per configuration and invokes
        Armada to apply the application manifest.

        For OpenStack app (system app), the method generates combined
        overrides (a merge between system and user overrides if available)
        for the charts that comprise the app before downloading docker images
        and applying the manifest.

        Usage: the method can be invoked at initial install or after the
               user has either made some manual configuration changes or
               or applied (new) user overrides to some Helm chart(s) to
               correct/update a previous manifest apply.

        :param rpc_app: application object in the RPC request
        :return boolean: whether application apply was successful
        """

        app = AppOperator.Application(rpc_app)
        LOG.info("Application (%s) apply started." % app.name)

        self._process_node_labels(app)

        overrides_str = ''
        ready = True
        try:
            app.charts = self._get_list_of_charts(app.armada_mfile_abs)
            if app.system_app:
                self._update_app_status(
                    app, new_progress=constants.APP_PROGRESS_GENERATE_OVERRIDES)
                LOG.info("Generating application overrides...")
                self._helm.generate_helm_application_overrides(
                    app.name, cnamespace=None, armada_format=True,
                    combined=True)
                overrides_files = self._get_overrides_files(app.charts)
                if overrides_files:
                    LOG.info("Application overrides generated.")
                    # Ensure all chart overrides are readable by Armada
                    for file in overrides_files:
                        os.chmod(file, 0o644)
                    overrides_str =\
                        self._generate_armada_overrides_str(overrides_files)
                    self._update_app_status(
                        app, new_progress=constants.APP_PROGRESS_DOWNLOAD_IMAGES)
                    self._download_images(app)
                else:
                    ready = False
            else:
                # No support for custom app overrides at this point, just
                # download the needed images.
                self._update_app_status(
                    app, new_progress=constants.APP_PROGRESS_DOWNLOAD_IMAGES)
                self._download_images(app)

            if ready:
                self._update_app_status(
                    app, new_progress=constants.APP_PROGRESS_APPLY_MANIFEST)
                if self._make_armada_request_with_monitor(app,
                                                          constants.APP_APPLY_OP,
                                                          overrides_str):
                    self._update_app_status(app,
                                            constants.APP_APPLY_SUCCESS)
                    LOG.info("Application (%s) apply completed." % app.name)
                    return True
        except Exception as e:
            LOG.exception(e)

        # If it gets here, something went wrong
        self._abort_operation(app, constants.APP_APPLY_OP)
        return False

    def perform_app_remove(self, rpc_app):
        """Process application remove request

        This method invokes Armada to delete the application manifest.
        For system app, it also cleans up old test pods.

        :param rpc_app: application object in the RPC request
        :return boolean: whether application remove was successful
        """

        app = AppOperator.Application(rpc_app)
        LOG.info("Application (%s) remove started." % app.name)

        app.charts = self._get_list_of_charts(app.armada_mfile_abs)
        self._update_app_status(
            app, new_progress=constants.APP_PROGRESS_DELETE_MANIFEST)

        if self._make_armada_request_with_monitor(app, constants.APP_DELETE_OP):
            if app.system_app:

                # TODO convert these kubectl commands to use the k8s api
                p1 = subprocess.Popen(
                    ['kubectl', '--kubeconfig=/etc/kubernetes/admin.conf',
                     'get', 'pvc', '--no-headers', '-n', 'openstack'],
                    stdout=subprocess.PIPE)
                p2 = subprocess.Popen(['awk', '{print $3}'],
                                      stdin=p1.stdout,
                                      stdout=subprocess.PIPE)
                p3 = subprocess.Popen(
                    ['xargs', '-i', 'kubectl',
                     '--kubeconfig=/etc/kubernetes/admin.conf', 'delete',
                     'pv', '{}', '--wait=false'],
                    stdin=p2.stdout,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE)
                timer = threading.Timer(10, p3.kill)
                try:
                    timer.start()
                    p1.stdout.close()
                    p2.stdout.close()
                    out, err = p3.communicate()
                    if out and not err:
                        LOG.info("Persistent Volumes marked for deletion.")
                    else:
                        self._abort_operation(app, constants.APP_REMOVE_OP)
                        LOG.error("Failed to clean up PVs after app removal.")
                except Exception as e:
                    self._abort_operation(app, constants.APP_REMOVE_OP)
                    LOG.exception("Failed to clean up PVs after app "
                                  "removal: %s" % e)
                finally:
                    timer.cancel()

                p4 = subprocess.Popen(
                    ['kubectl', '--kubeconfig=/etc/kubernetes/admin.conf',
                     'delete', 'namespace', 'openstack'],
                    stdout=subprocess.PIPE)
                timer2 = threading.Timer(10, p4.kill)
                try:
                    timer2.start()
                    out, err = p4.communicate()
                    if out and not err:
                        LOG.info("Openstack namespace delete completed.")
                    else:
                        self._abort_operation(app, constants.APP_REMOVE_OP)
                        LOG.error("Failed to clean up openstack namespace"
                                  " after app removal.")
                except Exception as e:
                    self._abort_operation(app, constants.APP_REMOVE_OP)
                    LOG.exception("Failed to clean up openstack namespace "
                                  "after app removal: %s" % e)
                finally:
                    timer2.cancel()

            self._update_app_status(app, constants.APP_UPLOAD_SUCCESS)
            LOG.info("Application (%s) remove completed." % app.name)
            return True
        else:
            self._abort_operation(app, constants.APP_REMOVE_OP)

        return False

    def perform_app_delete(self, rpc_app):
        """Process application remove request

        This method removes the application entry from the database and
        performs cleanup which entails removing node labels where applicable
        and purge all application files from the system.

        :param rpc_app: application object in the RPC request
        """

        app = AppOperator.Application(rpc_app)
        try:
            self._dbapi.kube_app_destroy(app.name)
            self._cleanup(app)
            LOG.info("Application (%s) has been purged from the system." %
                     app.name)
            msg = None
        except Exception as e:
            # Possible exceptions are KubeAppDeleteFailure,
            # OSError and unexpectedly KubeAppNotFound
            LOG.exception(e)
            msg = str(e)
        return msg

    class Application(object):
        """ Data object to encapsulate all data required to
            support application related operations.
        """

        def __init__(self, rpc_app):
            self._kube_app = rpc_app
            self.path = os.path.join(constants.APP_INSTALL_PATH,
                                     self._kube_app.get('name'))
            self.charts_dir = os.path.join(self.path, 'charts')
            self.images_dir = os.path.join(self.path, 'images')
            self.tarfile = None
            self.downloaded_tarfile = False
            self.system_app =\
                (self._kube_app.get('name') == constants.HELM_APP_OPENSTACK)

            self.armada_mfile = generate_armada_manifest_filename(
                self._kube_app.get('name'),
                self._kube_app.get('manifest_file'))
            self.armada_mfile_abs = generate_armada_manifest_filename_abs(
                self._kube_app.get('name'),
                self._kube_app.get('manifest_file'))
            self.mfile_abs = generate_manifest_filename_abs(
                self._kube_app.get('name'),
                self._kube_app.get('manifest_file'))
            self.imgfile_abs = generate_images_filename_abs(
                self._kube_app.get('name'))

            self.charts = []

        @property
        def name(self):
            return self._kube_app.get('name')

        @property
        def status(self):
            return self._kube_app.get('status')

        @property
        def progress(self):
            return self._kube_app.get('progress')

        def update_status(self, new_status, new_progress):
            self._kube_app.status = new_status
            if new_progress:
                self._kube_app.progress = new_progress
            self._kube_app.save()

        def regenerate_manifest_filename(self, new_mname, new_mfile):
            self._kube_app.manifest_name = new_mname
            self._kube_app.manifest_file = new_mfile
            self.armada_mfile = generate_armada_manifest_filename(
                self.name, new_mfile)
            self.armada_mfile_abs = generate_armada_manifest_filename_abs(
                self.name, new_mfile)
            self.mfile_abs = generate_manifest_filename_abs(
                self.name, new_mfile)


class DockerHelper(object):
    """ Utility class to encapsulate Docker related operations """

    def _start_armada_service(self, client):
        try:
            container = client.containers.get(ARMADA_CONTAINER_NAME)
            if container.status != 'running':
                LOG.info("Restarting Armada service...")
                container.restart()
            return container
        except Exception:
            LOG.info("Starting Armada service...")
            try:
                # First make kubernetes config accessible to Armada. This
                # is a work around the permission issue in Armada container.
                kube_config = os.path.join(constants.APP_SYNCED_DATA_PATH,
                                           'admin.conf')
                shutil.copy('/etc/kubernetes/admin.conf', kube_config)
                os.chown(kube_config, 1000, grp.getgrnam("wrs").gr_gid)

                overrides_dir = common.HELM_OVERRIDES_PATH
                manifests_dir = constants.APP_SYNCED_DATA_PATH
                LOG.info("kube_config=%s, manifests_dir=%s, "
                         "overrides_dir=%s." % (kube_config, manifests_dir,
                                                overrides_dir))
                binds = {
                    kube_config: {'bind': '/armada/.kube/config', 'mode': 'ro'},
                    manifests_dir: {'bind': '/manifests', 'mode': 'ro'},
                    overrides_dir: {'bind': '/overrides', 'mode': 'ro'}}

                container = client.containers.run(
                    CONF.armada_image_tag,
                    name=ARMADA_CONTAINER_NAME,
                    detach=True,
                    volumes=binds,
                    restart_policy={'Name': 'always'},
                    command=None)
                LOG.info("Armada service started!")
                return container
            except OSError as oe:
                LOG.error("Unable to make kubernetes config accessible to "
                          "armada: %s" % oe)
            except Exception as e:
                # Possible docker exceptions are: RuntimeError, ContainerError,
                # ImageNotFound and APIError
                LOG.error("Docker error while launching Armada container: %s", e)
                os.unlink(kube_config)
            return None

    def make_armada_request(self, request, manifest_file, overrides_str='',
                            logfile=None):

        if logfile is None:
            logfile = request + '.log'

        rc = True

        try:
            client = docker.from_env(timeout=INSTALLATION_TIMEOUT)
            armada_svc = self._start_armada_service(client)
            if armada_svc:
                if request == 'validate':
                    cmd = 'armada validate ' + manifest_file
                    (exit_code, exec_logs) = armada_svc.exec_run(cmd)
                    if exit_code == 0:
                        LOG.info("Manifest file %s was successfully validated." %
                                 manifest_file)
                    else:
                        rc = False
                        if exit_code == CONTAINER_ABNORMAL_EXIT_CODE:
                            LOG.error("Failed to validate application manifest %s. "
                                      "Armada service has exited abnormally." %
                                      manifest_file)
                        else:
                            LOG.error("Failed to validate application manifest "
                                      "%s: %s." % (manifest_file, exec_logs))
                elif request == constants.APP_APPLY_OP:
                    cmd = "/bin/bash -c 'armada apply --debug " + manifest_file +\
                          overrides_str + " | tee " + logfile + "'"
                    LOG.info("Armada apply command = %s" % cmd)
                    (exit_code, exec_logs) = armada_svc.exec_run(cmd)
                    if exit_code == 0:
                        if ARMADA_MANIFEST_APPLY_SUCCESS_MSG in exec_logs:
                            LOG.info("Application manifest %s was successfully "
                                     "applied/re-applied." % manifest_file)
                        else:
                            rc = False
                            LOG.error("Received a false positive response from "
                                      "Docker/Armada. Failed to apply application "
                                      "manifest %s: %s." % (manifest_file, exec_logs))
                    else:
                        rc = False
                        if exit_code == CONTAINER_ABNORMAL_EXIT_CODE:
                            LOG.error("Failed to apply application manifest %s. "
                                      "Armada service has exited abnormally." %
                                      manifest_file)
                        else:
                            LOG.error("Failed to apply application manifest %s: "
                                      "%s." % (manifest_file, exec_logs))
                elif request == constants.APP_DELETE_OP:
                    cmd = "/bin/bash -c 'armada delete --debug --manifest " +\
                          manifest_file + " | tee " + logfile + "'"
                    (exit_code, exec_logs) = armada_svc.exec_run(cmd)
                    if exit_code == 0:
                        LOG.info("Application charts were successfully "
                                 "deleted.")
                    else:
                        rc = False
                        if exit_code == CONTAINER_ABNORMAL_EXIT_CODE:
                            LOG.error("Failed to delete application manifest %s. "
                                      "Armada service has exited abnormally." %
                                      manifest_file)
                        else:
                            LOG.error("Failed to delete application manifest %s: "
                                      "%s" % (manifest_file, exec_logs))
                else:
                    rc = False
                    LOG.error("Unsupported armada request: %s." % request)
            else:
                # Armada sevice failed to start/restart
                rc = False
        except Exception as e:
            # Failed to get a docker client
            rc = False
            LOG.error("Armada request %s for manifest %s failed: %s " %
                      (request, manifest_file, e))
        return rc

    def download_an_image(self, loc_img_tag):

        rc = True
        start = time.time()
        try:
            # Pull image from local docker registry
            LOG.info("Image %s download started from local registry" % loc_img_tag)
            client = docker.APIClient(timeout=INSTALLATION_TIMEOUT)
            client.pull(loc_img_tag)
        except docker.errors.NotFound:
            try:
                # Image is not available in local docker registry, get the image
                # from the public registry and push to the local registry
                LOG.info("Image %s is not available in local registry, "
                         "download started from public registry" % loc_img_tag)
                pub_img_tag = loc_img_tag[1 + loc_img_tag.find('/'):]
                client.pull(pub_img_tag)
                client.tag(pub_img_tag, loc_img_tag)
                client.push(loc_img_tag)
            except Exception as e:
                rc = False
                LOG.error("Image %s download failed from public registry: %s" % (pub_img_tag, e))
        except Exception as e:
            rc = False
            LOG.error("Image %s download failed from local registry: %s" % (loc_img_tag, e))
        elapsed_time = time.time() - start

        if rc:
            LOG.info("Image %s download succeeded in %d seconds" %
                     (loc_img_tag, elapsed_time))
        return loc_img_tag, rc
# vim: tabstop=4 shiftwidth=4 softtabstop=4 # # Copyright (c) 2018 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # # All Rights Reserved. # System Inventory Kubernetes Application Operator. # Log and config # Constants # Helper functions Class to encapsulate Kubernetes App operations for System Inventory " Remove application directories and override files Persist new app status # One time set up of Armada manifest path for the system # Temporarily change /scratch group ownership to wrs_protected # Extract the tarfile as sysinv user # Save the official manifest file info. They will be persisted # in the next status update Mine the image tags from values.yaml files in the chart directory,             intended for custom apps.              TODO(awang): Support custom apps to pull images from local registry Mine the image tags from the chart paths. Add the converted             image tags to the overrides if the image tags from the chart             paths do not exist. Intended for system app.              The image tagging conversion(local docker registry address prepended):             ${LOCAL_DOCKER_REGISTRY_IP}:${REGISTRY_PORT}/<image-name>             (ie..192.168.204.2:9001/docker.io/mariadb:10.2.13) # Get the image tags from the chart path # Get the image tags from the overrides file # Add the converted image tags to the overrides if the images from # the chart path do not exist in the overrides TODO(tngo): When we're ready to support air-gap scenario and private         images, the following need to be done:             a. load the embedded images             b. tag and push them to the docker registery on the controller             c. find image tag IDs in each chart and replace their values with                new tags. Alternatively, document the image tagging convention                ${MGMT_FLOATING_IP}:${REGISTRY_PORT}/<image-name>                (e.g. 192.168.204.2:9001/prom/mysqld-exporter)                to be referenced in the application Helm charts. # Extract the list of images from the charts and overrides where # applicable. Save the list to the same location as the armada manifest # so it can be sync'ed. # Get the list of images from the updated images overrides # For custom apps, mine image tags from application path # TODO(tngo): We may want to support the deployment of apps that # set up resources only in the future. In which case, generate # an info log and let it advance to the next step. # Some images could have been overwritten via user overrides # between upload and apply, or between applies. Refresh the # saved images list. # Eliminate redundant validation for system app # Set env path for helm-upload execution # Temporarily change /scratch group ownership to wrs_protected # Node labels are host personality based and are defined in # metadata.yaml file in the following format: # labels: #   controller: '<label-key1>=<value>, <label-key2>=<value>, ...' #   compute: '<label-key1>=<value>, <label-key2>=<value>, ...' # Add the default labels for system app. They must exist for # the app manifest to be applied successfully. If the nodes have # been assigned these labels manually before, these # reassignments are simply ignored. # Get controller host(s) # AIO system # Standard system Returns list of override files or None, used in            application-install and application-delete. # Now handle any meta-overrides files.  These can affect # sections of the chart schema other than "values, and can # affect the chartgroup or even the manifest. Initiate armada request with monitoring          This method delegates the armada request to docker helper and starts         a monitoring thread to persist status and progress along the way.          :param app: application data object         :param request: type of request (apply or delete)         :param overrides_str: list of overrides in string format to be applied TODO(tngo): In the absence of an Armada API that provides the current             status of an apply/delete manifest operation, the progress is derived             from specific log entries extracted from the execution logs. This             inner method is to be replaced with an official API call when             it becomes available. # Strip out ANSI color code that might be in the text stream Progress monitoring task, to be run in a separate thread # helm-toolkit doesn't count # timeout or subprocess error # Body of the outer method Process application upload request          This method validates the application manifest. If Helm charts are         included, they are validated and uploaded to local Helm repo. It also         downloads the required docker images for custom apps during upload         stage.          :param rpc_app: application object in the RPC request         :param tarfile: location of application tarfile # Full extraction of application tarball at /scratch/apps. # Manifest file is placed under /opt/platform/armada # which is managed by drbd-sync and visible to Armada. Process application install request          This method processes node labels per configuration and invokes         Armada to apply the application manifest.          For OpenStack app (system app), the method generates combined         overrides (a merge between system and user overrides if available)         for the charts that comprise the app before downloading docker images         and applying the manifest.          Usage: the method can be invoked at initial install or after the                user has either made some manual configuration changes or                or applied (new) user overrides to some Helm chart(s) to                correct/update a previous manifest apply.          :param rpc_app: application object in the RPC request         :return boolean: whether application apply was successful # Ensure all chart overrides are readable by Armada # No support for custom app overrides at this point, just # download the needed images. # If it gets here, something went wrong Process application remove request          This method invokes Armada to delete the application manifest.         For system app, it also cleans up old test pods.          :param rpc_app: application object in the RPC request         :return boolean: whether application remove was successful # TODO convert these kubectl commands to use the k8s api Process application remove request          This method removes the application entry from the database and         performs cleanup which entails removing node labels where applicable         and purge all application files from the system.          :param rpc_app: application object in the RPC request # Possible exceptions are KubeAppDeleteFailure, # OSError and unexpectedly KubeAppNotFound Data object to encapsulate all data required to             support application related operations. Utility class to encapsulate Docker related operations # First make kubernetes config accessible to Armada. This # is a work around the permission issue in Armada container. # Possible docker exceptions are: RuntimeError, ContainerError, # ImageNotFound and APIError # Armada sevice failed to start/restart # Failed to get a docker client # Pull image from local docker registry # Image is not available in local docker registry, get the image # from the public registry and push to the local registry