class Job(object):  # pylint: disable=too-many-instance-attributes
    """
    Populated by the parser, the Job contains all of the
    Actions and their pipelines.
    parameters provides the immutable data about this job:
        action_timeout
        job_name
        priority
        device_type (mapped to target by scheduler)
        yaml_line
        logging_level
        job_timeout
    Job also provides the primary access to the Device.
    The NewDevice class only loads the specific configuration of the
    device for this job - one job, one device.
    """

    def __init__(self, job_id, parameters, logger):  # pylint: disable=too-many-arguments
        self.job_id = job_id
        self.logger = logger
        self.device = None
        self.parameters = parameters
        self.__context__ = PipelineContext()
        self.pipeline = None
        self.connection = None
        self.triggers = []  # actions can add trigger strings to the run a diagnostic
        self.diagnostics = [
            DiagnoseNetwork,
        ]
        self.timeout = None
        self.protocols = []
        self.compatibility = 2
        # Was the job cleaned
        self.cleaned = False
        # Root directory for the job tempfiles
        self.tmp_dir = None
        # override in use
        self.base_overrides = {}
        self.started = False

    @property
    def context(self):
        return self.__context__.pipeline_data

    @context.setter
    def context(self, data):
        self.__context__.pipeline_data.update(data)

    def diagnose(self, trigger):
        """
        Looks up the class to execute to diagnose the problem described by the
         specified trigger.
        """
        trigger_tuples = [(cls.trigger(), cls) for cls in self.diagnostics]
        for diagnostic in trigger_tuples:
            if trigger is diagnostic[0]:
                return diagnostic[1]()
        return None

    def describe(self):
        return {'device': self.device,
                'job': self.parameters,
                'compatibility': self.compatibility,
                'pipeline': self.pipeline.describe()}

    def mkdtemp(self, action_name, override=None):
        """
        Create a tmp directory in DISPATCHER_DOWNLOAD_DIR/{job_id}/ because
        this directory will be removed when the job finished, making cleanup
        easier.
        """
        if override is None:
            if self.tmp_dir is None:
                create_base_dir = True
                base_dir = DISPATCHER_DOWNLOAD_DIR
            else:
                create_base_dir = False
                base_dir = self.tmp_dir
        else:
            if override in self.base_overrides:
                create_base_dir = False
                base_dir = self.base_overrides[override]
            else:
                create_base_dir = True
                base_dir = override

        if create_base_dir:
            # Try to create the directory.
            base_dir = os.path.join(base_dir, str(self.job_id))
            try:
                os.makedirs(base_dir, mode=0o755)
            except OSError as exc:
                if exc.errno != errno.EEXIST:
                    # When running unit tests
                    base_dir = tempfile.mkdtemp(prefix='pipeline-')
                    atexit.register(shutil.rmtree, base_dir, ignore_errors=True)
            # Save the path for the next calls (only if that's not an override)
            if override is None:
                self.tmp_dir = base_dir
            else:
                self.base_overrides[override] = base_dir

        # Create the sub-directory
        tmp_dir = tempfile.mkdtemp(prefix=action_name + '-', dir=base_dir)
        os.chmod(tmp_dir, 0o755)
        return tmp_dir

    def _validate(self):
        """
        Validate the pipeline and raise an exception (that inherit from
        LAVAError) if it fails.
        """
        self.logger.info("Start time: %s (UTC)", pytz.utc.localize(datetime.datetime.utcnow()))
        for protocol in self.protocols:
            try:
                protocol.configure(self.device, self)
            except LAVAError:
                self.logger.error("Configuration failed for protocol %s", protocol.name)
                raise
            except Exception as exc:
                self.logger.error("Configuration failed for protocol %s", protocol.name)
                self.logger.exception(traceback.format_exc())
                raise LAVABug(exc)

            if not protocol.valid:
                msg = "protocol %s has errors: %s" % (protocol.name, protocol.errors)
                self.logger.exception(msg)
                raise JobError(msg)

        # Check that namespaces are used in all actions or none
        namespaces = set()
        for action in self.parameters["actions"]:
            action_name = list(action.keys())[0]
            namespaces.add(action[action_name]["namespace"])

        # 'common' is a reserved namespace that should not be present with
        # other namespaces.
        if len(namespaces) > 1 and 'common' in namespaces:
            msg = "'common' is a reserved namespace that should not be present with other namespaces"
            self.logger.error(msg)
            self.logger.debug("Namespaces: %s", ", ".join(namespaces))
            raise JobError(msg)

        # validate the pipeline
        self.pipeline.validate_actions()

    def validate(self):
        """
        Public wrapper for the pipeline validation.
        Send a "fail" results if needed.
        """
        label = "lava-dispatcher, installed at version: %s" % debian_package_version(split=False)
        self.logger.info(label)
        self.logger.info("start: 0 validate")
        start = time.time()

        success = False
        try:
            self._validate()
        except LAVAError as exc:
            raise
        except Exception as exc:
            # provide useful info on command line, e.g. failed unit tests.
            self.logger.exception(traceback.format_exc())
            raise LAVABug(exc)
        else:
            success = True
        finally:
            if not success:
                self.cleanup(connection=None)
            self.logger.info("validate duration: %.02f", time.time() - start)
            self.logger.results({"definition": "lava",
                                 "case": "validate",
                                 "result": "pass" if success else "fail"})

    def _run(self):
        """
        Run the pipeline under the run() wrapper that will catch the exceptions
        """
        self.started = True

        # Setup the protocols
        for protocol in self.protocols:
            try:
                protocol.set_up()
            except LAVAError:
                raise
            except Exception as exc:
                self.logger.error("Unable to setup the protocols")
                self.logger.exception(traceback.format_exc())
                raise LAVABug(exc)

            if not protocol.valid:
                msg = "protocol %s has errors: %s" % (protocol.name, protocol.errors)
                self.logger.exception(msg)
                raise JobError(msg)

        # Run the pipeline and wait for exceptions
        with self.timeout() as max_end_time:
            self.pipeline.run_actions(self.connection, max_end_time)

    def run(self):
        """
        Top level routine for the entire life of the Job, using the job level timeout.
        Python only supports one alarm on SIGALRM - any Action without a connection
        will have a default timeout which will use SIGALRM. So the overarching Job timeout
        can only stop processing actions if the job wide timeout is exceeded.
        """
        try:
            self._run()
        finally:
            # Cleanup now
            self.cleanup(self.connection)

    def cleanup(self, connection):
        if self.cleaned:
            self.logger.info("Cleanup already called, skipping")
            return

        # exit out of the pipeline & run the Finalize action to close the
        # connection and poweroff the device (the cleanup action will do that
        # for us)
        self.logger.info("Cleaning after the job")
        self.pipeline.cleanup(connection)

        for tmp_dir in self.base_overrides.values():
            self.logger.info("Override tmp directory removed at %s", tmp_dir)
            try:
                shutil.rmtree(tmp_dir)
            except OSError as exc:
                if exc.errno != errno.ENOENT:
                    self.logger.error("Unable to remove the directory: %s",
                                      exc.strerror)

        if self.tmp_dir is not None:
            self.logger.info("Root tmp directory removed at %s", self.tmp_dir)
            try:
                shutil.rmtree(self.tmp_dir)
            except OSError as exc:
                if exc.errno != errno.ENOENT:
                    self.logger.error("Unable to remove the directory: %s",
                                      exc.strerror)

        # Mark cleanup as done to avoid calling it many times
        self.cleaned = True class Pipeline(object):  # pylint: disable=too-many-instance-attributes
    """
    Pipelines ensure that actions are run in the correct sequence whilst
    allowing for retries and other requirements.
    When an action is added to a pipeline, the level of that action within
    the overall job is set along with the formatter and output filename
    of the per-action log handler.
    """
    def __init__(self, parent=None, job=None, parameters=None):
        self.actions = []
        self.parent = None
        self.parameters = {} if parameters is None else parameters
        self.job = job
        if parent is not None:
            # parent must be an Action
            if not isinstance(parent, Action):
                raise LAVABug("Internal pipelines need an Action as a parent")
            if not parent.level:
                raise LAVABug("Tried to create a pipeline using a parent action with no level set.")
            self.parent = parent

    def _check_action(self, action):  # pylint: disable=no-self-use
        if not action or not issubclass(type(action), Action):
            raise LAVABug("Only actions can be added to a pipeline: %s" % action)
        # if isinstance(action, DiagnosticAction):
        #     raise LAVABug("Diagnostic actions need to be triggered, not added to a pipeline.")
        if not action:
            raise LAVABug("Unable to add empty action to pipeline")

    def add_action(self, action, parameters=None):  # pylint: disable=too-many-branches
        self._check_action(action)
        self.actions.append(action)
        # FIXME: if this is only happening in unit test, this has to be fixed later on
        # should only be None inside the unit tests
        action.job = self.job
        if self.parent:  # action
            self.parent.pipeline = self
            action.level = "%s.%s" % (self.parent.level, len(self.actions))
            action.section = self.parent.section
        else:
            action.level = "%s" % (len(self.actions))

        # Use the pipeline parameters if the function was walled without
        # parameters.
        if parameters is None:
            parameters = self.parameters
        # if the action has an internal pipeline, initialise that here.
        action.populate(parameters)

        # Compute the timeout
        timeouts = []
        # FIXME: Only needed for the auto-tests
        if self.job is not None:
            if self.job.device is not None:
                # First, the device level overrides
                timeouts.append(self.job.device.get('timeouts', {}))
            # Then job level overrides
            timeouts.append(self.job.parameters.get('timeouts', {}))

        # pylint: disable=invalid-name

        def dict_merge_get(dicts, key):
            value = None
            for d in dicts:
                value = d.get(key, value)
            return value

        def subdict_merge_get(dicts, key, subkey):
            value = None
            for d in dicts:
                value = d.get(key, {}).get(subkey, value)
            return value

        # Set the timeout. The order is:
        # 1/ the global action timeout
        # 2/ the individual action timeout
        # 3/ the action block timeout
        # pylint: disable=protected-access
        action._override_action_timeout(dict_merge_get(timeouts, 'action'))
        action._override_action_timeout(subdict_merge_get(timeouts, 'actions', action.name))
        action._override_action_timeout(parameters.get('timeout', None))

        action._override_connection_timeout(dict_merge_get(timeouts, 'connection'))
        action._override_connection_timeout(subdict_merge_get(timeouts, 'connections', action.name))

        action.parameters = parameters

    def describe(self, verbose=True):
        """
        Describe the current pipeline, recursing through any
        internal pipelines.
        :return: a recursive dictionary
        """
        desc = []
        for action in self.actions:
            if verbose:
                current = action.explode()
            else:
                cls = str(type(action))[8:-2].replace('lava_dispatcher.', '')
                current = {'class': cls, 'name': action.name}
            if action.pipeline is not None:
                current['pipeline'] = action.pipeline.describe(verbose)
            desc.append(current)
        return desc

    @property
    def errors(self):
        sub_action_errors = [a.errors for a in self.actions]
        if not sub_action_errors:  # allow for jobs with no actions
            return []
        return reduce(lambda a, b: a + b, sub_action_errors)

    def validate_actions(self):
        for action in self.actions:
            action.validate()

        # If this is the root pipeline, raise the errors
        if self.parent is None and self.errors:
            raise JobError("Invalid job data: %s\n" % self.errors)

    def cleanup(self, connection):
        """
        Recurse through internal pipelines running action.cleanup(),
        in order of the pipeline levels.
        """
        for child in self.actions:
            try:
                child.cleanup(connection)
            except Exception as exc:
                # Just log the exception and continue the cleanup
                child.logger.error("Failed to clean after action '%s': %s", child.name, str(exc))
                child.logger.exception(traceback.format_exc())

    def _diagnose(self, connection):
        """
        Pipeline Jobs have a number of Diagnostic classes registered - all
        supported DiagnosticAction classes should be registered with the Job.
        If an Action.run() function reports a JobError or InfrastructureError,
        the Pipeline calls Job.diagnose(). The job iterates through the DiagnosticAction
        classes declared in the diagnostics list, checking if the trigger classmethod
        matches the requested complaint. Matching diagnostics are run using the current Connection.
        Actions generate a complaint by appending the return of the trigger classmethod
        to the triggers list of the Job. This can be done at any point prior to the
        exception being raised in the run function.
        The trigger list is cleared after each diagnostic operation is complete.
        """
        for complaint in self.job.triggers:
            diagnose = self.job.diagnose(complaint)
            if diagnose:
                connection = diagnose.run(connection, None)
            else:
                raise LAVABug("No diagnosis for trigger %s" % complaint)
        self.job.triggers = []
        # Diagnosis is not allowed to alter the connection, do not use the return value.
        return None

    def run_actions(self, connection, max_end_time, args=None):  # pylint: disable=too-many-branches,too-many-statements,too-many-locals
        for action in self.actions:
            failed = False
            namespace = action.parameters.get('namespace', 'common')
            # Begin the action
            try:
                with action.timeout(max_end_time) as action_max_end_time:
                    # Add action start timestamp to the log message
                    # Log in INFO for root actions and in DEBUG for the other actions
                    timeout = seconds_to_str(action_max_end_time - action.timeout.start)
                    msg = 'start: %s %s (timeout %s) [%s]' % (
                        action.level, action.name, timeout, namespace)
                    if self.parent is None:
                        action.logger.info(msg)
                    else:
                        action.logger.debug(msg)

                    new_connection = action.run(connection,
                                                action_max_end_time, args)
            except LAVAError as exc:
                action.logger.exception(str(exc))
                # allows retries without setting errors, which make the job incomplete.
                failed = True
                action.results = {'fail': str(exc)}
                self._diagnose(connection)
                raise
            except Exception as exc:
                action.logger.exception(traceback.format_exc())
                # allows retries without setting errors, which make the job incomplete.
                failed = True
                action.results = {'fail': str(exc)}
                # Raise a LAVABug that will be correctly classified later
                raise LAVABug(str(exc))
            finally:
                # Add action end timestamp to the log message
                duration = round(action.timeout.elapsed_time)
                msg = "end: %s %s (duration %s) [%s]" % (
                    action.level, action.name,
                    seconds_to_str(duration), namespace)
                if self.parent is None:
                    action.logger.info(msg)
                else:
                    action.logger.debug(msg)
                # set results including retries and failed actions
                action.log_action_results(fail=failed)

            if new_connection:
                connection = new_connection
        return connection class Timeout(object):
    """
    The Timeout class is a declarative base which any actions can use. If an Action has
    a timeout, that timeout name and the duration will be output as part of the action
    description and the timeout is then exposed as a modifiable value via the device_type,
    device or even job inputs. (Some timeouts may be deemed "protected" which may not be
    altered by the job. All timeouts are subject to a hardcoded maximum duration which
    cannot be exceeded by device_type, device or job input, only by the Action initialising
    the timeout.
    If a connection is set, this timeout is used per pexpect operation on that connection.
    If a connection is not set, this timeout applies for the entire run function of the action.
    """
    def __init__(self, name, duration=ACTION_TIMEOUT, exception=JobError):
        self.name = name
        self.start = 0
        self.elapsed_time = -1
        self.duration = duration  # Actions can set timeouts higher than the clamp.
        self.exception = exception

    @classmethod
    def default_duration(cls):
        return ACTION_TIMEOUT

    @classmethod
    def parse(cls, data):
        """
        Parsed timeouts can be set in device configuration or device_type configuration
        and can therefore exceed the clamp.
        """
        if not isinstance(data, dict):
            raise ConfigurationError("Invalid timeout data")
        duration = datetime.timedelta(days=data.get('days', 0),
                                      hours=data.get('hours', 0),
                                      minutes=data.get('minutes', 0),
                                      seconds=data.get('seconds', 0))
        if not duration:
            return Timeout.default_duration()
        return int(duration.total_seconds())

    def _timed_out(self, signum, frame):  # pylint: disable=unused-argument
        duration = int(time.time() - self.start)
        raise self.exception("%s timed out after %s seconds" % (self.name, duration))

    @contextmanager
    def __call__(self, action_max_end_time=None):
        self.start = time.time()
        if action_max_end_time is None:
            # action_max_end_time is None when cleaning the pipeline after a
            # timeout.
            # In this case, the job timeout is not taken into account.
            max_end_time = self.start + self.duration
        else:
            max_end_time = min(action_max_end_time, self.start + self.duration)

        duration = int(max_end_time - self.start)
        if duration <= 0:
            # If duration is lower than 0, then the timeout should be raised now.
            # Calling signal.alarm in this case will only deactivate the alarm
            # (by passing 0 or the unsigned value).
            # Deactivate any previous alarm and set elapse_time prior to raise
            signal.alarm(0)
            self.elapsed_time = 0
            self._timed_out(None, None)

        signal.signal(signal.SIGALRM, self._timed_out)
        signal.alarm(duration)

        try:
            yield max_end_time
        finally:
            # clear the timeout alarm, the action has returned
            signal.alarm(0)
            self.elapsed_time = time.time() - self.start class JobError(LAVAError):
    """
    An Error arising from the information supplied as part of the TestJob
    e.g. HTTP404 on a file to be downloaded as part of the preparation of
    the TestJob or a download which results in a file which tar or gzip
    does not recognise.
    """
    error_help = "JobError: Your job cannot terminate cleanly."
    error_type = "Job" class Deployment(object):
    """
    Deployment is a strategy class which aggregates Actions
    until the request from the YAML can be validated or rejected.
    Translates the parsed pipeline into Actions and populates
    each Action with parameters.
    Primary purpose of the Deployment class is to allow the
    parser to select the correct deployment before initialising
    any Actions.
    """

    priority = 0
    action_type = 'deploy'
    compatibility = 0
    name = 'unset-deployment-name'

    def __init__(self, parent):
        self.__parameters__ = {}
        self.pipeline = parent
        self.job = parent.job
        if self.compatibility > self.job.compatibility:
            self.job.compatibility = self.compatibility

    @property
    def parameters(self):
        """
        All data which this action needs to have available for
        the prepare, run or post_process functions needs to be
        set as a parameter. The parameters will be validated
        during pipeline creation.
        This allows all pipelines to be fully described, including
        the parameters supplied to each action, as well as supporting
        tests on each parameter (like 404 or bad formatting) during
        validation of each action within a pipeline.
        Parameters are static, internal data within each action
        copied directly from the YAML or Device configuration.
        Dynamic data is held in the context available via the parent Pipeline()
        """
        return self.__parameters__

    @parameters.setter
    def parameters(self, data):
        self.__parameters__.update(data)

    @classmethod
    def accepts(cls, device, parameters):  # pylint: disable=unused-argument
        """
        Returns True if this deployment strategy can be used the the
        given device and details of an image in the parameters.

        Must be implemented by subclasses.
        """
        return NotImplementedError("accepts %s" % cls)

    @classmethod
    def deploy_check(cls, device, parameters):
        if not device:
            raise JobError('job "device" was None')
        if 'actions' not in device:
            raise ConfigurationError('Invalid device configuration, no "actions" in device configuration')
        if 'to' not in parameters:
            raise ConfigurationError('"to" not specified in deploy parameters')
        if 'deploy' not in device['actions']:
            raise ConfigurationError('"deploy" is not in the device configuration actions')
        if 'methods' not in device['actions']['deploy']:
            raise ConfigurationError('Device misconfiguration, no "methods" in device configuration deploy actions')

    @classmethod
    def uses_deployment_data(cls):
        return True

    @classmethod
    def select(cls, device, parameters):
        cls.deploy_check(device, parameters)
        candidates = cls.__subclasses__()  # pylint: disable=no-member
        replies = {}
        willing = []
        for c in candidates:
            res = c.accepts(device, parameters)
            if not isinstance(res, tuple):
                raise LAVABug('class %s accept function did not return a tuple' % c.__name__)
            if res[0]:
                willing.append(c)
            else:
                replies[c.name] = res[1]

        if len(willing) == 0:
            replies_string = ""
            for name, reply in replies.items():
                replies_string += ("%s: %s\n" % (name, reply))
            raise JobError(
                "None of the deployment strategies accepted your deployment parameters, reasons given:\n%s" % replies_string)

        willing.sort(key=lambda x: x.priority, reverse=True)
        return willing[0] class Boot(object):
    """
    Allows selection of the boot method for this job within the parser.
    """

    priority = 0
    action_type = 'boot'
    compatibility = 0

    def __init__(self, parent):
        self.__parameters__ = {}
        self.pipeline = parent
        self.job = parent.job
        if self.compatibility > self.job.compatibility:
            self.job.compatibility = self.compatibility

    @classmethod
    def boot_check(cls, device, parameters):
        if not device:
            raise JobError('job "device" was None')
        if 'method' not in parameters:
            raise ConfigurationError("method not specified in boot parameters")
        if 'actions' not in device:
            raise ConfigurationError('Invalid device configuration, no "actions" in device configuration')
        if 'boot' not in device['actions']:
            raise ConfigurationError('"boot" is not in the device configuration actions')
        if 'methods' not in device['actions']['boot']:
            raise ConfigurationError('Device misconfiguration, no "methods" in device configuration boot action')

    @classmethod
    def accepts(cls, device, parameters):  # pylint: disable=unused-argument
        """
        Returns True if this deployment strategy can be used the the
        given device and details of an image in the parameters.

        Must be implemented by subclasses.
        """
        return NotImplementedError("accepts %s" % cls)

    @classmethod
    def select(cls, device, parameters):
        cls.boot_check(device, parameters)
        candidates = cls.__subclasses__()  # pylint: disable=no-member
        replies = {}
        willing = []
        for c in candidates:
            res = c.accepts(device, parameters)
            if not isinstance(res, tuple):
                raise LAVABug('class %s accept function did not return a tuple' % c.__name__)
            if res[0]:
                willing.append(c)
            else:
                class_name = c.name if hasattr(c, 'name') else c.__name__
                replies[class_name] = res[1]

        if len(willing) == 0:
            replies_string = ""
            for name, reply in replies.items():
                replies_string += ("%s: %s\n" % (name, reply))
            raise JobError(
                "None of the boot strategies accepted your boot parameters, reasons given:\n%s" % replies_string)

        willing.sort(key=lambda x: x.priority, reverse=True)
        return willing[0] class LavaTest(object):
    """
    Allows selection of the LAVA test method for this job within the parser.
    """

    priority = 1
    action_type = 'test'
    compatibility = 1  # used directly

    def __init__(self, parent):
        self.__parameters__ = {}
        self.pipeline = parent
        self.job = parent.job
        if self.compatibility > self.job.compatibility:
            self.job.compatibility = self.compatibility

    @classmethod
    def accepts(cls, device, parameters):  # pylint: disable=unused-argument
        """
        Returns True if this Lava test strategy can be used on the
        given device and details of an image in the parameters.

        Must be implemented by subclasses.
        """
        return NotImplementedError("accepts %s" % cls)

    @classmethod
    def select(cls, device, parameters):
        candidates = cls.__subclasses__()  # pylint: disable=no-member
        replies = {}
        willing = []
        for c in candidates:
            res = c.accepts(device, parameters)
            if not isinstance(res, tuple):
                raise LAVABug('class %s accept function did not return a tuple' % c.__name__)
            if res[0]:
                willing.append(c)
            else:
                class_name = c.name if hasattr(c, 'name') else c.__name__
                replies[class_name] = res[1]

        if len(willing) == 0:
            replies_string = ""
            for name, reply in replies.items():
                replies_string += ("%s: %s\n" % (name, reply))
            raise JobError(
                "None of the test strategies accepted your test parameters, reasons given:\n%s" % replies_string)

        willing.sort(key=lambda x: x.priority, reverse=True)
        return willing[0]

    @classmethod
    def needs_deployment_data(cls):
        return NotImplementedError("needs_deployment_data %s" % cls)

    @classmethod
    def needs_overlay(cls):
        return NotImplementedError("needs_overlay %s" % cls)

    @classmethod
    def has_shell(cls):
        return NotImplementedError("has_shell %s" % cls) def get_deployment_data(distro):
    """
    Returns the deployment data by name, for the cases where we actually need that.
    """
    if distro is '':
        raise JobError("Missing 'os' value for deployment - unable to identify operating system for deployment data.")
    this_module = sys.modules[__name__]
    try:
        return getattr(this_module, distro)
    except AttributeError:
        raise JobError("%s is not a supported distribution" % distro) class FinalizeAction(Action):

    section = "finalize"
    name = "finalize"
    description = "finish the process and cleanup"
    summary = "finalize the job"

    def __init__(self):
        """
        The FinalizeAction is always added as the last Action in the top level pipeline by the parser.
        The tasks include finalising the connection (whatever is the last connection in the pipeline)
        and writing out the final pipeline structure containing the results as a logfile.
        """
        super(FinalizeAction, self).__init__()
        self.ran = False

    def populate(self, parameters):
        self.internal_pipeline = Pipeline(job=self.job, parent=self, parameters=parameters)
        self.internal_pipeline.add_action(PowerOff())
        self.internal_pipeline.add_action(ReadFeedback(finalize=True, repeat=True))

    def run(self, connection, max_end_time, args=None):
        """
        The pexpect.spawn here is the ShellCommand not the ShellSession connection object.
        So call the finalise() function of the connection which knows about the raw_connection inside.
        The internal_pipeline of FinalizeAction is special - it needs to run even in the case of error / cancel.
        """
        self.ran = True
        try:
            connection = super(FinalizeAction, self).run(connection, max_end_time, args)
            if connection:
                connection.finalise()

        except Exception as exc:  # pylint: disable=unused-variable,broad-except
            pass
        finally:
            for protocol in self.job.protocols:
                protocol.finalise_protocol(self.job.device)
        return connection

    def cleanup(self, connection):
        # avoid running Finalize in validate or unit tests
        if not self.ran and self.job.started:
            self.run(connection, None, None) class Protocol(object):
    """
    Similar to a Connection object, provides a transport layer for the dispatcher.
    Uses a pre-defined API instead of pexpect using Shell.

    Testing a protocol involves either basing the protocol on SocketServer and using threading
    or adding a main function in the protocol python file and including a demo server script which
    can be run on the command line - using a different port to the default. However, this is likely
    to be of limited use because testing the actual API calls will need a functional test.

    If a Protocol requires another Protocol to be available in order to run, the depending
    Protocol *must* specify a higher level. All Protocol objects of a lower level are setup and
    run before Protocol objects of a higher level. Protocols with the same level can be setup or run
    in an arbitrary order (as the original source data is a dictionary).
    """
    name = 'protocol'
    level = 0

    def __init__(self, parameters, job_id):
        self.logger = logging.getLogger("dispatcher")
        self.poll_timeout = Timeout(self.name)
        self.parameters = None
        self.__errors__ = []
        self.parameters = parameters
        self.configured = False
        self.job_id = job_id

    @classmethod
    def select_all(cls, parameters):
        """
        Multiple protocols can apply to the same job, each with their own parameters.
        Jobs may have zero or more protocols selected.
        """
        candidates = cls.__subclasses__()  # pylint: disable=no-member
        return [(c, c.level) for c in candidates if c.accepts(parameters)]

    @property
    def errors(self):
        return self.__errors__

    @errors.setter
    def errors(self, error):
        self.__errors__.append(error)

    @property
    def valid(self):
        return len([x for x in self.errors if x]) == 0

    def set_up(self):
        raise LAVABug("'set_up' not implemented")

    def configure(self, device, job):  # pylint: disable=unused-argument
        self.configured = True

    def finalise_protocol(self, device=None):
        raise LAVABug("'finalise_protocol' not implemented")

    def check_timeout(self, duration, data):  # pylint: disable=unused-argument,no-self-use
        """
        Use if particular protocol calls can require a connection timeout
        larger than the default_connection_duration.
        :param duration: A minimum number of seconds
        :param data: the API call
        :return: True if checked, False if no limit is specified by the protocol.
        raises JobError if the API call is invalid.
        """
        return False

    def _api_select(self, data, action=None):  # pylint: disable=no-self-use
        if not data:
            return None
        raise LAVABug("'_api_select' not implemented")

    def __call__(self, *args, **kwargs):  # pylint: disable=no-self-use
        """ Makes the Protocol callable so that actions can send messages just using the protocol.
        This function may block until the specified API call returns. Some API calls may involve a
        substantial period of polling.
        :param args: arguments of the API call to make
        :return: A Python object containing the reply dict from the API call
        """
        # implementations will usually need a try: except: block around _api.select()
        return self._api_select(args, action=None)

    def collate(self, reply_dict, params_dict):  # pylint: disable=unused-argument,no-self-use
        return None class CommandAction(Action):

    compatibility = 1
    name = 'user-command'
    description = "execute one of the commands listed by the admin"
    summary = "execute commands"

    def __init__(self):
        super(CommandAction, self).__init__()
        self.section = "command"
        self.cmd = None
        self.ran = False

    def validate(self):
        super(CommandAction, self).validate()
        cmd_name = self.parameters['name']
        try:
            user_commands = self.job.device['commands']['users']
        except KeyError:
            raise ConfigurationError("Unable to get device.commands.users dictionary")

        try:
            self.cmd = user_commands[cmd_name]
            if not isinstance(self.cmd['do'], str) or \
               not isinstance(self.cmd.get('undo', ""), str):
                raise ConfigurationError("User command \"%s\" is invalid: "
                                         "'do' and 'undo' should be strings" % cmd_name)
            return True
        except KeyError:
            self.errors = "Unknown user command '%s'" % cmd_name
            return False

    def run(self, connection, max_end_time, args=None):
        connection = super(CommandAction, self).run(connection, max_end_time, args)

        self.logger.debug("Running user command '%s'", self.parameters['name'])
        self.ran = True
        self.run_command(self.cmd['do'].split(' '), allow_silent=True)
        return connection

    def cleanup(self, connection):
        if not self.ran:
            self.logger.debug("Skipping %s 'undo' as 'do' was not called", self.name)
            return

        if self.cmd is not None and 'undo' in self.cmd:
            self.logger.debug("Running cleanup for user command '%s'", self.parameters['name'])
            if not isinstance(self.cmd['undo'], str):
                self.logger.error("Unable to run cleanup: 'undo' is not a string")
            else:
                self.run_command(self.cmd['undo'].split(' '), allow_silent=True)
import yaml
import lava_dispatcher.actions.deploy.strategies
import lava_dispatcher.actions.boot.strategies
import lava_dispatcher.actions.test.strategies
import lava_dispatcher.protocols.strategies
from yaml.composer import Composer
from yaml.constructor import Constructor
from lava_dispatcher.job import Job
from lava_dispatcher.action import (
    Pipeline,
    Timeout,
    JobError,
)
from lava_dispatcher.logical import (
    Deployment,
    Boot,
    LavaTest,
)
from lava_dispatcher.deployment_data import get_deployment_data
from lava_dispatcher.power import FinalizeAction
from lava_dispatcher.connection import Protocol
from lava_dispatcher.actions.commands import CommandAction
# Copyright (C) 2014 Linaro Limited
#
# Author: Neil Williams <neil.williams@linaro.org>
#
# This file is part of LAVA Dispatcher.
#
# LAVA Dispatcher is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# LAVA Dispatcher is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along
# with this program; if not, see <http://www.gnu.org/licenses>.

# Bring in the strategy subclass lists, ignore pylint warnings.
# pylint: disable=unused-import,too-many-arguments,too-many-nested-blocks,too-many-branches


def parse_action(job_data, name, device, pipeline, test_info, test_count):
    """
    If protocols are defined, each Action may need to be aware of the protocol parameters.
    """
    parameters = job_data[name]
    parameters['test_info'] = test_info
    if 'protocols' in pipeline.job.parameters:
        parameters.update(pipeline.job.parameters['protocols'])

    if name == 'boot':
        Boot.select(device, parameters)(pipeline, parameters)
    elif name == 'test':
        # stage starts at 0
        parameters['stage'] = test_count - 1
        LavaTest.select(device, parameters)(pipeline, parameters)
    elif name == 'deploy':
        candidate = Deployment.select(device, parameters)
        if parameters['namespace'] in test_info and candidate.uses_deployment_data():
            if any([testclass for testclass in test_info[parameters['namespace']] if testclass['class'].needs_deployment_data()]):
                parameters.update({'deployment_data': get_deployment_data(parameters.get('os', ''))})
        if 'preseed' in parameters:
            parameters.update({'deployment_data': get_deployment_data(parameters.get('os', ''))})
        Deployment.select(device, parameters)(pipeline, parameters)


class JobParser(object):
    """
    Creates a Job object from the Device and the job YAML by selecting the
    Strategy class with the highest priority for the parameters of the job.

    Adding new behaviour is a two step process:
     - always add a new Action, usually with an internal pipeline, to implement the new behaviour
     - add a new Strategy class which creates a suitable pipeline to use that Action.

    Re-use existing Action classes wherever these can be used without changes.

    If two or more Action classes have very similar behaviour, re-factor to make a
    new base class for the common behaviour and retain the specialised classes.

    Strategy selection via select() must only ever rely on the device and the
    job parameters. Add new parameters to the job to distinguish strategies, e.g.
    the boot method or deployment method.
    """

    # FIXME: needs a Schema and a check routine

    loader = None

    # annotate every object in data with line numbers so we can use
    # them is user-friendly validation messages, combined with the action.level
    # each action will also include an output_line to map to the stdout log,
    # once executed.

    def compose_node(self, parent, index):
        # the line number where the previous token has ended (plus empty lines)
        line = self.loader.line
        node = Composer.compose_node(self.loader, parent, index)
        node.__line__ = line + 1
        return node

    def construct_mapping(self, node, deep=False):
        mapping = Constructor.construct_mapping(self.loader, node, deep=deep)
        mapping['yaml_line'] = node.__line__
        return mapping

    def _timeouts(self, data, job):
        if data.get('timeouts', None) is not None:
            if 'job' in data['timeouts']:
                duration = Timeout.parse(data['timeouts']['job'])
                job.timeout = Timeout(data['job_name'], duration)

    # pylint: disable=too-many-locals,too-many-statements
    def parse(self, content, device, job_id, logger, dispatcher_config,
              env_dut=None):
        self.loader = yaml.Loader(content)
        self.loader.compose_node = self.compose_node
        self.loader.construct_mapping = self.construct_mapping
        data = self.loader.get_single_data()

        job = Job(job_id, data, logger)