class Action(BasicConfigElement):
    """Action

    Action performed on the basis of expected results processing for probes
    which match the `matching_rules` rules.

    `kind`: type of action.

    `descr` (optional): brief description of the action.

    `when` (optional): when the action must be performed (with regards of
    expected results processing output); one of "on_match", "on_mismatch",
    "always". Default: "on_mismatch".

    When a probe matches a rule, it's expected results are processed; on the
    basis of the output, actions given in the rule's `actions` list are
    performed.
    For each expected result, if the probe's collected result matches the
    expectation actions whose `when` = "on_match" or "always" are performed.
    If the collected result does not match the expected result, actions
    whose `when` = "on_mismatch" or "always" are performed.
    """

    CFG_ACTION_KIND = None
    MANDATORY_CFG_FIELDS = ["kind"]
    OPTIONAL_CFG_FIELDS = ["descr", "when"]

    @classmethod
    def get_cfg_fields(cls):
        m = set(Action.MANDATORY_CFG_FIELDS)
        o = set(Action.OPTIONAL_CFG_FIELDS)

        if cls != Action:
            m.update(set(cls.MANDATORY_CFG_FIELDS))
            o.update(set(cls.OPTIONAL_CFG_FIELDS))

        return m, o

    def __init__(self, monitor, name, cfg):
        BasicConfigElement.__init__(self, cfg)

        self.monitor = monitor

        self.normalize_fields()

        self.name = name

        self.descr = cfg["descr"]

        self.kind = self._enforce_param("kind", str)

        if not self.kind:
            raise ConfigError("Missing action kind.")

        if self.kind != self.CFG_ACTION_KIND:
            raise ConfigError(
                "Wrong action kind: {} expected, {} found.".format(
                    self.CFG_ACTION_KIND, self.kind
                )
            )

        self.when = self._enforce_param("when", str) or "on_mismatch"

        WHEN = ("on_match", "on_mismatch", "always")

        if self.when not in WHEN:
            raise ConfigError(
                "Unexpected when ({}): must be one of {}".format(
                    self.when, ", ".join(WHEN)
                )
            )

    def __str__(self):
        raise NotImplementedError()

    def perform(self, result, expres, result_matches):
        raise NotImplementedError()

    def ping_output(self, result):
        # Stolen from https://github.com/RIPE-NCC/ripe-atlas-tools
        # (/blob/master/ripe/atlas/tools/renderers/ping.py)

        packets = result.packets

        if not packets:
            return "No packets found"

        # Because the origin value is more reliable as "from" in v4 and as
        # "packet.source_address" in v6.
        origin = result.origin
        if ":" in origin:
            origin = packets[0].source_address

        line = "{} bytes from probe #{:<5} {:15} to {} ({}): ttl={} times:{}\n"
        return line.format(
            result.packet_size,
            result.probe_id,
            origin,
            result.destination_name,
            result.destination_address,
            packets[0].ttl,
            " ".join(["{:8}".format(str(_.rtt) + ",") for _ in packets])
        )

    def traceroute_output(self, result):
        # Based on https://github.com/RIPE-NCC/ripe-atlas-tools

        r = ""
        for hop in result.hops:
            if hop.is_error:
                r += "{}\n".format(hop.error_message)
                continue

            name = ""
            rtts = []
            ip_info = None
            asn = ""
            details = ""
            for packet in hop.packets:
                if packet.origin and packet.origin != "*":
                    if not ip_info:
                        ip_info = self.monitor.ip_cache.get_ip_info(
                            packet.origin
                        )

                        asn = ip_info["ASN"]
                        ixp = ip_info["IsIXP"]
                        ixp_name = ip_info["IXPName"]

                        if asn and asn.isdigit():
                            details = "AS{}".format(asn)
                        elif ixp:
                            details = "IX {}".format(ixp_name)
                        else:
                            details = ""

                name = name or packet.origin or "*"
                if packet.rtt:
                    rtts.append("{:8} ms".format(packet.rtt))
                else:
                    rtts.append("          *")

            r += "{:>3} {:39} {}\n".format(
                hop.index,
                "{} {}".format(name, details),
                "  ".join(rtts)
            )
        return r

    def sslcert_output(self, result):
        r = ""

        TPL = ("SHA256 Fingerprint={sha256fp}\n"
               "  Issuer: C={issuer_c}, O={issuer_o}, CN={issuer_cn}\n"
               "  Subject: C={subject_c}, O={subject_o}, CN={subject_cn}\n")

        for certificate in result.certificates:
            r += TPL.format(
                issuer_c=certificate.issuer_c,
                issuer_o=certificate.issuer_o,
                issuer_cn=certificate.issuer_cn,
                subject_c=certificate.subject_c,
                subject_o=certificate.subject_o,
                subject_cn=certificate.subject_cn,
                sha256fp=certificate.checksum_sha256
            )

        return r

    def dns_output(self, result):
        r = ""

        if not result.responses:
            return "No response found"

        response_idx = 0
        indent = ""

        for response in result.responses:
            response_idx += 1

            if response_idx > 1:
                r += "\n"

            if len(result.responses) > 1:
                r += "Response n. {}\n\n".format(response_idx)
                indent = "  "

            if not response.abuf:
                r += indent + "Can't parse response's abuf\n"
                continue

            r += indent + "Header: {}, {}, id: {}\n".format(
                response.abuf.header.opcode,
                response.abuf.header.return_code,
                response.abuf.header.id
            )

            header_flags = []
            for flag in ("aa", "ad", "cd", "qr", "ra", "rd",):
                if getattr(response.abuf.header, flag):
                    header_flags.append(flag)

            r += indent + "Header flags: {}\n".format(", ".join(header_flags))

            if response.abuf.edns0:
                r += indent + "EDNS: version {}, size {}{}\n".format(
                        response.abuf.edns0.version,
                        response.abuf.edns0.udp_size,
                        ", DO flag" if response.abuf.edns0.do else ""
                    )

            for section in ("answers", "authorities", "additionals"):
                r += indent + "Section: {}\n".format(section)

                section = getattr(response.abuf, section)

                if len(section) > 0:
                    for record in section:
                        r += indent + "  " + str(record) + "\n"
                else:
                    r += indent + "  " + "no records\n"

        return r

    def get_result_text(self, result, expres):
        if self.monitor.msm_type == "traceroute":
            return self.traceroute_output(result)
        elif self.monitor.msm_type == "ping":
            return self.ping_output(result)
        elif self.monitor.msm_type == "sslcert":
            return self.sslcert_output(result)
        elif self.monitor.msm_type == "dns":
            return self.dns_output(result)
        else:
            raise NotImplementedError(
                "Action non implemented for {} "
                "measurements.".format(self.monitor.msm_type)
            )

    def get_notification_text(self, result, expres):

        probe = self.monitor.get_probe(result)

        r = ("{monitor}\n\n"
             "Received result from {probe} at {time}\n\n"
             "Expected result: {expres}\n\n"
             "{result}").format(
                monitor=self._capitalize_first(str(self.monitor)),
                expres=str(expres) if expres else "none",
                probe=str(probe),
                time=str(result.created),
                result=self.get_result_text(result, expres)
            )

        return r ACTION_CLASSES = [
    ActionLog,
    ActionSendEMail,
    ActionRunProgram,
    ActionSysLog,
    ActionLabel
] class ExpectedResult(BasicConfigElement):
    """Expected result

    A group of criteria used to match probes' results.

    `descr` (optional): a brief description of this group of criteria.

    Matching rules reference this on their `expected_results` list.

    When a probe matches a rule, the keys in the `expected_results` list
    of that rule are used to obtain the group of criteria to be used to
    process the result.

    Example:

    matching_rules:
    - descr: Probes from France via AS64496
      src_country: FR
      expected_results: ViaAS64496
    expected_results:
      ViaAS64496:
        upstream_as: 64496
    """

    OPTIONAL_CFG_FIELDS = ["descr"]
    MANDATORY_CFG_FIELDS = []

    @classmethod
    def get_cfg_fields(cls):
        m = set(cls.MANDATORY_CFG_FIELDS)
        o = set(cls.OPTIONAL_CFG_FIELDS)

        for criterion_class in CRITERIA_CLASSES:
            o.update(criterion_class.get_all_cfg_fields())

        return m, o

    def __init__(self, monitor, name, cfg):
        BasicConfigElement.__init__(self, cfg)

        self.monitor = monitor

        self.name = name
        self.normalize_fields()

        self.descr = cfg["descr"]

        self.criteria = []

        for criterion_class in CRITERIA_CLASSES:
            if self.cfg[criterion_class.CRITERION_NAME] is not None:
                self.criteria.append(criterion_class(self.cfg, self))

        if len(self.criteria) == 0:
            raise ConfigError("No criteria found.")

    def __str__(self):
        if self.descr:
            return self.descr
        else:
            ret = []
            for criterion in self.criteria:
                ret.append(str(criterion))
            if len(ret) > 0:
                return "; ".join(ret)
            else:
                return "No expected results"

    def display(self):
        if self.descr:
            print("    {}: {}".format(self.name, self.descr))
        else:
            print("    {}".format(self.name))

        for criterion in self.criteria:
            print(criterion.display_string())

        print("")

    def result_matches(self, result):
        # result is a ripe.atlas.sagan result

        probe = self.monitor.get_probe(result)

        result_descr = "{}, {}".format(probe, result.created)

        for criterion in self.criteria:
            try:
                criterion.prepare(result)
                if not criterion.result_matches(result):
                    return False
            except ResultProcessingError as e:
                logger.warning(
                    "Error processing result {}: {}".format(
                        result_descr, str(e)
                    )
                )
                return False

        return True CRITERIA_CLASSES = \
    CRITERIA_CLASSES_COMMON + \
    CRITERIA_CLASSES_TRACEROUTE + \
    CRITERIA_CLASSES_SSL + \
    CRITERIA_CLASSES_DNS CRITERIA_CLASSES_COMMON = [
    ExpResCriterion_RTT,
    ExpResCriterion_DstResponded,
    ExpResCriterion_DstIP
] CRITERIA_CLASSES_TRACEROUTE = [
    ExpResCriterion_DstAS,
    ExpResCriterion_ASPath,
    ExpResCriterion_UpstreamAS,
] CRITERIA_CLASSES_SSL = [
    ExpResCriterion_CertFP,
] CRITERIA_CLASSES_DNS = [
    ExpResCriterion_DNSRCode,
    ExpResCriterion_DNSFlags,
    ExpResCriterion_EDNS,
    ExpResCriterion_DNSAnswers
] class ExpResCriterion_DNSAnswers(ExpResCriterion_DNSBased):
    """Criterion: dns_answers

    Verify if the responses received by a probe contain the expected
    records.

    Available for: dns.

    `dns_answers`: one or more sections where records are searched on. Must
    be one of "answers", "authorities", "additionals".

    Each section must contain a list of records.

    Match when all the responses received by a probe contain at least one
    record matching the expected ones in each of the given sections.

    Example:

    dns_answers:
        answers:
            - <record1>
            - <record2>
        authorities:
            - <record3>
            - <record4>
    """

    CRITERION_NAME = "dns_answers"
    AVAILABLE_FOR_MSM_TYPE = ["dns"]
    MANDATORY_CFG_FIELDS = []
    OPTIONAL_CFG_FIELDS = []

    def __init__(self, cfg, expres):
        ExpResCriterion_DNSBased.__init__(self, cfg, expres)

        self.dns_answers = self._enforce_param("dns_answers", dict)

        self.sections = []

        SECTIONS = ["answers", "authorities", "additionals"]
        for section in self.dns_answers:
            if section.lower() not in SECTIONS:
                raise ConfigError(
                    "Invalid section: {}. Must be one of {}".format(
                        section, ", ".join(SECTIONS)
                    )
                )

            if isinstance(self.dns_answers[section], list):
                section_cfg = self.dns_answers[section]
            elif isinstance(self.dns_answers[section], dict):
                section_cfg = [self.dns_answers[section]]
            else:
                raise ConfigError(
                    "Invalid section {}".format(section)
                )
            self.sections.append(
                ExpResCriterion_AnswersSection(
                    section, section_cfg
                )
            )

    def __str__(self):
        r = ""
        for section in self.sections:
            if r != "":
                r += "; "
            r += "{} section: {}".format(
                section.name,
                ", ".join(map(str, section.records))
            )
        return r

    def prepare_response(self, result, response):
        # This class doesn't behave like others and doesn't use
        # the prepare() method to parse results.
        # It implements checks directly in the response_matches()
        # method, that verifes every record directly.
        pass

    def response_matches(self, response):
        for section in self.sections:
            answer_section = getattr(response.abuf, section.name)
            if len(answer_section) == 0:
                logger.debug(
                    "  section {} not found in the response".format(
                        section.name
                    )
                )
                return False

            for record in section.records:
                for answer_record in answer_section:
                    if record.record_matches(answer_record):
                        return True

        return False class ExpResCriterion_AnswersSection(object):
    """DNS answer section

    One of "answers", "authorities", "additionals".

    Each section must contain a list of DNS records.
    """

    def __init__(self, name, cfg):
        self.name = name
        self.records = []
        for record_cfg in cfg:
            self.add_record(record_cfg)

    def add_record(self, record_cfg):

        if "type" not in record_cfg:
            raise ConfigError("Missing mandatory attribute: type")

        record_class = None
        for record_class in HANDLED_RECORD_TYPES:
            if record_class.RECORD_TYPE.lower() == record_cfg["type"].lower():
                self.records.append(record_class(record_cfg))
                return

        raise ConfigError(
            "Unhandled record type: {}".format(
                record_cfg["type"]
            )
        )

    def __str__(self):
        return "Section {}: {}".format(
            self.name,
            ", ".join(map(str, self.records))
        ) class ExpResCriterion_DNSRecord(BasicConfigElement):
    """DNS record

    Test properties which are common to all DNS record types.

    `type`: record's type. Must be one of the DNS record types implemented
    and described below.

    `name` (optional): list of expected names.

    `ttl_min` (optional): minimum TTL that is expected for the record.

    `ttl_max` (optional): maximum TTL that is expected for the record.

    `class` (optional): expected class for the record.

    Match when all the defined criteria are met:

    - record name must be within the list of given names (`name`);

    - record TTL must be >= `ttl_min` and <= `ttl_max`;

    - record class must be equal to `class`.

    On the basis of record's `type`, further parameters may be needed.

    Example:

    dns_answers:
        answers:
            - type: A
              name: www.ripe.net.
              address: 193.0.6.139
            - type: AAAA
              name:
              - www.ripe.net.
              - ripe.net.
              ttl_min: 604800
              address: 2001:67c:2e8:22::c100:0/64
    """

    RECORD_TYPE = ""
    MANDATORY_CFG_FIELDS = ["type"]
    OPTIONAL_CFG_FIELDS = ["name", "ttl_min", "ttl_max", "class"]

    @classmethod
    def get_cfg_fields(cls):
        m = set(ExpResCriterion_DNSRecord.MANDATORY_CFG_FIELDS)
        o = set(ExpResCriterion_DNSRecord.OPTIONAL_CFG_FIELDS)

        m.update(cls.MANDATORY_CFG_FIELDS)
        o.update(cls.OPTIONAL_CFG_FIELDS)

        return m, o

    def __init__(self, cfg):
        BasicConfigElement.__init__(self, cfg)
        self.normalize_fields()

        self.type = self._enforce_param("type", str)
        self.name = self._enforce_list("name", str)
        self.ttl_min = self._enforce_param("ttl_min", int)
        self.ttl_max = self._enforce_param("ttl_max", int)
        self.klass = self._enforce_param("class", str)

    def __str__(self):
        r = "{}: ".format(self.RECORD_TYPE)
        if self.name:
            if len(self.name) > 1:
                r += "name in {}, "
            else:
                r += "name {}, "
            r = r.format(", ".join(self.name))

        if self.ttl_min:
            r += "ttl < {}, ".format(self.ttl_min)

        if self.ttl_max:
            r += "ttl > {}, ".format(self.ttl_max)

        if self.klass:
            r += "class {}, ".format(self.klass)

        return r

    def record_base_matches(self, record):
        if self.RECORD_TYPE != record.type:
            logger.debug(
                "  record type {} is not {}".format(
                    record.type, self.RECORD_TYPE
                )
            )
            return False
        if self.name:
            if record.name not in self.name:
                logger.debug(
                    "  record name {} is not {}".format(
                        record.name, ", ".join(self.name)
                    )
                )
                return False
        if self.ttl_min:
            if record.ttl < self.ttl_min:
                logger.debug(
                    "  record TTL {} < {}".format(
                        record.ttl, self.ttl_min
                    )
                )
                return False
        if self.ttl_max:
            if record.ttl > self.ttl_max:
                logger.debug(
                    "  record TTL {} > {}".format(
                        record.ttl, self.ttl_max
                    )
                )
                return False
        if self.klass:
            if record.klass != self.klass:
                logger.debug(
                    "record class {} != {}".format(
                        record.klass, self.klass
                    )
                )
                return False
        return True

    def _record_matches(self, record):
        raise NotImplementedError()

    def record_matches(self, record):
        return self.record_base_matches(record) and \
            self._record_matches(record) HANDLED_RECORD_TYPES = [
    ExpResCriterion_DNSRecord_A,
    ExpResCriterion_DNSRecord_AAAA,
    ExpResCriterion_DNSRecord_NS,
    ExpResCriterion_DNSRecord_CNAME
] class Monitor(BasicConfigElement, MsmProcessingUnit):
    """Monitor

    A monitor allows to process results from a measurement.

    `descr` (optional): monitor's brief description.

    `measurement-id` (optional): measurement ID used to gather results. It can
    be given (and/or overwritten) via command line argument `--measurement-id`.

    `matching_rules`: list of rules to match probes against. When a probe
    matches one of these rules, its expected results are processed and its
    actions are performed.

    `expected_results` (optional): list of expected results. Probe's expected
    results contain references to this list.

    `actions` (optional): list of actions to be executed on the basis of
    probe's expected results.

    `stream` (optional): boolean indicating if results streaming must be used.
    It can be given (and/or overwritten) via command line argument `--stream`.

    `stream_timeout` (optional): how long to wait (in seconds) before stopping
    a streaming monitor if no results are received on the stream.

    `key` (optional): RIPE Atlas key to access the measurement. It can be
    given (and/or overwritten) via command line argument `--key`.

    `key_file` (optional): a file containing the RIPE Atlas key to access the
    measurement. The file must contain only the RIPE Atlas key, in plain text.
    If `key` is given, this field is ignored.
    """

    MANDATORY_CFG_FIELDS = ["matching_rules"]

    OPTIONAL_CFG_FIELDS = ["measurement-id", "stream", "stream_timeout",
                           "actions", "key", "key_file", "descr",
                           "expected_results"]

    def _get_statusfile_path(self):
        if self.monitor_name:
            return "{}/status/{}.{}.json".format(
                Config.get("var_dir"),
                self.monitor_name,
                self.msm_id
            )
        return None

    def _get_lockfile_path(self):
        if self.monitor_name:
            return "{}/locks/{}.{}.lock".format(
                Config.get("var_dir"),
                self.monitor_name,
                self.msm_id
            )
        return None

    def _load_from_file(self, monitor_name):
        file_path = "{}/monitors/{}.yaml".format(
            Config.get("var_dir"), monitor_name
        )

        if not os.path.isfile(file_path):
            raise MissingFileError(path=file_path)

        try:
            with open(file_path, "r") as monitor_file:
                try:
                    return yaml.load(monitor_file.read())
                except yaml.parser.ParserError as e:
                    raise ConfigError(
                        "Error in YAML syntax: {}".format(str(e))
                    )
        except Exception as e:
            raise ConfigError(
                "Can't read from the monitor configuration file {}: {}".format(
                    file_path, str(e)
                )
            )

    def __init__(self, cfg_or_name, ip_cache=None, msm_id=None, key=None):

        if isinstance(cfg_or_name, str):
            self.monitor_name = cfg_or_name
            cfg = self._load_from_file(self.monitor_name)

        elif isinstance(cfg_or_name, dict):
            self.monitor_name = None
            cfg = cfg_or_name

        else:
            raise ConfigError(
                "Invalid monitor name or configuration type: {}".format(
                    type(cfg_or_name)
                )
            )

        BasicConfigElement.__init__(self, cfg)

        self.normalize_fields()

        self.descr = self._enforce_param("descr", str)

        self._enforce_param("matching_rules", list)

        self._enforce_param("expected_results", dict) or {}

        self._enforce_param("actions", dict)

        MsmProcessingUnit.__init__(
            self,
            ip_cache=ip_cache,
            msm_id=msm_id or self._enforce_param("measurement-id", int),
            key=key or self._enforce_param("key", str),
            key_file=self._enforce_param("key_file", str)
        )

        self.stream = self._enforce_param("stream", bool) or False

        self.stream_timeout = self._enforce_param("stream_timeout", int)

        if self.stream:
            self.ensure_streaming_enabled(ConfigError)

        # Expected results normalization

        self.expected_results = {}

        if self.cfg["expected_results"]:
            for expres_name in self.cfg["expected_results"]:
                expres_cfg = self.cfg["expected_results"][expres_name]
                try:
                    expres = ExpectedResult(self, expres_name, expres_cfg)
                    self.expected_results[expres_name] = expres
                except ConfigError as e:
                    raise ConfigError(
                        "Syntax error for expected result {} - {}".format(
                            expres_name, str(e)
                        )
                    )

        # Actions normalization

        self.actions = {}

        if self.cfg["actions"]:
            for action_name in self.cfg["actions"]:
                action_cfg = self.cfg["actions"][action_name]
                try:
                    if "kind" in action_cfg:
                        action_kind = action_cfg["kind"]
                        action = None

                        for action_class in ACTION_CLASSES:
                            if action_class.CFG_ACTION_KIND == action_kind:
                                action = action_class(self, action_name,
                                                      action_cfg)
                                break

                        if action is None:
                            raise ConfigError(
                                "Unknown action kind: {}".format(
                                    action_kind
                                )
                            )

                        self.actions[action_name] = action
                    else:
                        raise ConfigError("Missing action kind")
                except ConfigError as e:
                    raise ConfigError(
                        "Syntax error for action {} - {}".format(
                            action_name, str(e)
                        )
                    )

        # Rules normalization

        self.rules = []

        rule_n = 0
        for rule_cfg in self.cfg["matching_rules"]:
            rule_n += 1

            try:
                rule = Rule(self, rule_cfg)
                self.rules.append(rule)

                if rule.expected_results and rule.expected_results != []:
                    for expres_name in rule.expected_results:
                        if expres_name not in self.expected_results:
                            raise ConfigError(
                                "Expected result not found: "
                                "{}".format(expres_name)
                            )

                if rule.actions and rule.actions != []:
                    for action_name in rule.actions:
                        if action_name not in self.actions:
                            raise ConfigError(
                                "Action not found: "
                                "{}".format(action_name)
                            )

            except ConfigError as e:
                raise ConfigError(
                    "Syntax error for rule n. {} - {}".format(
                        rule_n, str(e)
                    )
                )

        self.internal_labels = {
            "probes": {},
            "results": {}
        }

        self.exit_thread = False
        self.results_queue = None

        self.lock_fd = None
        self.lock_file = LockFile()

        self.status = {}
        self.load_status()
        self._epoch = datetime.datetime.fromtimestamp(0, tz=pytz.UTC)

    def load_status(self):
        status_filepath = self._get_statusfile_path()
        if status_filepath:
            if os.path.isfile(status_filepath):
                try:
                    with open(status_filepath, "r") as f:
                        self.status = json.loads(f.read())
                except:
                    raise ProgramError(
                        "Can't read status from {}".format(status_filepath)
                    )

    def write_status(self):
        status_filepath = self._get_statusfile_path()
        if status_filepath:
            try:
                with open("{}.tmp".format(status_filepath), "w") as f:
                    f.write(json.dumps(self.status))
                os.rename("{}.tmp".format(status_filepath), status_filepath)
            except:
                raise ProgramError(
                    "Can't write status to {}".format(status_filepath)
                )

    def __str__(self):
        if self.descr:
            return "monitor '{}'".format(self.descr)
        else:
            if self.monitor_name:
                tpl = ("monitor '{name}' "
                       "(measurement ID {msm_id}, {msm_type}, IPv{af})")
            else:
                tpl = ("monitor for measurement ID {msm_id} "
                       "({msm_type}, IPv{af})")

            return tpl.format(
                name=self.monitor_name,
                msm_id=self.msm_id,
                msm_type=self.msm_type,
                af=self.msm_af
            )

    def display(self):
        if self.msm_id:
            print("Measurement ID: {}".format(self.msm_id))
        else:
            print(
                "Measurement ID not specified, it must be given using the "
                "command line argument --measurement-id."
            )
        print("")

        if self.stream:
            if self.stream_timeout:
                print("Streaming of results enabled - "
                      "{} seconds timeout.".format(self.stream_timeout))
            else:
                print("Streaming of results enabled - no timeout.")
            print("")

        rule_n = 0
        for rule in self.rules:
            rule_n += 1

            print("Matching rule n. {}".format(rule_n))
            print("")

            rule.display()

            if len(rule.expected_results) > 0:
                print("  Expected results:")
                print("")
                for expres_name in rule.expected_results:
                    self.expected_results[expres_name].display()
            else:
                print("  No expected results for this rule.")
                print("")

            if len(rule.actions) > 0:
                print("  Actions:")
                print("")
                for action_name in rule.actions:
                    if self.actions[action_name].when == "on_match":
                        tpl = "    Action fired on match: {}"
                    elif self.actions[action_name].when == "on_mismatch":
                        tpl = "    Action fired on mismatch: {}"
                    else:
                        tpl = "    Action always fired: {}"
                    print(tpl.format(self.actions[action_name]))
                    print("")

    def process_matching_probe(self, probe, rule_n, rule, result):
        logger.info("  probe ID {} matches".format(probe.id))

        if len(rule.expected_results) == 0:
            logger.result(
                "Rule n. {} ({}), {}, "
                "no expected result.".format(
                    rule_n, str(rule), probe
                )
            )
            rule.perform_actions(result=result)
            return

        for expres_name in rule.expected_results:
            expres = self.expected_results[expres_name]

            logger.info(
                "Verifying expected result {}: {}...".format(
                    expres_name, str(expres)
                )
            )

            result_matches = expres.result_matches(result)

            logger.result(
                "Rule n. {} ({}), {}, "
                "expected result {}: {}.".format(
                    rule_n, str(rule), probe, expres_name,
                    "OK" if result_matches else "MISMATCH"
                )
            )

            rule.perform_actions(result=result, expres=expres,
                                 result_matches=result_matches)

    def update_latest_result_ts(self, result):
        ts = (result.created - self._epoch).total_seconds()

        if "latest_result_ts" in self.status:
            if self.status["latest_result_ts"] > ts:
                return
        self.status["latest_result_ts"] = ts
        self.write_status()

    def process_results(self, results, probes_filter):
        logger.info("Processing results...")

        # Be sure to have info for every probe in the resultset
        self.update_probes(results)

        if len(results) == 0:
            logger.info("No results found.")
            return

        # Processing results...

        for json_result in results:
            result = Result.get(json_result, on_error=Result.ACTION_IGNORE,
                                on_malformation=Result.ACTION_IGNORE)

            probe = self.get_probe(result)
            if probe not in probes_filter:
                logger.debug(
                    "  skipping {} because of probes filter".format(probe)
                )
                continue

            self.process_result(result)
            self.update_latest_result_ts(result)
            self.internal_labels["results"] = {}

        self.ip_cache.save()

    def process_result(self, result):
        self.internal_labels["results"] = {}

        is_success = not result.is_error and not result.is_malformed

        if not is_success:
            return

        probe = self.get_probe(result)

        logger.info(
            "Processing result for {} at {}...".format(
                str(probe), result.created
            )
        )

        rule_n = 0
        for rule in self.rules:
            rule_n += 1

            logger.info(
                "Testing rule n. {}: {}...".format(rule_n, str(rule))
            )

            if rule.probe_matches(probe):
                self.process_matching_probe(probe, rule_n, rule, result)

                if rule.process_next is True:
                    logger.info("  next rule processing forcedly enabled")
                else:
                    break
            else:
                logger.info(
                    "  {} does not match".format(probe)
                )

                if rule.process_next is False:
                    logger.info("  next rule processing forcedly inhibited")
                    break

    def on_result_response(self, result):
        self.results_queue.put(result)

    def acquire_lock(self):
        if not self.monitor_name:
            return

        if not self.lock_file.acquire(self._get_lockfile_path()):
            raise LockError(
                "Another instance of this program is already "
                "executing this monitor."
            )

    def release_lock(self):
        self.lock_file.release()

    def ensure_streaming_enabled(self, exception_class):
        err = "Can't use results streaming for this measurement: {}"

        if self.msm_is_oneoff:
            raise exception_class(
                err.format("it's a one-off measurement.")
            )

        if not self.msm_is_running:
            raise exception_class(
                err.format("it is not running anymore.")
            )

        if not self.msm_is_public:
            raise exception_class(
                err.format("it is not public.")
            )

    def run_stream(self, probes_filter):
        logger.info(" - using real-time results streaming")

        self.ensure_streaming_enabled(MeasurementProcessingError)

        try:
            atlas_stream = AtlasStream()
            atlas_stream.connect()
            atlas_stream.bind_channel("result", self.on_result_response)
            stream_params = {"msm": self.msm_id}
        except Exception as e:
            raise MeasurementProcessingError(
                "Error while creating the stream: {}".format(str(e))
            )

        self.results_queue = Queue()
        thread = MonitorResultsThread(self, probes_filter)

        try:
            thread.start()

            atlas_stream.start_stream(stream_type="result",
                                      **stream_params)
            atlas_stream.timeout(seconds=self.stream_timeout)
            atlas_stream.disconnect()
        except:
            try:
                atlas_stream.disconnect()
            except:
                pass
        finally:
            try:
                atlas_stream.disconnect()
            except:
                pass
            self.exit_thread = True
            thread.join(timeout=10)

    def get_latest_result_ts(self):
        if "latest_result_ts" in self.status:
            return self.status["latest_result_ts"]
        return None

    def run_once(self, probes_filter, start=None, stop=None,
                 latest_results=None):
        fetch_only_probe_ids = probes_filter.probe_ids

        results = self.download(start=start, stop=stop,
                                latest_results=latest_results,
                                probe_ids=fetch_only_probe_ids)

        self.process_results(results, probes_filter)

    def run_continously(self, start, probes_filter):
        try:
            while True:
                self.run_once(probes_filter, start=start)

                logger.info(
                    "Waiting {} seconds (measurement's interval) before "
                    "downloading new results...".format(self.msm_interval)
                )

                time.sleep(self.msm_interval)
        except KeyboardInterrupt:
            pass

    def run(self, start=None, stop=None, latest_results=None, dont_wait=False,
            probes_filter=None):
        self.acquire_lock()

        if not probes_filter:
            probes_filter = ProbesFilter()

        logger.info("Starting {}".format(str(self)))

        try:
            if self.stream:
                self.run_stream(probes_filter)

            elif not self.msm_is_oneoff and self.msm_is_running and \
                    not latest_results and not stop and not dont_wait:

                self.run_continously(start, probes_filter)

            else:
                self.run_once(probes_filter, start=start, stop=stop,
                              latest_results=latest_results)
        finally:
            self.release_lock() class Rule(BasicConfigElement):
    """Rule

    Probes which produced the results fetched from the measurement are matched
    against these rules to determine whether those results must be processed
    or not.

    `descr` (optional): a brief description of the rule.

    `process_next` (optional): determine whether the rule following the current
    one has to be elaborated or nor. More details on the description below.

    `src_country` (optional): list of two letters country ISO codes.

    `src_as` (optional): list of Autonomous System numbers.

    `probe_id` (optional): list of probes' IDs.

    `internal_labels` (optional): list of internal labels. More details on the
    description below.

    `reverse` (optional): boolean, indicating if the aforementioned criteria
    identify probes which have to be exluded from the matching.

    `expected_results` (optional): list of expected results' names which
    have to be processed on match. Must be one or more of the expected results
    defined in Monitor.`expected_results`. If empty or missing, the rule will
    be treated as if a match occurred and its actions are performed.

    `actions` (optional): list of actions' names which have to be perormed for
    matching probes. Must be one or more of the actions defined in
    Monitor.`actions`.

    The `src_country` criterion matches when probe's source country is one of
    the country ISO codes given in the list.

    The `src_as` criterion matches when probe's source AS is one of the ASN
    given in the list. Since RIPE Atlas defines two ASs for each probe (ASN_v4
    and ASN_v6) the one corresponding to the measurement's address family is
    taken into account.

    The `probe_id` criterion matches when probe's ID is one of the IDs given
    in the list.

    The `internal_labels` criterion matches when a probe has been previously
    tagged with a label falling in the given list. See the `label` Action for
    more details.

    A probe matches the rule when all the given criteria are satisfied or when
    no criteria are defined at all. If `reverse` is True, a probe matches when
    none of the criteria is satisfied.

    When a probe matches the rule, the expected results given
    in `expected_results` are processed; actions given in the `actions` list
    are performed on the basis of expected results processing output. If
    no `expected_results` are given, actions will be performed too.

    When a probe matches the current rule's criteria:

    - if `process_next` is True, the rule which follows the current one is
      forcedly elaborated;

    - if `process_next` if False or missing, the rules processing is stopped.

    If a probe does not match the current rule's criteria:

    - if `process_next` is False, the rule processing is forcedly stopped;

    - if `process_next` is True or missing, the rule which follows the current
      one is regularly processed.

    Examples:

    matching_rules:
    - descr: Do not process results for probe ID 123 and 456
      probe_id:
      - 123
      - 456
    - descr: Check dst AS for any probe, errors to NOC; process next rule
      expected_results: DstAS
      actions: SendEMailToNOC
      process_next: True
    - descr: Italian probes must reach target via AS64496
      src_country: IT
      expected_results: ViaAS64496
      actions: LogErrors
    - descr: German and French probes must reach target with low RTT
      src_country:
      - DE
      - FR
      expected_results: LowRTT
      actions: LogErrors

    matching_rules:
    - descr: Set 'VIP' (Very Important Probe) label to ID 123 and 456
      probe_id:
      - 123
      - 456
      process_next: True
      actions: SetVIPLabel
    - descr: Set 'VIP' label to Italian probes too
      src_country: IT
      process_next: True
      actions: SetVIPLabel
    - descr: VIPs must have low RTT
      internal_labels: VIP
      expected_results: LowRTT
    """

    MANDATORY_CFG_FIELDS = []

    OPTIONAL_CFG_FIELDS = ["expected_results", "descr", "process_next",
                           "src_country", "src_as", "probe_id", "reverse",
                           "internal_labels", "actions"]

    def __init__(self, monitor, cfg):
        BasicConfigElement.__init__(self, cfg)

        self.monitor = monitor

        self.normalize_fields()

        self.descr = cfg["descr"]

        self.process_next = self._enforce_param("process_next", bool)

        self.src_country = self._enforce_list("src_country", str)

        self.src_as = self._enforce_list("src_as", int)

        self.probe_id = self._enforce_list("probe_id", int)

        self.internal_labels = self._enforce_list("internal_labels", str)

        self.reverse = self._enforce_param("reverse", bool) or False

        self.expected_results = self._enforce_list("expected_results", str)

        if self.expected_results is None:
            self.expected_results = []

        self.actions = self._enforce_list("actions", str)

        if self.src_country:
            for cc in self.src_country:
                if not re.match(r"^[a-zA-Z][a-zA-Z]$", cc):
                    raise ConfigError(
                        "Invalid country code: {}. "
                        "Countries must be defined with a two-letter "
                        "ISO code.".format(cc)
                    )

    def _str_src_county(self):
        return ", ".join(self.src_country)

    def _str_src_as(self):
        return ", ".join(map(str, self.src_as))

    def _str_probe_id(self):
        return ", ".join(map(str, self.probe_id))

    def _str_internal_labels(self):
        return ", ".join(self.internal_labels)

    def __str__(self):
        if self.descr:
            return self.descr
        else:
            ret = []
            if self.reverse:
                ret.append("Reverse")
            if len(self.src_country) > 0:
                ret.append("Country: {}".format(self._str_src_county()))
            if len(self.src_as) > 0:
                ret.append("Source AS: {}".format(self._str_src_as()))
            if len(self.probe_id) > 0:
                ret.append("Probe ID: {}".format(self._str_probe_id()))
            if len(self.internal_labels) > 0:
                ret.append("Internal labels: {}".format(
                    self._str_internal_labels())
                )
            if len(ret) > 0:
                return "; ".join(ret)
            else:
                return "Match any probe"

    def display(self):
        criteria_found = 0

        if self.descr:
            print("  Description  : {}".format(self.descr))
            print("")

        if self.reverse:
            print("  Reverse      : {}".format(self.reverse))

        if len(self.src_country) > 0:
            criteria_found += 1
            print("  Country      : {}".format(self._str_src_county()))

        if len(self.src_as) > 0:
            criteria_found += 1
            print("  Source AS    : {}".format(self._str_src_as()))

        if len(self.probe_id) > 0:
            criteria_found += 1
            print("  Probe ID     : {}".format(self._str_probe_id()))

        if len(self.internal_labels) > 0:
            criteria_found += 1
            print("  Internal labels: {}".format(self._str_internal_labels()))

        if criteria_found > 1:
            print("")
            print(
                "  The rule matches for source probes that {}satisfy all the "
                "above criteria.".format("do not " if self.reverse else "")
            )
        elif criteria_found == 1:
            print("")
            print(
                "  The rule matches for source probes that {}satisty the "
                "above criterion.".format("do not " if self.reverse else "")
            )
        else:
            print(
                "  No criteria defined for the rule: it {}  matches for "
                "any source probe.".format(
                    "never " if self.reverse else "always"
                )
            )

        print("")
        if self.process_next:
            if self.process_next is True:
                print(
                    "  The rules following this one are processed even if a "
                    "matching condition is found."
                )
            else:
                print(
                    "  If a matching condition is not found, the rule that "
                    "follows this one is not elaborated and the rules "
                    "processing is forcedly stopped."
                )
        else:
            print(
                "  If a matching condition is not found, the rule that "
                "follows this one is elaborated."
            )
            print(
                "  Once a matching condition is found, the rules following "
                "this one are not processed and the execution is stopped."
            )

        print("")

    def probe_matches(self, probe):
        criteria_cnt = 0
        match_cnt = 0

        if self.reverse:
            logger.debug("  excluding rule!")

        if len(self.src_country) > 0:
            criteria_cnt += 1
            logger.debug(
                "  testing probe ID {}: "
                "country [{}] in {}".format(probe.id, probe.country_code,
                                            self._str_src_county())
            )
            if probe.country_code in self.src_country:
                match_cnt += 1

        if len(self.src_as) > 0:
            criteria_cnt += 1
            logger.debug(
                "  testing probe ID {}: "
                "src AS [{}] in {}".format(
                    probe.id,
                    probe.asn,
                    self._str_src_as())
            )
            if probe.asn in self.src_as:
                match_cnt += 1

        if len(self.probe_id) > 0:
            criteria_cnt += 1
            logger.debug("  testing probe ID {}: ID in {}".format(
                probe.id, self._str_probe_id())
            )
            if probe.id in self.probe_id:
                match_cnt += 1

        if len(self.internal_labels) > 0:
            criteria_cnt += 1
            probe_labels = set()
            for scope in ["probes", "results"]:
                if str(probe.id) in self.monitor.internal_labels[scope]:
                    probe_labels.update(
                        self.monitor.internal_labels[scope][str(probe.id)]
                    )
            logger.debug(
                "  testing probe ID {}: "
                "internal labels: {}, expected labels: {}".format(
                    probe.id,
                    ", ".join(probe_labels) if probe_labels else "none",
                    self._str_internal_labels()
                )
            )
            for label in self.internal_labels:
                if label in probe_labels:
                    match_cnt += 1
                    break

        if self.reverse:
            if criteria_cnt == 0:
                logger.debug(
                    "  excluding rule: probe did not pass because "
                    "no criteria are defined for this rule"
                )
                return False
            elif criteria_cnt == match_cnt:
                logger.debug(
                    "  excluding rule: probe did not pass because "
                    "it matched all the criteria for this rule"
                )
                return False
            else:
                return True
        else:
            return criteria_cnt == match_cnt

    def perform_actions(self, result=None, expres=None, result_matches=None):
        for action_name in self.actions:
            action = self.monitor.actions[action_name]

            if action.when == "always" or \
                    (action.when == "on_match" and result_matches is None) or \
                    (action.when == "on_match" and result_matches is True) or \
                    (action.when == "on_mismatch" and result_matches is False):

                try:
                    action.perform(result, expres, result_matches)
                except RIPEAtlasMonitorError as e:
                    logger.error(
                        "Error while performing action '{}': {}".format(
                            str(action), str(e)
                        )
                    )
import re
import six
    import argparse
from .Action import Action, ACTION_CLASSES
from .ExpectedResult import ExpectedResult
from .ExpResCriteria import CRITERIA_CLASSES, \
                            CRITERIA_CLASSES_COMMON, \
                            CRITERIA_CLASSES_TRACEROUTE, \
                            CRITERIA_CLASSES_SSL, \
                            CRITERIA_CLASSES_DNS
from .ExpResCriteriaDNS import ExpResCriterion_DNSAnswers, \
                               ExpResCriterion_AnswersSection
from .ExpResCriteriaDNSRecords import ExpResCriterion_DNSRecord, \
                                      HANDLED_RECORD_TYPES
from .Monitor import Monitor
from .Rule import Rule
            cls_cnt += 1
            r += dump_doc(subc, 4)

    if cls_cnt != len(CRITERIA_CLASSES):
        raise ValueError(
            "One or more criteria classes have not been processed. "
            "{} processed vs {} total.".format(cls_cnt, len(CRITERIA_CLASSES))
        )

    r += dump_doc(Action, 2)
    for subc in ACTION_CLASSES:
        r += dump_doc(subc, 3)

    return r


def dump_yaml(c, indent="", show_doc=True, elements_type="",
              comment_optional=True, index=0):
    INDENT = "    "

    r = ""

    s = format_docstring(c.__doc__)

    if not s:
        return ""

    if show_doc:
        # print the whole docstring
        r += indent + "# {}\n".format("="*75)
        for line in s.split("\n"):
            r += indent + "# " + line + "\n"

        r += "\n"

    params = get_params(s, c)

    first_list_element = True

    class_description = get_class_descr(s)

    if elements_type == "dict":
        dict_key = class_description.replace(" ", "_")
        r += indent + "{}_{}:\n".format(dict_key, index+1)
    elif elements_type == "list":
        list_comment = class_description
        r += indent + "# {} n. {}\n".format(list_comment, index+1)

    if not show_doc:
        if c in HANDLED_RECORD_TYPES or c in ACTION_CLASSES:
            r += indent + "# {}\n".format(class_description)

    prefix = ""
    if elements_type == "dict":
        prefix = INDENT

    for param in params:
        if elements_type == "list":
            if first_list_element:
                r += indent + "- \n"
                first_list_element = False
            prefix = "  "

        r += indent + "{prefix}{comment}{param}: \n".format(
            prefix=prefix,
            comment="#" if comment_optional and param[1] else "",
            param=param[0]
        )

        if c == Monitor and param[0] == "expected_results":
            for i in range(2):
                r += dump_yaml(ExpectedResult, indent=indent + INDENT,
                               elements_type="dict",
                               show_doc=show_doc and i == 0,
                               comment_optional=comment_optional, index=i)

        elif c == Monitor and param[0] == "matching_rules":
            for i in range(2):
                r += dump_yaml(Rule, indent=indent,
                               elements_type="list",
                               show_doc=show_doc and i == 0,
                               comment_optional=comment_optional, index=i)

        elif c == Monitor and param[0] == "actions":
            for i in range(2):
                r += dump_yaml(Action, indent=indent + INDENT,
                               elements_type="dict",
                               show_doc=show_doc and i == 0,
                               comment_optional=comment_optional, index=i)

        else:
            if param[2]:
                for i in range(3):
                    param_name = param[0]

                    if c == Rule and param_name == "expected_results":
                        param_name = get_class_descr(ExpectedResult)
                    if c == Rule and param_name == "actions":
                        param_name = get_class_descr(Action)

                    param_name = param_name.replace(" ", "_")

                    r += indent + "{prefix}{comment}- {param}_{i}\n".format(
                        comment="#" if comment_optional and param[1] else "",
                        prefix=prefix,
                        param=param_name,
                        i=i+1
                    )
            r += "\n"

    if c == ExpectedResult:
        r += indent + prefix + "# one or more of the following criteria\n\n"

        for subc in CRITERIA_CLASSES:
            r += dump_yaml(subc, indent=indent + INDENT, show_doc=show_doc,
                           comment_optional=comment_optional, index=index)
    elif c == Action:
        r += indent + prefix + ("# one or more of the following "
                                "action-specific parameters\n\n")


        for subc in ACTION_CLASSES: