class ProbesFilter(object):
    def __init__(self, probe_ids=None, countries=None):
        self.probe_ids = probe_ids or []
        self.countries = countries or []

    def __contains__(self, probe):
        # probe = Probe object
        if not self.probe_ids and not self.countries:
            return True
        if self.probe_ids and probe.id not in self.probe_ids:
            return False
        if self.countries and probe.country_code not in self.countries:
            return False
        return True class MsmProcessingUnit(object):
    @staticmethod
    def _load_msm(id=None, key=None):
        try:
            return Measurement(id=id, key=key)
        except (CousteauGenericError, APIResponseError) as e:
            raise MeasurementProcessingError(
                "Error while retrieving measurement details: {}".format(
                    repr(e)
                )
            )

    def __init__(self, ip_cache=None, msm_id=None, key=None, key_file=None):
        self.ip_cache = ip_cache

        self.msm_id = msm_id

        if self.msm_id is None:
            raise ConfigError(
                "Missing measurement ID: it must be specified in the "
                "monitor configuration file or provided using the "
                "command line argument --measurement-id."
            )

        self.key = key
        self.key_file = key_file

        if not self.key and self.key_file:
            if os.path.isfile(self.key_file):
                try:
                    with open(self.key_file, "r") as f:
                        self.key = f.read().rstrip()
                except Exception as e:
                    raise ConfigError(
                        "Can't read RIPE Atlas Key file {}: {}".format(
                            self.key_file, str(e)
                        )
                    )
            else:
                raise MissingFileError(self.key_file)

        # Measurement validation

        msm = self._load_msm(id=self.msm_id, key=self.key)

        self.msm_is_public = msm.is_public
        self.msm_type = msm.type.lower()
        self.msm_af = int(msm.protocol)
        self.msm_status = msm.status
        self.msm_status_id = msm.status_id
        self.msm_is_running = self.msm_status_id in [0, 1, 2]
        self.msm_is_oneoff = msm.is_oneoff
        self.msm_interval = msm.interval

        if self.msm_type not in ["traceroute", "ping", "sslcert", "dns"]:
            raise MeasurementProcessingError(
                "Unhandled measurement's type: "
                "{}".format(self.msm_type)
            )

        # dictionary of "<probe_id>": "<json>"
        #   where <json> = https://atlas.ripe.net/docs/rest/#probe
        self.probes = {}      # TODO: add a cache for probes

        # Used by ExpResCriterion [get|set]_parsed_res.
        # ExpResCriterion objects use this dictionary as
        # a cache when they parse results and transform
        # them in a way they can subsequently understand.
        # Multiple criteria for the same ExpectedResult
        # can parse results only once and store here the
        # data they need for the result_matches method.
        self.parsed_res = {}

    def get_probe(self, origin):
        if isinstance(origin, Result):
            prb_id = origin.probe_id
        elif isinstance(origin, str):
            prb_id = int(origin)
        elif isinstance(origin, int):
            prb_id = origin
        else:
            raise ResultProcessingError(
                "Unknown origin type: {}".format(type(origin))
            )

        return Probe(self.probes[str(prb_id)], self.msm_af)

    def update_probes(self, results):
        # List of probe IDs whose details must be retrieved
        unknown_probes_ids = []

        for json_result in results:
            probe_id = json_result["prb_id"]
            if str(probe_id) not in self.probes:
                # Probe never seen before, add it to the list of
                # probes to request details for
                if probe_id not in unknown_probes_ids:
                    unknown_probes_ids.append(probe_id)

        # Get details about missing probes

        if len(unknown_probes_ids) > 0:
            try:
                json_probes = ProbeRequest(id__in=unknown_probes_ids)
                for json_probe in json_probes:
                    self.probes[str(json_probe["id"])] = json_probe
            except Exception as e:
                raise ResultProcessingError(
                    "Error while retrieving probes info: {}".format(str(e))
                )

    def get_latest_result_ts(self):
        # Must be implemented in child classes
        raise NotImplementedError()

    def download(self, start=None, stop=None, latest_results=None,
                 probe_ids=None):
        atlas_params = {
            "msm_id": self.msm_id,
            "key": self.key
        }

        if probe_ids:
            assert isinstance(probe_ids, list)
            assert all(isinstance(_, int) for _ in probe_ids)
            atlas_params["probe_ids"] = probe_ids

            logger.info(
                "Downloading results for probe ID{s}{IDs}{more}...".format(
                    s="s " if len(probe_ids) > 1 else " ",
                    IDs=", ".join(map(str, probe_ids[0:3])),
                    more=" and {} more".format(len(probe_ids) - 3)
                         if len(probe_ids) > 3 else ""
                )
            )
        else:
            logger.info("Downloading results...")

        if not latest_results:
            atlas_class = AtlasResultsRequest

            msm_results_days_limit = Config.get("misc.msm_results_days_limit")
            latest_result_ts = self.get_latest_result_ts()

            if start:
                tpl = "{start}"

            elif latest_result_ts:
                start = datetime.datetime.fromtimestamp(
                    latest_result_ts + 1, tz=pytz.UTC
                )
                tpl = "{start} (last result received)"
            elif msm_results_days_limit > 0:
                start = datetime.date.today() + datetime.timedelta(
                    days=-msm_results_days_limit
                )
                tpl = ("{start} (last {limit} days on the basis of global "
                       "config msm_results_days_limit option)")
            else:
                tpl = (" - no start time given, downloading results from "
                       "the beginning of the measurement")

            logger.info(" - start time: " + tpl.format(
                start=start,
                limit=msm_results_days_limit
            ))

            if start:
                atlas_params["start"] = start

            if stop:
                tpl = "{stop}"
            else:
                tpl = (" - no stop time given, downloading results "
                       "until the last one")

            logger.info(" - stop time: " + tpl.format(stop=stop))

            if stop:
                atlas_params["stop"] = stop
        else:
            logger.info(" - retrieving latest results only")

            atlas_class = AtlasLatestRequest

        try:
            atlas_request = atlas_class(**atlas_params)

            is_success, results = atlas_request.create()
        except Exception as e:
            raise MeasurementProcessingError(
                "Error while retrieving results: {}".format(str(e))
            )

        if is_success:
            return results
        else:
            err = str(results)
            if isinstance(results, dict):
                if "detail" in results:
                    err = results["detail"]
                elif "error" in results:
                    if "detail" in results["error"]:
                        err = results["error"]["detail"]
            raise MeasurementProcessingError(str(err)) class ParsedResult_RTT(ParsedResult):

    PROPERTIES = ["rtt"]

    @property
    def rtt(self):
        return self.get_attr_from_cache("rtt")

    def prepare(self):
        if isinstance(self.result, TracerouteResult):
            res_rtt = self.result.last_median_rtt
        elif isinstance(self.result, PingResult):
            res_rtt = self.result.rtt_median
        else:
            raise NotImplementedError()

        self.set_attr_to_cache("rtt", res_rtt) class ParsedResult_DstResponded(ParsedResult):

    PROPERTIES = ["responded"]

    @property
    def responded(self):
        return self.get_attr_from_cache("responded")

    def prepare(self):
        if isinstance(self.result, TracerouteResult):
            res_responded = self.result.destination_ip_responded
        elif isinstance(self.result, PingResult):
            res_responded = self.result.packets_received > 0
        elif isinstance(self.result, SslResult):
            res_responded = len(self.result.certificates) > 0
        else:
            raise NotImplementedError()

        self.set_attr_to_cache("responded", res_responded) class ParsedResult_DstIP(ParsedResult):

    PROPERTIES = ["dst_ip"]

    @property
    def dst_ip(self):
        return self.get_attr_from_cache("dst_ip")

    def prepare(self):
        if isinstance(self.result, (TracerouteResult, PingResult, SslResult)):
            res_dst_ip = self.result.destination_address
        else:
            raise NotImplementedError()

        self.set_attr_to_cache("dst_ip", res_dst_ip) class ParsedResult_CertFps(ParsedResult):

    PROPERTIES = ["cer_fps"]

    @property
    def cer_fps(self):
        return self.get_attr_from_cache("cer_fps")

    def prepare(self):
        if not isinstance(self.result, SslResult):
            raise NotImplementedError()

        res_cer_fps = \
            [cer.checksum_sha256.upper() for cer in self.result.certificates]
        res_cer_fps = sorted(res_cer_fps)

        self.set_attr_to_cache("cer_fps", res_cer_fps) class ParsedResult_DstAS(ParsedResult_TracerouteBased):

    PROPERTIES = ["dst_as"]

    @property
    def dst_as(self):
        return self.get_attr_from_cache("dst_as")

    def prepare(self):
        ParsedResult_TracerouteBased.prepare(self)

        if len(self.as_path) > 0:
            dst_as = self.as_path[-1].replace(
                "S", str(self.msm_proc_unit.get_probe(self.result).asn)
            )
            dst_as = int(dst_as)
            self.set_attr_to_cache("dst_as", dst_as)
        else:
            self.set_attr_to_cache("dst_as", None) class ParsedResult_UpstreamAS(ParsedResult_TracerouteBased):

    PROPERTIES = ["upstream_as"]

    @property
    def upstream_as(self):
        return self.get_attr_from_cache("upstream_as")

    def prepare(self):
        ParsedResult_TracerouteBased.prepare(self)

        if len(self.as_path) > 1:
            upstream_as = self.as_path[-2].replace(
                "S", str(self.msm_proc_unit.get_probe(self.result).asn)
            )
            upstream_as = int(upstream_as)
            self.set_attr_to_cache("upstream_as", upstream_as)
        else:
            self.set_attr_to_cache("upstream_as", None) class ParsedResult_ASPath(ParsedResult_TracerouteBased):

    PROPERTIES = ["as_path", "as_path_ixps"] class ParsedResult_DNSHeader(ParsedResult_DNSBased):

    PROPERTIES = ["flags", "rcode"]

    DNS_HEADER_FLAGS = ("aa", "ad", "cd", "qr", "ra", "rd")

    # keep in sync with ExpResCriterion_DNSRCode docstring
    DNS_RCODES = ["NOERROR", "FORMERR", "SERVFAIL", "NXDOMAIN",
                  "NOTIMP", "REFUSED", "YXDOMAIN", "YXRRSET",
                  "NXRRSET", "NOTAUTH", "NOTZONE", "BADVERS"]

    @property
    def flags(self):
        return self.get_attr_from_cache("flags")

    @property
    def rcode(self):
        return self.get_attr_from_cache("rcode")

    def prepare(self):
        if not isinstance(self.result, DnsResult):
            raise NotImplementedError()

        self.set_attr_to_cache("flags", [])
        self.set_attr_to_cache("rcode", None)

        if not self.response.abuf:
            return

        response_flags = set()
        for flag in self.DNS_HEADER_FLAGS:
            if getattr(self.response.abuf.header, flag):
                response_flags.add(flag)

        self.set_attr_to_cache("flags", sorted(response_flags))
        self.set_attr_to_cache("rcode", self.response.abuf.header.return_code) class ParsedResult_EDNS(ParsedResult_DNSBased):

    PROPERTIES = ["edns", "edns_size", "edns_do", "edns_nsid"]

    @property
    def edns(self):
        return self.get_attr_from_cache("edns")

    @property
    def edns_size(self):
        return self.get_attr_from_cache("edns_size")

    @property
    def edns_do(self):
        return self.get_attr_from_cache("edns_do")

    @property
    def edns_nsid(self):
        return self.get_attr_from_cache("edns_nsid")

    def prepare(self):
        if not isinstance(self.result, DnsResult):
            raise NotImplementedError()

        self.set_attr_to_cache("edns", self.response.abuf.edns0 is not None)

        self.set_attr_to_cache("edns_size", None)
        self.set_attr_to_cache("edns_do", None)
        self.set_attr_to_cache("edns_nsid", None)

        if not self.response.abuf.edns0:
            return

        self.set_attr_to_cache("edns_size", self.response.abuf.edns0.udp_size)
        self.set_attr_to_cache("edns_do", self.response.abuf.edns0.do)
        for option in self.response.abuf.edns0.options:
            if option.nsid:
                self.set_attr_to_cache("edns_nsid", option.nsid) class ParsedResult_DNSAnswers(ParsedResult_DNSBased):
    # Used only for measurement analysis.
    # The ExpResCriterion_DNSAnswers class doesn't behave like
    # other ExpResCriterion-derived classes.

    PROPERTIES = ["dns_answers"]

    @property
    def dns_answers(self):
        return self.get_attr_from_cache("dns_answers")

    @staticmethod
    def get_record_info(record):
        # return touple (name, type, value)

        r = (record.name, record.type)
        if record.type in ["A", "AAAA"]:
            r += (record.address,)
        elif record.type in ["CNAME", "NS", "PTR"]:
            r += (record.target,)
        elif record.type == "MX":
            r += ("{} {}".format(record.preference, record.mail_exchanger),)
        elif record.type == "SOA":
            r += ("{} {} {} {} {} {} {}".format(
                record.mname, record.rname, record.serial, record.refresh,
                record.retry, record.expire, record.minimum),)
        elif record.type == "TXT":
            r += (record.data_string,)
        else:
            r += ("unhandled record type",)
        return r

    def prepare(self):
        if not isinstance(self.result, DnsResult):
            raise NotImplementedError()

        self.set_attr_to_cache("dns_answers", [])

        if not self.response.abuf:
            return

        records = [self.get_record_info(r)
                   for r in self.response.abuf.answers if r.name and r.type]

        if len(records) == 0:
            return

        ordered_records = sorted(records, key=lambda x: (x[0], x[1], x[2]))

        self.set_attr_to_cache("dns_answers", ordered_records)
from collections import Counter
from itertools import groupby, combinations, chain
from .Helpers import ProbesFilter
from .MsmProcessingUnit import MsmProcessingUnit
from .ParsedResults import ParsedResult_RTT, ParsedResult_DstResponded, \
                           ParsedResult_DstIP, ParsedResult_CertFps, \
                           ParsedResult_DstAS, ParsedResult_UpstreamAS, \
                           ParsedResult_ASPath, ParsedResult_DNSHeader, \
                           ParsedResult_EDNS, ParsedResult_DNSAnswers
from ripe.atlas.sagan import Result
import json
    }
    PROPERTIES_ORDER = ["rtt", "responded", "dst_ip", "cer_fps", "dst_as",
                        "upstream_as", "as_path", "as_path_ixps", "rcode",
                        "flags", "edns", "edns_size", "edns_do", "edns_nsid",
                        "dns_answers"]

    def __init__(self, analyzer, results, **kwargs):
        self.analyzer = analyzer

        self.use_json = kwargs.get("use_json", False)

        self.results = results  # Result objects

        self.kwargs = kwargs

        self.props = {}  # property_name: [(property_val, probe_id), ...]

    def get_parsed_results(self):
        for result in self.results:
            yield (
                self.PARSED_RESULTS_CLASS(self.analyzer, result),
                result.probe_id
            )

    def analyze(self):
        for parsed_result, probe_id in self.get_parsed_results():
            try:
                parsed_result.prepare()
            except NotImplementedError:
                continue

            for prop in parsed_result.PROPERTIES:
                if prop not in self.props:
                    self.props[prop] = []
                self.props[prop].append(
                    (getattr(parsed_result, prop), probe_id)
                )

        text = ""
        json_object = {}

        for prop in self.PROPERTIES_ORDER:
            if prop not in self.PROPERTIES_ANALYZERS_CLASSES:
                raise NotImplementedError()
            if prop not in self.props:
                continue
            prop_analyzer_class = self.PROPERTIES_ANALYZERS_CLASSES[prop]
            prop_analyzer = prop_analyzer_class(self.analyzer,
                                                self.props[prop],
                                                **self.kwargs)

            if self.use_json:
                json_object[prop] = prop_analyzer.analyze()
            else:
                text += prop_analyzer.analyze()

        if self.use_json:
            return json_object
        else:
            return text


class ResultsAnalyzer_RTT(BaseResultsAnalyzer):

    PARSED_RESULTS_CLASS = ParsedResult_RTT


class ResultsAnalyzer_DstResponded(BaseResultsAnalyzer):

    PARSED_RESULTS_CLASS = ParsedResult_DstResponded


class ResultsAnalyzer_DstIP(BaseResultsAnalyzer):

    PARSED_RESULTS_CLASS = ParsedResult_DstIP


class ResultsAnalyzer_CertFps(BaseResultsAnalyzer):

    PARSED_RESULTS_CLASS = ParsedResult_CertFps


class ResultsAnalyzer_DstAS(BaseResultsAnalyzer):

    PARSED_RESULTS_CLASS = ParsedResult_DstAS


class ResultsAnalyzer_UpstreamAS(BaseResultsAnalyzer):

    PARSED_RESULTS_CLASS = ParsedResult_UpstreamAS


class ResultsAnalyzer_ASPath(BaseResultsAnalyzer):

    PARSED_RESULTS_CLASS = ParsedResult_ASPath


class ResultsAnalyzer_DNSBased(BaseResultsAnalyzer):

    def get_parsed_results(self):
        for result in self.results:
            try:
                for response in result.responses:
                    if response.abuf:
                        yield (
                            self.PARSED_RESULTS_CLASS(
                                self.analyzer, result, response),
                            result.probe_id
                        )
            except AttributeError:
                return


class ResultsAnalyzer_DNSHeader(ResultsAnalyzer_DNSBased):

    PARSED_RESULTS_CLASS = ParsedResult_DNSHeader


class ResultsAnalyzer_EDNS(ResultsAnalyzer_DNSBased):


    PARSED_RESULTS_CLASS = ParsedResult_EDNS