class BuildProcessor:
    """
    Build programs
    """

    block_scheme = Dict({
        "attributes": ATTRIBUTES_TYPE,
        "run_config": Dict(unknown_keys=True) // Description("Run configuration for this program block"),
        "build_config": Dict({
            "cmd": Str() // Default("") // Description("Command to build this program block, might randomize it"),
            "number": (PositiveInt() | NonExistent()) // Default(1)
                      // Description("Number of times to build this program"),
            "working_dir": (DirName() | NonExistent()) // Default(".")
                // Description("Working directory in which the build command is run"),
            "revision": (Str() | Int() | NonExistent()) // Default(-1)
                // Description("Used version control system revision of the program (-1 is the current revision)"),
            "branch": (Str() | NonExistent()) // Default("")
                // Description("Used version control system branch (default is the current branch)"),
            "base_dir": (DirName() | NonExistent()) // Default(".")
                // Description("Base directory that contains everything to build an run the program")
        }, unknown_keys=True) // Description("Build configuration for this program block")
    })   # type: Dict
    """ Type scheme of the program block configurations """

    def __init__(self, build_blocks: t.Optional[t.List[t.Dict[str, t.Any]]] = None):
        """
        Creates a build processor for the passed build block configurations.

        :param build_blocks: passed build block configurations
        """
        if build_blocks is None:
            typecheck(Settings()["build/in"], ValidYamlFileName())
            with open(Settings()["build/in"], "r") as f:
                build_blocks = yaml.safe_load(f)
        self.build_blocks = self.preprocess_build_blocks(build_blocks)
        typecheck(Settings()["build/out"], FileName())
        typecheck_locals(build_blocks=List())
        self.out = Settings()["build/out"]  # type. str
        """ Temporary directory in which the building takes place """

    @classmethod
    def preprocess_build_blocks(cls, blocks: t.List[t.Dict[str, t.Any]]) -> t.List[t.Dict[str, t.Any]]:
        """
        Pre process and check build blocks

        :return: pre processed copy
        """
        typecheck(blocks, List(cls.block_scheme))
        build_blocks = [cls.block_scheme.get_default() for _ in range(len(blocks))]
        """
        Build block configurations.
        type: t.Optional[t.List[t.Dict[str, t.Any]]]
        """
        for i, block in enumerate(blocks):
            for key in block.keys():
                build_blocks[i][key].update(block[key])
            typecheck(build_blocks[i], cls.block_scheme, "build block {}".format(i))
        return build_blocks

    def build(self):
        """
        Build the configured programs.
        """
        run_blocks = []
        try:
            for id, block in enumerate(self.build_blocks):
                error = None
                try:
                    block_builder = Builder(id, block["build_config"]["working_dir"],
                                            block["build_config"]["cmd"], block["build_config"]["revision"],
                                            block["build_config"]["number"],
                                            block["build_config"]["base_dir"], block["build_config"]["branch"])
                    working_dirs = block_builder.build()
                except BuilderKeyboardInterrupt as err:
                    working_dirs = err.result
                    error = err.error
                except BuildError as err:
                    err.log()
                    raise err
                block["run_config"]["cwd"] = working_dirs
                run_blocks.append({
                    "attributes": block["attributes"],
                    "run_config": block["run_config"]
                })

                if error:
                    raise error
        except KeyboardInterrupt as err:
            with open(self.out, "w") as f:
                yaml.dump(run_blocks, f)
            raise err
        with open(self.out, "w") as f:
            yaml.dump(run_blocks, f)

    @classmethod
    def store_example_config(cls, file: str, comment_out_defaults: bool = False):
        import click
        with click.open_file(file, "w") as f:
            print(List(cls.block_scheme).get_default_yaml(comment_out_defaults=comment_out_defaults),
                  file=f) class Builder:
    """
    Allows the building of a program configured by a program block configuration.
    """

    def __init__(self, id: int, build_dir: str, build_cmd: str, revision: t.Union[str, int], number: int,
                 base_dir: str, branch: str):
        """
        Creates a new builder for a program block.

        :param build_dir: working directory in which the build command is run
        :param build_cmd: command to build this program block
        :param revision: used version control systemrand revision of the program (-1 is the current revision)
        :param number: number of times to build this program
        :param base_dir: base directory that contains everything to build an run the program
        :param branch: used version control system branch
        """
        typecheck(build_dir, DirName())
        typecheck(build_cmd, str)
        typecheck(revision, Int() | Str())
        typecheck(number, PositiveInt())
        self.build_dir = os.path.join(base_dir, build_dir) if base_dir != "." else build_dir # type: str
        """ Working directory in which the build command is run """
        self.build_cmd = build_cmd  # type: str
        """ Command to build this program block """
        self.revision = revision  # type: t.Union[str, int]
        """ Used version control system revision of the program """
        self.number = number  # type: int
        """ Number of times to build this program """
        self.vcs_driver = VCSDriver.get_suited_vcs(dir=self.build_dir, branch=None if branch == "" else branch)
        """ Used version control system driver """
        self.id = id

    def build(self, thread_count: t.Optional[int] = None) -> t.List[str]:
        """
        Build the program block in parallel with at maximum `thread_count` threads in parallel.

        :param thread_count: number of threads to use at maximum to build the configured number of time,
               defaults to `build/threads`
        :return: list of base directories for the different builds
        """
        thread_count = thread_count or Settings()["build/threads"]

        time_tag = datetime.datetime.now().strftime("%s%f")

        def tmp_dirname(i: t.Union[int, str] = "base"):
            tmp_dir = os.path.join(Settings()["tmp_dir"], "build", time_tag, str(i))
            return tmp_dir

        tmp_dir = tmp_dirname()
        if self.revision == -1 and self.number == 1:
            tmp_dir = self.build_dir
            os.makedirs(tmp_dir, exist_ok=True)
            submit_queue = queue.Queue()
            submit_queue.put(BuilderQueueItem(self.id, None, tmp_dir, tmp_dir, self.build_cmd))
            BuilderThread(0, submit_queue).run()

            logging.info("Finished building")
            return [tmp_dir]

        logging.info("Create base temporary directory and copy build directory")
        os.makedirs(tmp_dir)
        self.vcs_driver.copy_revision(self.revision, self.build_dir, tmp_dir)

        ret_list = []
        submit_queue = queue.Queue()
        threads = []
        for i in range(0, self.number):
            tmp_build_dir = tmp_dirname(i)
            submit_queue.put(BuilderQueueItem(self.id, i, tmp_build_dir, tmp_dir, self.build_cmd))
            ret_list.append(tmp_build_dir)
        try:
            for i in range(min(thread_count, self.number)):
                thread = BuilderThread(i, submit_queue)
                threads.append(thread)
                thread.start()
            for thread in threads:
                thread.join()
        except BaseException as err:
            for thread in threads:
                thread.stop = True
            shutil.rmtree(tmp_dir)
            logging.info("Error while building")
            raise BuilderKeyboardInterrupt(err, ret_list)
        logging.info("Finished building")
        shutil.rmtree(tmp_dir)
        return ret_list class BuildError(Exception):

    def __init__(self, thread: int, item: BuilderQueueItem, error: 'RecordedError'):
        super().__init__("Thread {}: Build error for block {!r}".format(thread, item.id))
        self.thread = thread
        self.item = item
        self.error = error

    def log(self):
        logging.error("Build error for {}".format(self.item.id))
        logging.error("out: {!r}".format(self.error.out))
        logging.error("err: {!r}".format(self.error.err))
        logging.error("cmd: {!r}".format(self.item.build_cmd)) class Screen:
    """
    A simple curses based screen to build a watch like behavior for temci exec
    """

    def __init__(self, scroll: bool = False, keep_first_lines: int = 0, print_buffer_on_exit: bool = True):
        """
        :param keep_first_lines: keep the first n lines on the screen fixed when scrolling
        """
        self.scr = curses.initscr()
        curses.noecho()
        try:
            curses.cbreak()
        except:
            pass
        curses.curs_set(0)
        self.scr.keypad(True)
        self.current_line = 0
        self.buffer = [""]  # type: List[str]
        self._shown_buffer = [""]  # type: List[str]
        self.keep_first_lines = keep_first_lines
        self.x = 0
        self.y = self.keep_first_lines
        self.print_buffer_on_exit = print_buffer_on_exit
        self.enabled = True
        if scroll:
            t = threading.Thread(target=self._create_scroll_thread())
            t.setDaemon(True)
            t.start()

    def _create_scroll_thread(self) -> Callable:
        def func():
            while True:
                c = self.scr.getch()
                curses.flushinp()
                updated = True
                if c == curses.KEY_LEFT:
                    self._move_cursor(x_offset=-1)
                elif c == curses.KEY_RIGHT:
                    self._move_cursor(x_offset=1)
                elif c == curses.KEY_UP or c == curses.KEY_SR:
                    self._move_cursor(y_offset=-1)
                elif c == curses.KEY_DOWN or c == curses.KEY_SF:
                    self._move_cursor(y_offset=1)
                else:
                    updated = False
                if updated:
                    self._flush2()
                time.sleep(0.05)
        return func

    def __enter__(self):
        return self

    def _copy_over(self):
        self._shown_buffer = self.buffer

    def reset(self):
        """
        Clear the current screen recording (does not change the displayed screen)
        """
        self.current_line = 0
        self.buffer = [""]

    def isatty(self):
        """ Only required for click """
        return True

    def write(self, text: str):
       if not self.enabled:
           pass
       text = text.replace("\n", "\r\n")
       for line in text.splitlines(keepends=True):
           first = True
           for part in line[0:len(line) - (1 if line.endswith("\n") else 0)].split("\r"):
              self._write_single_line(part, -1 if first else 0)
              first = False
           if line.endswith("\n"):
               self.advance_line()

    def _write_single_line(self, text: str, y: int = -1):
        """ -1: end of line """
        assert "\r" not in text and "\r" not in text
        if y == -1:
            self.buffer[self.current_line] += text
        else:
            cur = self.buffer[self.current_line]
            self.buffer[self.current_line] = cur[0:y] + text + cur[len(text)+y:]

    def advance_line(self):
        self.current_line += 1
        self.buffer.append("")

    def flush(self):
        pass

    def display(self):
        """
        Replace the current screen with the recorded. Call reset() if you don't want to add to this screen.
        """
        self._copy_over()
        self._flush2()


    def _flush2(self):
        """ Refreshes the screen """
        max_y, max_x = self.scr.getmaxyx()
        for y in range(0, min(max_y, self.keep_first_lines, len(self._shown_buffer))):
            self.scr.addstr(0, y, self._shown_buffer[y][0:max_x].replace("\\[", ""))
        end = max(0, min(len(self._shown_buffer) - 1, max_y - self.keep_first_lines + self.y))
        for y in range(self.y, end):
            self.scr.addstr(y + self.keep_first_lines - self.y, 0, " " * max_x)
            self.scr.addstr(y + self.keep_first_lines - self.y, 0,
                            self._shown_buffer[y][self.x:min(len(self._shown_buffer[y]) - self.x, max_x) + self.x])
        for y in range(end + self.keep_first_lines - self.y, max_y):
            try:
                self.scr.addstr(y, 0, " " * max_x)
            except:
                pass
        self.scr.refresh()

    def _move_cursor(self, x_offset: int = 0, y_offset: int = 0):
        self.x = max(0, self.x + x_offset)
        self.y = max(self.keep_first_lines, self.y + y_offset)

    def writelines(self, lines: List[str]):
        for line in lines:
            self.write(line)

    def __exit__(self, exc_type, exc_val, exc_tb):
        try:
            curses.nocbreak()
            self.scr.keypad(False)
            curses.echo()
            curses.endwin()
            os.system('stty sane')
        except BaseException as ex:
            pass
        if self.print_buffer_on_exit:
            sys.stdout.writelines([s + "\n" for s in self._shown_buffer])

    def enable(self):
        self.enabled = True

    def disable(self):
        self.enabled = False def chown(path: Union[str, TextIOWrapper, BufferedRandom, BufferedRWPair, BufferedWriter, IOBase]):
    if not bench_as_different_user():
        return
    if isinstance(path, IOBase) and path.isatty():
        return
    if not isinstance(path, str):
        return chown(path.name)
    ids = ()
    try:
        ids = get_bench_uid_and_gid()
    except:
        global _logged_chown_error
        if not _logged_chown_error:
            logging.warn("Could not get user id for user {} therefore no chowning possible".format(get_bench_user()))
            _logged_chown_error = True
        return
    try:
        os.chown(path, *ids)
    except FileNotFoundError:
        pass def recursive_exec_for_leafs(data: dict, func, _path_prep = []):
def has_root_privileges() -> bool:
def has_pdflatex() -> bool:
def does_command_succeed(cmd: str) -> bool:
def warn_for_pdflatex_non_existence_once(_warned = [False]):
def get_cache_line_size(cache_level: int = None) -> t.Optional[int]:
def get_memory_page_size() -> int:
def get_distribution_name() -> str:
def get_distribution_release() -> str:
def does_program_exist(program: str) -> bool:
def on_apple_os() -> bool:
    def __enter__(self):
        def try_wait(self, wait_flags):
    def __exit__(self, exc_type, exc_val, exc_tb):
def join_strs(strs: t.List[str], last_word: str = "and") -> str:
def can_import(module: str) -> bool:
def sphinx_doc() -> bool:
def get_doc_for_type_scheme(type_scheme: 'Type') -> str:
def document(**kwargs: t.Dict[str, str]):
    def dec(klass):
    def __call__(cls, *args, **kwargs):
    def __init__(self):
    def __delitem__(self, key):
    def __getitem__(self, key):
    def __setitem__(self, key, value):
    def __iter__(self):
    def values(self) -> t.List:
    def keys(self) -> t.List:
    def __len__(self):
    def items(self) -> t.List[t.Tuple[t.Any, t.Any]]:
    def from_list(cls, items: t.Optional[list], key_func: t.Callable[[t.Any], t.Any]) -> 'InsertionTimeOrderedDict':
def geom_std(values: t.List[float]) -> float:
def parse_timespan(time: str) -> float:
def rusage_header() -> str:
class proc_wait_with_rusage:
class Singleton(type):
class InsertionTimeOrderedDict: def send_mail(recipient: str, subject: str, content: str, attached_files: t.List[str] = None):
    """
    Sends a mail to the recipient with the passed subject, content and attached files.

    :param recipient: recipient of the mail, i.e. a mail address
    :param subject: subject of the mail
    :param content: content of the mail
    :param attached_files: optional list of names of files that are attached to the mail
    """
    from email.mime.multipart import MIMEMultipart
    from email.mime.text import MIMEText
    import smtplib
    if recipient == "":
        return
    sender = ""
    try:
        sender = "temci@" + hostname()
    except subprocess.CalledProcessError:
        sender = "temci@temci"
    try:
        attached_files = attached_files or []
        msg = MIMEMultipart()
        msg["From"] = sender
        msg["To"] = recipient
        msg["Subject"] = subject
        msg.attach(MIMEText(content))
        for file in attached_files:
            try:
                with open(file, "r") as f:
                    msg_part = MIMEText(f.read())
                    msg_part.add_header('Content-Disposition', 'attachment', filename=file)
                    msg.attach(msg_part)
            except IOError:
                pass
        smtp = smtplib.SMTP("localhost")
        smtp.sendmail(sender, recipient, msg.as_string())
        smtp.quit()
    except BaseException as ex:
        logging.error(ex) class RunWorkerPool(AbstractRunWorkerPool):
    """
    This run worker pool implements the sequential benchmarking of program blocks.
    """

    def __init__(self, run_driver_name: str = None, end_time: float = -1):
        super().__init__(run_driver_name, end_time)
        if run_driver_name is None:
            run_driver_name = RunDriverRegistry().get_used()
        self.cpuset = CPUSet(parallel=0) if Settings()["run/cpuset/active"] else CPUSet(active=False)
        self.parallel_number = 1

    def submit(self, block: RunProgramBlock, id: int, runs: int):
        typecheck(block, RunProgramBlock)
        typecheck(runs, NaturalNumber())
        typecheck(id, NaturalNumber())
        block.is_enqueued = True
        self.result_queue.put((block, self.run_driver.benchmark(block, runs, self.cpuset, timeout=self.next_block_timeout()), id))
        block.is_enqueued = False

    def results(self, expected_num: int) -> ResultGenerator:
        for i in range(expected_num):#while not self.result_queue.empty():
            yield self.result_queue.get()

    def teardown(self):
        super().teardown()
        self.run_driver.teardown()
        if self.cpuset is not None:
            self.cpuset.teardown() class ParallelRunWorkerPool(AbstractRunWorkerPool):
    """
    This run worker pool implements the parallel benchmarking of program blocks.
    It uses a server-client-model to benchmark on different cpu cores.
    """

    def __init__(self, run_driver_name: str = None, end_time: float = -1):
        super().__init__(run_driver_name, end_time)
        if run_driver_name is None:
            run_driver_name = RunDriverRegistry().get_used()
        if Settings()["run/cpuset/active"]:
            self.cpuset = CPUSet()
        else:
            self.cpuset = CPUSet(active=False)
            #raise ValueError("Only works with run/cpuset/active=True")
        self.parallel_number = self.cpuset.parallel_number
        logging.info("Using {} parallel processes to benchmark.".format(self.parallel_number))
        self.threads = []  # type: t.List[BenchmarkingThread]
        """ Running benchmarking threads """
        try:
            for i in range(0, self.parallel_number):
                thread = BenchmarkingThread(i, self, self.run_driver, self.cpuset)
                self.threads.append(thread)
                thread.start()
        except BaseException:
            logging.error("Forced teardown of ParallelRunWorkerPool")
            self.teardown()
            raise

    def submit(self, block: RunProgramBlock, id: int, runs: int):
        if self.time_left() <= 0:
            return
        typecheck(block, RunProgramBlock)
        typecheck(runs, NaturalNumber())
        typecheck(id, NaturalNumber())
        block.is_enqueued = True
        self.submit_queue.put((block, id, runs))

    def results(self, expected_num: int) -> ResultGenerator:
        #while not self.intermediate_queue.empty() or not self.submit_queue.empty() or not self.result_queue.empty():
        for i in range(expected_num):#while not self.submit_queue.empty() or not self.result_queue.empty() or not self.submit_queue.all_tasks_done:
            yield self.result_queue.get()

    def teardown(self):
        super().teardown()
        self.cpuset.teardown()
        self.run_driver.teardown()
        try:
            for thread in self.threads:
                thread.stop = True
                #thread.teardown()
        except:
            pass class AbstractRunWorkerPool:
    """
    An abstract run worker pool that just deals with the hyper threading setting.
    """

    def __init__(self, run_driver_name: str = None, end_time: float = -1):
        """
        Create an instance.

        :param run_driver_name: name of the used run driver, if None the one configured in the settings is used
        """
        self.submit_queue = Queue()  # type: Queue
        """ Queue for submitted but not benchmarked run program blocks """
        self.result_queue = Queue()  # type: Queue
        """
        Queue of benchmarking results.
        The queue items are tuples consisting of
        the benchmarked block, the benchmarking result and the
        blocks id.
        """
        self.parallel_number = 1  # type: int
        """ Number of instances in which the benchmarks takes place in parallel """
        run_driver_name = run_driver_name or RunDriverRegistry.get_used()
        self.run_driver = RunDriverRegistry().get_for_name(run_driver_name)  # type: AbstractRunDriver
        """ Used run driver instance """
        self.cpuset = None  # type: CPUSet
        """ Used cpu set instance """
        self.end_time = end_time
        self._disabled_ht_cores = []  # type: t.List[int]
        if Settings()["run/disable_hyper_threading"]:
            if not has_root_privileges():
                logging.warning("Can't disable hyper threading as root privileges are missing")
                return
            #if Settings()["run/cpuset/active"]:
            #    logging.warning("Currently disabling hyper threading doesn't work well in combination with cpusets")
            #    return
            self._disabled_ht_cores = self.disable_hyper_threading()

    def submit(self, block: RunProgramBlock, id: int, runs: int):
        """
        Submits the passed block for "runs" times benchmarking.
        It also sets the blocks is_enqueued property to True.

        :param block: passed run program block
        :param id: id of the passed block
        :param runs: number of individual benchmarking runs
        """
        raise NotImplementedError()

    def results(self, expected_num: int) \
            -> ResultGenerator:
        """
        A generator for all available benchmarking results.
        The items of this generator are tuples consisting of
        the benchmarked block, the benchmarking result and the
        blocks id.

        :param expected_num: expected number of results
        """
        raise NotImplementedError()

    def teardown(self):
        """
        Tears down the inherited run driver.
        This should be called if all benchmarking with this pool is finished.
        """
        if Settings()["run/disable_hyper_threading"]:
            self.enable_hyper_threading(self._disabled_ht_cores)

    def time_left(self) -> float:
        """
        Does not work properly if self.end_time == -1
        """
        return max(self.end_time - time.time(), 0)

    def has_time_left(self) -> bool:
        return self.end_time == -1 or self.time_left() > 0

    @classmethod
    def get_hyper_threading_cores(cls) -> t.List[int]:
        """
         Adapted from http://unix.stackexchange.com/a/223322
        """
        total_logical_cpus = 0
        total_physical_cpus = 0
        total_cores = 0
        cpu = None

        logical_cpus = {}
        physical_cpus = {}
        cores = {}

        hyperthreading = False

        for line in open('/proc/cpuinfo').readlines():
            if re.match('processor', line):
                cpu = int(line.split()[2])

                if cpu not in  logical_cpus:
                    logical_cpus[cpu] = []
                    total_logical_cpus += 1

            if re.match('physical id', line):
                phys_id = int(line.split()[3])

                if phys_id not in physical_cpus:
                    physical_cpus[phys_id] = []
                    total_physical_cpus += 1

            if re.match('core id', line):
                core = int(line.split()[3])

                if core not in cores:
                    cores[core] = []
                    total_cores += 1

                cores[core].append(cpu)

        if (total_cores * total_physical_cpus) * 2 == total_logical_cpus:
            hyperthreading = True

        ht_cores = []  # type: t.List[int]

        if hyperthreading:

            for c in cores:
                for p, val in enumerate(cores[c]):
                    if p > 0:
                        ht_cores.append(val)
        return ht_cores

    def next_block_timeout(self) -> float:
        timeout = parse_timespan(Settings()["run/max_block_time"])
        if not self.has_time_left():
            return 0
        if timeout > -1:
            return max(min(self.time_left() if self.end_time != -1 else timeout, timeout), 0)
        return -1 if self.end_time == -1 else max(self.time_left(), 0)

    @classmethod
    def disable_hyper_threading(cls) -> t.List[int]:
        if has_root_privileges():
            cores = cls.get_hyper_threading_cores()
            cls._set_status_of_ht_cores(cores, 0)
            return cores
        return []

    @classmethod
    def enable_hyper_threading(cls, disabled_cores: t.List[int]):
        if has_root_privileges():
            cls._set_status_of_ht_cores(disabled_cores, 1)

    @classmethod
    def _set_status_of_ht_cores(cls, ht_cores: t.List[int], online_status: int):
        if len(ht_cores) == 0:
            return
        arg = "\n".join("echo {} > /sys/devices/system/cpu/cpu{}/online"
                        .format(online_status, core_id) for core_id in ht_cores)
        proc = subprocess.Popen(["/bin/sh", "-c", arg],
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                universal_newlines=True)
        out, err = proc.communicate()
        if proc.poll() > 0:
            raise EnvironmentError("Error while disabling the hyper threaded cores: " + str(err)) class RunProgramBlock:
    """
    An object that contains every needed information of a program block.
    """

    def __init__(self, id: int, data: t.Dict[str, t.Any], attributes: t.Dict[str, str], run_driver_class: type = None):
        """
        Creates an instance.

        :param data: run driver configuration for this run program block
        :param attributes: attributes of this run program block
        :param run_driver_class: used type of run driver with this instance
        """
        self.run_driver_class = run_driver_class or RunDriverRegistry.get_class(
            RunDriverRegistry.get_used())  # type: type
        """ Used type of run driver """
        self.type_scheme = self.run_driver_class.block_type_scheme  # type: Type
        """ Configuration type scheme of the used run driver """
        self.data = deepcopy(self.run_driver_class.block_type_scheme.get_default())  # type: t.Dict[str, t.Any]
        """ Run driver configuration """
        self.data.update(data)
        self.attributes = attributes  # type: t.Dict[str, str]
        """ Describing attributes of this run program block """
        self.is_enqueued = False  # type: bool
        """ Is this program block enqueued in a run worker pool queue? """
        self.id = id  # type: int
        """ Id of this run program block """
        self.tags = attributes["tags"] if "tags" in self.attributes else None
        from temci.report.rundata import get_for_tags
        self.max_runs = get_for_tags("run/max_runs_per_tag", "run/max_runs", self.tags, min)
        if "max_runs" in self.data and self.data["max_runs"] > -1:
            self.max_runs = min(self.max_runs, self.data["max_runs"])
        self.min_runs = get_for_tags("run/min_runs_per_tag", "run/min_runs", self.tags, max)
        if "min_runs" in self.data and self.data["min_runs"] > -1:
            self.min_runs = max(self.min_runs, self.data["min_runs"])
        self.runs = get_for_tags("run/runs_per_tag", "run/runs", self.tags, max)
        if "runs" in self.data:
            self.runs = self.data["runs"]
        if self.runs != -1:
            self.min_runs = max(self.min_runs, self.runs)
            self.max_runs = min(self.max_runs, self.runs)

    def __getitem__(self, key: str) -> t.Any:
        """
        Returns the value associated with the given key.
        """
        return self.data[key]

    def __setitem__(self, key: str, value):
        """
        Sets the value associated with the passed key to the new value.

        :param key: passed key
        :param value: new value
        :raises TypeError: if the value hasn't the expected type
        """
        value_name = "run programm block[{}]".format(key)
        typecheck(self.type_scheme, Dict)
        typecheck(value, self.type_scheme[key], value_name=value_name)
        self.data[key] = value

    def __contains__(self, key) -> bool:
        """ Does the run driver configuration data contain the passed key? """
        return key in self.data

    def __repr__(self) -> str:
        return "RunDataBlock({}, {})".format(self.data, self.attributes)

    def copy(self) -> 'RunProgramBlock':
        """
        Copy this run program block.
        Deep copies the data and uses the same type scheme and attributes.
        """
        return RunProgramBlock(self.id, deepcopy(self.data), self.attributes, self.run_driver_class)

    @classmethod
    def from_dict(cls, id: int, data: t.Dict, run_driver: type = None):
        """
        Structure of data::

             {
                "attributes": {"attr1": ..., ...},
                "run_config": {"prop1": ..., ...},
                "build_config": {"prop1": ..., ...}
             }

        :param id: id of the block (only used to track them later)
        :param data: used data
        :param run_driver: used RunDriver subclass
        :return: new RunProgramBlock
        """
        typecheck(data, Dict({
            "attributes": Dict(unknown_keys=True, key_type=Str()) // Default({}),
            "run_config": Dict(unknown_keys=True),
            "build_config": Optional(BuildProcessor.block_scheme["build_config"]) // Default({}),
        }))
        block = RunProgramBlock(id, data["run_config"], data["attributes"] if "attributes" in data else {}, run_driver)
        return block

    def to_dict(self) -> t.Dict:
        """
        Serializes this instance into a data structure that is accepted by the ``from_dict`` method.
        """
        return {
            "attributes": self.attributes,
            "run_config": self.data
        }

    def description(self) -> str:
        if "description" in self.attributes and self.attributes["description"] is not None:
            return self.attributes["description"]
        return ", ".join("{}={}".format(key, self.attributes[key]) for key in self.attributes) class BenchmarkingResultBlock:
    """
    Result of the benchmarking of one block.
    It includes the error object if an error occurred.
    """

    def __init__(self, data: t.Dict[str, t.List[Number]] = None, error: BaseException = None,
                 recorded_error: 'RecordedError' = None):
        """
        Creates an instance.

        :param data: measured data per measured property
        :param error: exception object if something went wrong during benchmarking
        :return:
        """
        self.data = collections.defaultdict(lambda: [])  # type: t.Dict[str, t.List[Number]]
        """ Measured data per measured property """
        if data:
            self.add_run_data(data)
        self.error = error  # type: t.Optional[BaseException]
        """ Exception object if something went wrong during benchmarking """
        self.recorded_error = recorded_error

    def properties(self) -> t.List[str]:
        """ Get a list of the measured properties """
        return list(self.data.keys())

    def add_run_data(self, data: t.Dict[str, t.Union[Number, t.List[Number]]]):
        """
        Add data.

        :param data: data to be added (measured data per property)
        """
        typecheck(data, Dict(unknown_keys=True, key_type=Str(), value_type=Int() | Float() | List(Int() | Float())))
        for prop in data:
            if isinstance(data[prop], list):
                self.data[prop].extend(data[prop])
            else:
                self.data[prop].append(data[prop])

                # def _to_dict(self):
                #    """
                #    Serializes this instance into a data structure that is accepted by the ``from_dict`` method.
                #    """
                #    return {
                #        "properties": self.properties(),
                #        "data": self.data
                #    }
                #
                # @classmethod
                # def _from_dict(cls, source: dict):
                #    typecheck(source, Dict({
                #        "data": Dict(unknown_keys=True)
                #    }, unknown_keys=True))
                #    return BenchmarkingResultBlock(source["data"]) class RunDriverRegistry(AbstractRegistry):
    """
    The registry for run drivers.
    """

    settings_key_path = "run"
    use_key = "driver"
    use_list = False
    default = "exec"
    registry = {}
    plugin_synonym = ("run driver", "run drivers")

    @classmethod
    def register(cls, name: str, klass: type, misc_type: Type, deprecated: bool = False):
        assert issubclass(klass, AbstractRunDriver)
        super().register(name, klass, misc_type, deprecated)
        if not sphinx_doc() or deprecated:
            return
        klass.__doc__ += """

    Block configuration format for the run configuration:

    .. code-block:: yaml

        {yaml}

    """.format(yaml="\n        ".join(klass.get_full_block_typescheme().string_representation().split("\n"))) class ExecRunDriver(AbstractRunDriver):
    """
    Implements a simple run driver that just executes one of the passed run_cmds
    in each benchmarking run.
    It measures the time  using the perf stat tool (runner=perf_stat).

    The constructor calls the ``setup`` method.
    """

    settings_key_path = "run/exec_plugins"
    use_key = "exec_active"
    use_list = True
    default = []
    block_type_scheme = Dict({
        "run_cmd": (List(Str()) | Str()) // Default("") // Description("Commands to benchmark"),
        "cmd": Str() // Default("") // Description("Command to benchmark, adds to run_cmd"),
        "env": Dict(unknown_keys=True, key_type=Str()) // Default({}) // Description("Environment variables"),
        "cmd_prefix": List(Str()) // Default([]) // Description("Command to append before the commands to benchmark"),
        "revision": (Int(lambda x: x >= -1) | Str()) // Default(-1) // Description("Used revision (or revision number)."
                                                                                   "-1 is the current revision, checks out "
                                                                                   "the revision"),
        "cwd": (List(Str()) | Str()) // Default(".") // Description("Execution directories for each command"),
        "runner": ExactEither().dont_typecheck_default() // Default("time") // Description("Used runner"),
        "disable_aslr": Bool() // Default(False) // Description("Disable the address space layout randomization"),
        "validator": ExecValidator.config_type_scheme // Description(
            "Configuration for the output and return code validator"),
        "max_runs": Int(lambda x: x >= -1) // Default(-1) // Description("Override all other max run"
                                                                         "specifications if > -1"),
        "min_runs": Int(lambda x: x >= -1) // Default(-1) // Description("Override all other min run"
                                                                         "specifications if > -1"),
        "runs": Int(lambda x: x >= -1) // Default(-1) // Description("Override min run and max run"
                                                                     "specifications if > -1"),
        "parse_output": Bool() // Default(False) // Description("Parse the program output as a YAML dictionary of "
                                                                "that gives for a specific property a measurement. "
                                                                "Not all runners support it."),
        "rusage_properties": ValidRusagePropertyList() // Default([])
                          // Description("Measured properties for rusage that are stored in the benchmarking result")
    }, unknown_keys=True)

    registry = {}

    def __init__(self, misc_settings: dict = None):
        super().__init__(misc_settings)
        self._dirs = {}
        self.runner = None  # type: t.Optional[ExecRunner]

    def _setup_block(self, block: RunProgramBlock):
        if isinstance(block["run_cmd"], List(Str())):
            block["run_cmds"] = block["run_cmd"] + [block["cmd"]] if block["cmd"] != "" else block["run_cmd"]
        else:
            block["run_cmds"] = [block["run_cmd"] + block["cmd"]]
        block["run_cmds"] = [cmd.replace("&", "&&").replace("$SUDO$", "&SUDO&") for cmd in block["run_cmds"]]
        if isinstance(block["cwd"], List(Str())):
            if len(block["cwd"]) != len(block["run_cmd"]) and not isinstance(block["run_cmd"], str):
                raise ValueError("Number of passed working directories {} "
                                 "is unequal with number of passed run commands {}"
                                 .format(len(block["cwd"]), len(block["run_cmd"])))
            block["cwds"] = block["cwd"]
        else:
            block["cwds"] = [block["cwd"]] * len(block["run_cmds"])
        self.uses_vcs = block["revision"] != -1
        self.vcs_driver = None
        self.tmp_dir = ""
        if self.uses_vcs and block.id not in self._dirs:
            self.vcs_driver = VCSDriver.get_suited_vcs(".")
            self.tmp_dir = os.path.join(Settings()["tmp_dir"], datetime.datetime.now().strftime("%s%f"))
            os.mkdir(self.tmp_dir)
            self._dirs[block.id] = os.path.join(self.tmp_dir, str(block.id))
            os.mkdir(self._dirs[block.id])
            self.vcs_driver.copy_revision(block["revision"], ".", self._dirs[block.id])
            block["working_dir"] = self._dirs[block.id]
        if self.misc_settings["runner"] != "":
            block["runner"] = self.misc_settings["runner"]
        if block["runner"] == "rusage":
            block["rusage_properties"].extend(block["rusage"]["properties"])
            block["rusage"]["properties"] = []
        block["parse_output"] |= self.misc_settings["parse_output"]
        super()._setup_block(block)

    def benchmark(self, block: RunProgramBlock, runs: int,
                  cpuset: CPUSet = None, set_id: int = 0,
                  timeout: float = -1) -> BenchmarkingResultBlock:
        from temci.report.rundata import RecordedInternalError
        block = block.copy()
        try:
            self._setup_block(block)
            gc.collect()
            gc.disable()
        except IOError as err:
            return BenchmarkingResultBlock(error=err, recorded_error=RecordedInternalError.for_exception(err))
        try:
            res = self._benchmark(block, runs, cpuset, set_id, timeout=timeout)
        except BenchmarkingProgramError as ex:
            return BenchmarkingResultBlock(error=ex, recorded_error=ex.recorded_error)
        except BaseException as ex:
            return BenchmarkingResultBlock(error=ex, recorded_error=RecordedInternalError.for_exception(ex))
        finally:
            gc.enable()
        try:
            self._teardown_block(block)
        except BaseException as err:
            return BenchmarkingResultBlock(error=err, recorded_error=RecordedInternalError.for_exception(err))
        return res

    ExecResult = namedtuple("ExecResult", ['time', 'stderr', 'stdout', 'rusage'])
    """ A simple named tuple named ExecResult with to properties: time, stderr and stdout """

    def _benchmark(self, block: RunProgramBlock, runs: int, cpuset: CPUSet = None,
                   set_id: int = 0, timeout: float = -1):
        block = block.copy()
        self.runner = self.get_runner(block)
        self.runner.setup_block(block, cpuset, set_id)
        results = []
        for i in range(runs):
            self._setup_block_run(block)
            results.append(self._exec_command(block["run_cmds"], block, cpuset, set_id, timeout=timeout))
        res = None  # type: BenchmarkingResultBlock
        for exec_res in results:
            if not self.runner.supports_parsing_out and block["parse_output"]:
                logging.warn("Runner {} does not support the `parse_output` option")
            res = self.runner.parse_result(exec_res, res, block["parse_output"]
                                           and self.runner.supports_parsing_out)
            res = self._parse_rusage(exec_res.rusage, res, block["rusage_properties"])
        return res

    def _parse_rusage(self, rusage: 'resource.rusage_struct', res: BenchmarkingResultBlock, properties: t.List[str]) \
            -> BenchmarkingResultBlock:
        res = res or BenchmarkingResultBlock()
        res.add_run_data({prop:rusage.__getattribute__("ru_" + prop) for prop in properties})
        return res

    def _exec_command(self, cmds: list, block: RunProgramBlock,
                      cpuset: CPUSet = None, set_id: int = 0, redirect_out: bool = True,
                      timeout: float = -1) -> ExecResult:
        """
        Executes one randomly chosen command of the passed ones.
        And takes additional settings in the passed run program block into account.

        :param cmds: list of commands
        :param block: passed run program block
        :return: time in seconds the execution needed to finish
        """
        typecheck(cmds, List(Str()))
        rand_index = random.randrange(0, len(cmds)) if self.misc_settings["random_cmd"] else 0
        cmd = cmds[rand_index]
        if "$SUDO$" not in cmd:
            cmd = "$SUDO$ " + cmd
        if cmd.count("$SUDO$") == 1:
            cmd += " $SUDO$"
        pre, center, post = cmd.split("$SUDO$")
        cmd_tmp_file = tempfile.NamedTemporaryFile(delete=False)
        try:
            arg = self.misc_settings["argument"] if "argument" in self.misc_settings else ""
            center = center.replace("&SUDO&", "$SUDO$").replace("&&", "&").replace("$ARGUMENT", arg)
            cmd_tmp_file.write(center.encode())
            os.chmod(cmd_tmp_file.name, os.stat(cmd_tmp_file.name).st_mode | stat.S_IEXEC)
            cmd_tmp_file.flush()
            cmd_tmp_file.close()
            if bench_as_different_user():
                shutil.chown(cmd_tmp_file.name, get_bench_user())
                cmd = pre + " sudo -u {} -E  PATH={} sh {}".format(get_bench_user(),
                                                               shlex.quote(Settings()["env"]["PATH"]),
                                                               cmd_tmp_file.name) + post
            else:
                cmd = pre + " " + cmd_tmp_file.name + " " + post

            cwd = block["cwds"][rand_index]
            executed_cmd = block["cmd_prefix"] + [cmd]
            if cpuset is not None and has_root_privileges():
                executed_cmd.insert(0, "cset proc --move --force --pid $$ {} > /dev/null" \
                                    .format(cpuset.get_sub_set(set_id)))
            env = get_env_setting() if bench_as_different_user() else os.environ.copy()
            env.update(block["env"])
            env.update({'LC_NUMERIC': 'en_US.UTF-8'})
            # print(env["PATH"])
            executed_cmd = "; ".join(executed_cmd)
            proc = None

            try:
                t = time.time()
                logging.debug("Running {} in directory {}".format(repr(executed_cmd), cwd))
                proc = subprocess.Popen(["/bin/sh", "-c", executed_cmd],
                                        stdout=subprocess.PIPE if redirect_out else None,
                                        stderr=subprocess.PIPE if redirect_out else None,
                                        universal_newlines=True,
                                        cwd=cwd,
                                        env=env, )
                # preexec_fn=os.setsid)
                rusage = None
                with proc_wait_with_rusage():
                    if not redirect_out:
                        proc.wait(timeout=timeout if timeout > -1 else None)
                        out = "<not redirected>"
                        err = out
                    else:
                        out, err = proc.communicate(timeout=timeout if timeout > -1 else None)
                    t = time.time() - t
                    rusage = proc.rusage
                    logging.debug("""
===STDOUT===
{}
===END STDOUT===
===STDERR===
{}
===END STDERR===""".format(str(out), str(err)))
                if redirect_out:
                    ExecValidator(block["validator"]).validate(cmd, clean_output(out), clean_output(err), proc.poll())
                # if proc.poll() > 0:
                #    msg = "Error executing " + cmd + ": "+ str(err) + " " + str(out)
                # logging.error(msg)
                #    raise BenchmarkingError(msg)
                return self.ExecResult(time=t, stderr=str(err), stdout=str(out), rusage=rusage)
            except Exception as ex:
                if proc is not None:
                    try:
                        proc.terminate()
                        # os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
                    except BaseException as err:
                        pass
                if isinstance(ex, subprocess.TimeoutExpired):
                    raise TimeoutException(executed_cmd, timeout, str(out), str(err), proc.returncode)
                raise
        finally:
            os.remove(cmd_tmp_file.name)

    def teardown(self):
        super().teardown()
        if hasattr(self, "tmp_dir") and os.path.exists(self.tmp_dir):
            shutil.rmtree(self.tmp_dir)

    runners = {}
    """ Dictionary mapping a runner name to a runner class """

    @classmethod
    def register_runner(cls) -> t.Callable[[type], type]:
        """ Decorator to register a runner (has to be sub class of ``ÈxecRunner``)."""

        def dec(klass):
            assert issubclass(klass, ExecRunner)
            cls.runners[klass.name] = klass
            cls.block_type_scheme["runner"] |= E(klass.name)
            Settings().modify_type_scheme("run/exec_misc/runner", lambda x: x | E(klass.name))
            cls.block_type_scheme[klass.name] = klass.misc_options
            if klass.__doc__ is not None:
                header = ""  # "Description of {} (class {}):\n".format(name, klass.__qualname__)
                lines = str(klass.__doc__.strip()).split("\n")
                lines = map(lambda x: "  " + x.strip(), lines)
                description = Description(header + "\n".join(lines))
                klass.__description__ = description.description
            else:
                klass.__description__ = ""
            # if not sphinx_doc():
            #    return
            klass.__doc__ = (klass.__doc__ or "") + """

    To use this runner with name ``{name}`` either set the ``runner`` property of a run configuration
    or the setting under the key ``run/exec_misc/runner`` to its name.

        """
            if klass.supports_parsing_out:
                klass.__doc__ += """
    This runner supports the ``parse_output`` option.   
             """
            if klass.misc_options not in [Dict(), Dict({}), None]:
                klass.__doc__ += """

    The runner is configured by modifying the ``{name}`` property of a run configuration. This configuration
    has the following structure:

    .. code-block:: yaml

        {yaml}

    """.format(name=klass.name, yaml="\n        ".join(klass.misc_options.string_representation().split("\n")))
            return klass

        return dec

    @classmethod
    def get_runner(cls, block: RunProgramBlock) -> 'ExecRunner':
        """
        Create the suitable runner for the passed run program block.

        :param block: passed run program block
        """
        return cls.runners[block["runner"]](block)

    def get_property_descriptions(self) -> t.Dict[str, str]:
        return self.runner.get_property_descriptions() if self.runner else {}

    def get_used_plugins(self) -> t.List[str]:
        """
        Get the list of name of the used plugins (use_list=True)
        or the names of the used plugin (use_list=False).
        """
        used = super().get_used()
        for plugin in PRESET_PLUGIN_MODES[self.misc_settings["preset"]][0].split(","):
            if plugin not in used and plugin != "":
                used.append(plugin)
        order = self.misc_settings["plugin_order"]
        used = sorted(used, key=lambda plugin: order.index(plugin) if plugin in order else -1)
        return used def is_perf_available() -> bool:
    """
    Is the ``perf`` tool available?
    """
    try:
        subprocess.check_call("perf stat -x';' -e cycles,cpu-clock,task-clock,instructions,branch-misses,"
                              "cache-references -- echo 1", shell=True,
                              stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
    except BaseException:
        return False
    return True def filter_runs(blocks: t.List[t.Union['RunProgramBlock','RunData']], included: t.List[str]) -> t.List['RunProgramBlock']:
    """
    Filter run blocks (all: include all), identified by their description or tag or their number in the file
    (starting at zero) and run datas (only identified by their description and tag). The include query can also
    consist of regular expressions

    :param blocks: blocks or run datas to filter
    :param included: include query
    :return: filtered list
    """
    def parts(block) -> t.List[str]:
        arr = []
        if "description" in block.attributes:
            arr.append(block.attributes["description"])
        if isinstance(block, RunProgramBlock):
            arr.append(str(block.id))
        if "tags" in block.attributes:
            arr.extend(block.attributes["tags"])
        return arr

    def match(block):
        return ["all"] == included or any(p in included or any(re.fullmatch(i, p) for i in included)
                                          for p in parts(block))
    l = [block for block in blocks if match(block)]
    for i, x in enumerate(l):
        if isinstance(x, RunProgramBlock):
            x.id = i
    return l def log_program_error(recorded_error: 'RecordedInternalError'):
    from temci.report.rundata import RecordedProgramError
    if isinstance(recorded_error, RecordedProgramError):
        if recorded_error.out:
            logging.error("output\n\n" + recorded_error.out)
        if recorded_error.err:
            logging.error("error\n\n" + recorded_error.err) class RunDataStatsHelper(object):
    """
    This class helps to simplify the work with a set of run data observations.
    """

    def __init__(self, runs: t.List[RunData], tester: Tester = None, external_count: int = 0,
                 property_descriptions: t.Dict[str, str] = None,
                 errorneous_runs: t.List[RunData] = None,
                 included_blocks: str = None, env_info: FORMATTED_ENV_INFO = None):
        """
        Don't use the constructor use init_from_dicts if possible.

        :param runs: list of run data objects
        :param tester: used tester or tester that is set in the settings
        :param external_count: Number of external program blocks (blocks for which the data was obtained in a
        different benchmarking session)
        :param property_descriptions: mapping of some properties to their descriptions or longer versions
        :param errorneous_runs: runs that resulted in errors
        :param included_blocks: include query
        :param env_info: formatted environment info
        """
        self.tester = tester or TesterRegistry.get_for_name(TesterRegistry.get_used(),
                                                            Settings()["stats/uncertainty_range"])  # type: Tester
        """ Used statistical tester """
        typecheck(runs, List(T(RunData)))
        self.runs = filter_runs(runs, included_blocks or Settings()["report/included_blocks"])  # type: t.List[RunData]
        self.errorneous_runs = errorneous_runs or [r for r in self.runs if r.has_error()]
        self.runs = [r for r in self.runs if not r.has_error() or (any(len(v) > 0 for v,p in r.data.items()))]
        """ Data of serveral runs from several measured program blocks """
        self.external_count = external_count  # type: int
        """
        Number of external program blocks (blocks for which the data was obtained in a different benchmarking session)
        """
        self.property_descriptions = property_descriptions or {}  # type: t.Dict[str, str]
        self.env_info = env_info or []

    def clone(self, runs: t.List[RunData] = None, tester: Tester = None, external_count: int = None,
              property_descriptions: t.Dict[str, str] = None) -> 'RunDataStatsHelper':
        """
        Clones this instance and replaces the given instance properties.

        :param runs: list of run data objects
        :param tester: used tester or tester that is set in the settings
        :param external_count: Number of external program blocks (blocks for which the data was obtained in a
        different benchmarking session)
        :param property_descriptions: mapping of some properties to their descriptions or longer versions
        :return: cloned instance
        """
        def alt(new, old):
            return new if new is not None else old
        return RunDataStatsHelper(runs=alt(runs, self.runs), tester=alt(tester, self.tester),
                                  external_count=alt(external_count, self.external_count),
                                  property_descriptions=alt(property_descriptions, self.property_descriptions),
                                  errorneous_runs=self.errorneous_runs, env_info=self.env_info)

    def make_descriptions_distinct(self):
        """
        Append numbers to descriptions if needed to make them unique
        """
        descr_attrs = defaultdict(lambda: 0)  # type: t.Dict[str, int]
        descr_nr_zero = {}  # type: t.Dict[str, RunData]

        for single in self.runs:
            if "description" in single.attributes and single.attributes["description"] is not None:
                descr = single.attributes["description"]
                num = descr_attrs[descr]
                descr_attrs[descr] += 1
                if num != 0:
                    single.attributes["description"] += " [{}]".format(num)
                    if num == 1:
                        descr_nr_zero[descr].attributes["description"] += " [0]"
                else:
                    descr_nr_zero[descr] = single
                single.attributes["__description"] = descr

    def get_description_clusters(self) -> t.Dict[str, t.List['RunData']]:
        """
        Set of runs per description, call RunDataStatsHelper.make_descriptions_distinct first

        :return: set of runs per description
        """
        clusters = util.InsertionTimeOrderedDict()
        for r in self.runs:
            d = r.attributes["__description" if "__description" in r.attributes else "description"] \
                if "description" in r.attributes \
                else ""
            if d not in clusters:
                clusters[d] = []
            clusters[d].append(r)
        return clusters

    def get_description_clusters_and_single(self) -> t.Tuple[t.List['RunData'], t.Dict[str, t.List['RunData']]]:
        """
        Set of runs per description, call RunDataStatsHelper.make_descriptions_distinct first

        :return: set of runs per description
        """
        clusters = self.get_description_clusters()
        new_clusters = util.InsertionTimeOrderedDict()
        single = []
        for n, c in clusters.items():
            if len(c) == 1:
                single.extend(c)
            else:
                new_clusters[n] = c
        return single, new_clusters

    def properties(self) -> t.List[str]:
        """
        Returns a sorted list of all properties that exist in all run data blocks.
        """
        if not self.runs:
            return []
        props = set(self.runs[0].properties)
        for rd in self.runs[1:]:
            if rd:
                props = props.intersection(rd.properties)
        return list(sorted(props))

    @classmethod
    def init_from_file(cls, filename: str) -> 'RunDataStatsHelper':
        """
        Load the runs from the passed file. This file has to encode the runs using YAML in the format stated in
        :func:`RunDataStatsHelper.init_from_dicts`

        :param filename: name of the file
        :raises ValueError: if the file has an incorrect structure
        :return: created stats helper
        """
        with open(filename) as f:
            return cls.init_from_dicts(yaml.safe_load(f))

    @classmethod
    def init_from_dicts(cls, runs: t.List[t.Union[t.Dict[str, str], t.Dict[str, t.List[Number]]]] = None,
                        external: bool = False,
                        included_blocks: str = None) -> 'RunDataStatsHelper':
        """
        Expected structure of the stats settings and the runs parameter::

            "stats": {
                "tester": ...,
                "properties": ["prop1", ...],
                # or
                "properties": ["prop1", ...],
                "uncertainty_range": (0.1, 0.3)
            }

            "runs": [
                {"attributes": {"attr1": ..., ..., ["description": …], ["tags": …]},
                 "data": {"__ov-time": [...], ...},
                 "error": {"return_code": …, "output": "…", "error_output": "…"},
                 "internal_error": {"message": "…"} (either "error" or "internal_error" might be present)
                 ["property_descriptions": {"__ov-time": "Overall time", …}]
                 ["env_info": … ]
                 },
                 ...
            ]


        :param runs: list of dictionaries representing the benchmarking runs for each program block
        :param external: are the passed runs not from this benchmarking session but from another?
        :param included_blocks: include query
        :raises ValueError: if runs parameter has an incorrect structure
        :return: created stats helper
        """
        typecheck(runs, List(
            Dict({
                "data": Dict(key_type=Str(), value_type=List(Int() | Float()), unknown_keys=True) | NonExistent(),
                "run_config": Dict(unknown_keys=True)
            }, unknown_keys=True) |
                    RunData.block_type_scheme |
                    RunData.property_descriptions_scheme |
                    RunData.env_info_scheme),
                value_name="runs parameter")
        run_datas = []
        runs = runs or [] # type: t.List[dict]
        prop_descrs = {}  # type: t.Dict[str, str]
        env_info = []
        for run in runs:
            props = {}
            if "property_descriptions" in run:
                prop_descrs.update(run["property_descriptions"])
            elif "env_info" in run:
                env_info = run["env_info"]
            else:
                if "data" not in run:
                    run["data"] = {}
                error = None
                if "error" in run:
                    error = RecordedProgramError(run["error"]["message"], run["error"]["output"],
                                                 run["error"]["error_output"], run["error"]["return_code"])
                elif "internal_error" in run:
                    error = RecordedInternalError(run["internal_error"]["message"])
                run_datas.append(RunData(run["data"], run["attributes"] if "attributes" in run else {}, recorded_error=error,
                                         external=external))
        return RunDataStatsHelper(run_datas, external_count=len(run_datas) if external else 0,
                                  property_descriptions=prop_descrs, included_blocks=included_blocks,
                                  env_info=env_info)

    def _is_uncertain(self, property: str, data1: RunData, data2: RunData) -> bool:
        return self.tester.is_uncertain(data1[property], data2[property])

    def _is_equal(self, property: str, data1: RunData, data2: RunData) -> bool:
        return self.tester.is_equal(data1[property], data2[property])

    def _is_unequal(self, property: str, data1: RunData, data2: RunData) -> bool:
        return self.tester.is_unequal(data1[property], data2[property])

    def is_uncertain(self, p_val: float) -> bool:
        """
        Does the passed probability of the null hypothesis for two samples lie in the uncertainty range?
        :param p_val: passed probability of the null hypothesis
        """
        return min(*Settings()["stats/uncertainty_range"]) <= p_val <= max(*Settings()["stats/uncertainty_range"])

    def is_equal(self, p_val: float) -> bool:
        """ Is the passed value above the uncertainty range for null hypothesis probabilities? """
        return p_val > max(*Settings()["stats/uncertainty_range"])

    def is_unequal(self, p_val: float) -> bool:
        """ Is the passed value above the uncertainty range for null hypothesis probabilities? """
        return p_val < min(*Settings()["stats/uncertainty_range"])

    def _speed_up(self, property: str, data1: RunData, data2: RunData):
        """
        Calculates the speed up from the second to the first
        (e.g. the first is RESULT * 100 % faster than the second).
        """
        return (np.mean(data1[property]) - np.mean(data2[property])) \
               / np.mean(data1[property])

    def _estimate_time_for_run_datas(self, run_bin_size: int, data1: RunData, data2: RunData,
                                     min_runs: int, max_runs: int) -> float:
        if min(len(data1), len(data2)) == 0 \
                or "__ov-time" not in data1.properties \
                or "__ov-time" not in data2.properties:
            return max_runs
        needed_runs = []
        for prop in set(data1.properties).intersection(data2.properties):
            estimate = self.tester.estimate_needed_runs(data1[prop], data2[prop],
                                                                run_bin_size, min_runs, max_runs)
            needed_runs.append(estimate)
        avg_time = max(np.mean(data1["__ov-time"]), np.mean(data2["__ov-time"]))
        return max(needed_runs) * avg_time

    def get_program_ids_to_bench(self) -> t.List[int]:
        """
        Returns the ids (the first gets id 0, …) of the program block / run data object that
        should be benchmarked again.
        """
        to_bench = set()
        for (i, run) in enumerate(self.runs):
            if i in to_bench or run is None or run.has_error():
                continue
            for j in range(i):
                if j in to_bench or self.runs[j] is None or self.runs[j].has_error():
                    continue
                run2 = self.runs[j]
                if run2.min_values() == 0 or run.min_values() == 0 or \
                        any(self._is_uncertain(prop, run, run2) for prop in set(run.properties)
                        .intersection(run2.properties)):
                    to_bench.add(i)
                    to_bench.add(j)
        return [i - self.external_count for i in to_bench if i >= self.external_count]

    def estimate_time(self, run_bin_size: int, min_runs: int, max_runs: int) -> float:
        """
        Roughly erstimates the time needed to finish benchmarking all program blocks.
        It doesn't take any parallelism into account. Therefore divide the number by the used parallel processes.

        :warning: Doesn't work well.

        :param run_bin_size: times a program block is benchmarked in a single block of time
        :param min_runs: minimum number of allowed runs
        :param max_runs: maximum number of allowed runs
        :return: estimated time in seconds or float("inf") if no proper estimation could be made
        """
        to_bench = self.get_program_ids_to_bench()
        max_times = [0 for i in self.runs]
        for i in to_bench:
            run = self.runs[i]
            for j in to_bench:
                max_time = self._estimate_time_for_run_datas(run_bin_size, run, self.runs[j],
                                                             min_runs, max_runs)
                max_times[i] = max(max_times[i], max_time)
                max_times[j] = max(max_times[j], max_time)
                if max_time == float("inf"):
                    return float("inf")
        return sum(max_times)

    def estimate_time_for_next_round(self, run_bin_size: int, all: bool) -> float:
        """
        Roughly estimates the time needed for the next benchmarking round.

        :param run_bin_size: times a program block is benchmarked in a single block of time and the size of a round
        :param all: expect all program block to be benchmarked
        :return: estimated time in seconds
        """
        if "__ov-time" not in self.properties():
            return -1
        summed = 0
        to_bench = range(0, len(self.runs)) if all else self.get_program_ids_to_bench()
        for i in to_bench:
            summed += np.mean(self.runs[i]["__ov-time"] if "__ov-time" in self.runs[i].data else 0) * run_bin_size
        return summed

    #ef add_run_data(self, data: t.Dict[str, t.List[Number]] = None, attributes: t.Dict[str, str] = None,
    #                property_descriptions: t.Dict[str, str] = None) -> int:
    #   """
    #   Adds a new run data (corresponding to a program block) and returns its id.
    #
    #   :param data: benchmarking data of the new run data object
    #   :param attributes: attributes of the new run data object
    #   :param property_descriptions: mapping of property to a description
    #   :return: id of the run data object (and its corresponding program block)
    #   """
    #   self.runs.append(RunData(data, attributes=attributes, property_descriptions))
    #   return len(self.runs) - 1

    def discard_run_data(self, id: int):
        """
        Disable that run data object with the given id.
        """
        self.runs[id].discarded = True

    def add_data_block(self, program_id: int, data_block: t.Dict[str, t.List[Number]]):
        """
        Add block of data for the program block with the given id.

        :param program_id: id of the program.
        :param data_block: list of data from several benchmarking runs of the program block
        :raises ValueError: if the program block with the given id doesn't exist
        """
        program_id += self.external_count
        assert program_id >= self.external_count
        if program_id >= len(self.runs):
            raise ValueError("Program block with id {} doesn't exist".format(program_id - self.external_count))
        self.runs[program_id].add_data_block(data_block)

    def add_error(self, program_id: id, error: RecordedError):
        """
        Set the error for a program
        """
        program_id += self.external_count
        assert program_id >= self.external_count
        self.runs[program_id].recorded_error = error
        self.errorneous_runs.append(self.runs[program_id])

    def has_error(self, program_id: int) -> bool:
        """ Is there an error recorded for the program with the given id? """
        return self.runs[program_id + self.external_count].has_error()

    def get_evaluation(self, with_equal: bool, with_unequal: bool, with_uncertain: bool,
                       blocks: t.List[RunData] = None) -> dict:
        """

        Structure of the returned list items::

            - data: # set of two run data objects
              properties: # information for each property that is equal, ...
                  -prop:
                      - equal: True/False
                        uncertain: True/False
                        p_val: probability of the null hypothesis
                        speed_up: speed up from the first to the second
                        description: description of the property

        :param with_equal: with tuple with at least one "equal" property
        :param with_unequal: ... unequal property
        :param with_uncertain: include also uncertain properties
        :param blocks: blocks to compare
        :return: list of tuples for which at least one property matches the criteria
        """
        blocks = blocks or self.runs
        arr = []
        for i in range(0, len(blocks) - 1):
            for j in range(i + 1, len(blocks)):
                if blocks[i].discarded or blocks[j].discarded:
                    continue
                data = (blocks[i], blocks[j])
                props = {}
                for prop in self.properties():
                    map = {"p_val": self.tester.test(data[0][prop], data[1][prop]),
                           "speed_up": self._speed_up(prop, *data),
                           "description": prop,
                           "equal": self._is_equal(prop, *data),
                           "unequal": self._is_unequal(prop, *data),
                           "uncertain": self._is_uncertain(prop, *data)}
                    if map["unequal"] == with_unequal and map["equal"] == with_equal \
                            and map["uncertain"] == with_uncertain:
                        props[prop] = map
                if len(props) > 0:
                    arr.append({
                        "data": data,
                        "properties": props
                    })
        return arr

    def serialize(self) -> t.List[t.Union[t.Dict[str, str], t.Dict[str, t.List[Number]], FORMATTED_ENV_INFO]]:
        """
        Serialize this instance into a data structure that is accepted by the ``init_from_dicts`` method.
        """
        ret = [x.to_dict() for x in self.runs if not x.discarded]
        if self.property_descriptions:
            ps = {}  # type: t.Dict[str, str]
            props = self.properties()
            for prop in props:
                if prop in self.property_descriptions:
                    ps[prop] = self.property_descriptions[prop]
            ret.append({"property_descriptions": ps})
            ret.append({"env_info": self.env_info})
        return ret

    def valid_runs(self) -> t.List[RunData]:
        """ Number of valid (with measured data) runs """
        res = [x for x in self.runs if not x.discarded]
        return res

    def exclude_properties(self, properties: t.List[str]) -> 'RunDataStatsHelper':
        """
        Create a new instance without the passed properties.

        :param properties: excluded properties
        :return: new instance
        """
        runs = []
        for run in self.runs:
            if not run.discarded:
                runs.append(run.exclude_properties(properties))
        return self.clone(runs=runs)

    def include_properties(self, properties: t.List[str]) -> 'RunDataStatsHelper':
        """
        Create a new instance with only the passed properties

        :param properties: included properties, can be regular expressions
        :return: new instance
        """
        runs = []
        for run in self.runs:
            if not run.discarded:
                runs.append(run.include_properties(properties))
        return self.clone(runs=runs)

    def exclude_invalid(self) -> t.Tuple['RunDataStatsHelper', 'ExcludedInvalidData']:
        """
        Exclude all properties of run datas that only have zeros or NaNs as measurements.

        :return: (new instance, info about the excluded data)
        """
        excl = ExcludedInvalidData()
        runs = []
        external_count = self.external_count
        for run in self.runs:
            run_data, excl_props = run.exclude_invalid()
            if run_data:
                if excl_props:
                    excl.excluded_properties_per_run_data[run.description()] = excl_props
                runs.append(run_data)
            else:
                excl.excluded_run_datas.append(run.description())
                if run.external:
                    external_count -= 1
        return self.clone(runs=runs, external_count=external_count), excl

    def add_property_descriptions(self, property_descriptions: t.Dict[str, str]):
        """
        Adds the given property descriptions.

        :param property_descriptions: mapping of some properties to their descriptions or longer versions
        """
        if not property_descriptions:
            return
        self.property_descriptions.update(property_descriptions)

    def long_properties(self, property_format: str = "[{}]") -> t.Tuple['RunDataStatsHelper', t.Dict[str, str]]:
        """
        Replace the short properties names with their descriptions if possible.

        :param property_format: format string that gets a property description and produces a longer property name
        :return: new instance, a dict that returns a long property name for a given short property name
        """
        runs = []
        formatted_properties = {}  # type: t.Dict[str, str]
        for p in self.property_descriptions:
            formatted_properties[p] = property_format.format(self.property_descriptions[p])
        for run in self.runs:
            if not run.discarded:
                runs.append(run.long_properties(formatted_properties))

        return self.clone(runs=runs), formatted_properties

    def update_env_info(self):
        """ Obtain the environment information for the current system and store it """
        self.env_info = format_env_info() class RunData(object):
    """
    A set of benchmarking data for a specific program block.
    """

    block_type_scheme = Dict({
                    "data": Dict(key_type=Str(), value_type=List(Int()|Float()), unknown_keys=True) // Default({}),
                    "attributes": ATTRIBUTES_TYPE,
                    "error": (Dict({"message": Str(), "return_code": Int(), "output": Str(),
                                    "error_output": Str()}) | NonExistent()),
                    "internal_error": (Dict({"message": Str()}) | NonExistent())
                }, unknown_keys=True) // Constraint(lambda d: [k in d for k in ["error", "internal_error"]].count(True) <= 1,
                                                    description="Either 'error' or 'internal_error' can be present")

    property_descriptions_scheme = Dict({
            "property_descriptions": NonExistent() |
                                     Dict(key_type=Str(), value_type=Str(), unknown_keys=True)})

    env_info_scheme = Dict({
            "env_info": NonExistent() | List(Tuple(Str(), List(Tuple(Str(), Str()))))})

    def __init__(self, data: t.Dict[str, t.List[Number]] = None, attributes: t.Dict[str, str] = None,
                 recorded_error: RecordedError = None,
                 external: bool = False):
        """
        Initializes a new run data object.
        
        :param data: optional dictionary mapping each property to a list of actual values
        :param attributes: dictionary of optional attributes that describe its program block
        :param recorded_error: either program error or internal error
        :param external: does the data come from a prior benchmarking?
        """
        typecheck(data, E(None) | Dict(unknown_keys=True))
        typecheck(attributes, Exact(None) | Dict(key_type=Str(), unknown_keys=True))
        self.external = external  # type: bool
        """ Does the data come from a prior benchmarking? """
        self.properties = [] # type: t.List[str]
        """ List of measured properties. They might not all be measured the same number of times. """
        self.data = {} # type: t.Dict[str, t.List[Number]]
        """ Raw benchmarking data, mapping properties to their corresponding values """
        if data is not None and len(data) > 0:
            self.add_data_block(data)
        self.attributes = attributes or {}  # type: t.Dict[str, str]
        """ Dictionary of optional attributes that describe its program block """
        self.tags = attributes["tags"] if "tags" in self.attributes else None
        self.max_runs = get_for_tags("run/max_runs_per_tag", "run/max_runs", self.tags, min)
        self.recorded_error = recorded_error
        self.discarded = False

    def clone(self, data: t.Dict[str, t.List[Number]] = None, attributes: t.Dict[str, str] = None,
              recorded_error: RecordedError = None, external: bool = None) -> 'RunData':
        """
        Clone this instance and replaces thereby some instance properties.

        :param data: optional dictionary mapping each property to a list of actual values
        :param attributes: dictionary of optional attributes that describe its program block
        :param external: does the data come from a prior benchmarking?
        :return: new instance
        """
        def alt(new, old):
            return new if new is not None else old
        return RunData(data=alt(data, self.data), attributes=alt(attributes, self.attributes),
                       recorded_error=alt(recorded_error, self.recorded_error),
                       external=alt(external, self.external))

    def add_data_block(self, data_block: t.Dict[str, t.List[Number]]):
        """
        Adds a block of data. 
        
        :param data_block: maps each of the run datas properties to list of actual values (from each benchmarking run).
        """
        typecheck(data_block, Dict(key_type=Str(), value_type= List(Int() | Float()), unknown_keys=True))
        self.properties = set(self.properties).union(set(data_block.keys()))
        for prop in data_block:
            if prop not in self.data:
                self.data[prop] = []
                self.properties.add(prop)
            self.data[prop].extend(data_block[prop])
        self.properties = sorted(list(self.properties))

    def __len__(self) -> int:
        """
        Returns the number of measured properties.
        """
        return len(self.data)

    def min_values(self) -> int:
        """
        Returns the minimum number of measured values for the associated program block
        over all properties.
        """
        return min(map(len, self.data.values())) if len(self) > 0 else 0

    def benchmarks(self) -> int:
        """
        Returns the maximum number of measured values for the associated program block
        over all properties.
        """
        return max(map(len, self.data.values())) if len(self) > 0 else 0

    def __getitem__(self, property: str) -> list:
        """
        Returns the benchmarking values associated with the passed property.
        """
        return self.data[property]

    def to_dict(self) -> t.Dict[str, t.Union[t.Dict[str, str], t.Dict[str, t.List[Number]]]]:
        """
        Returns a dictionary that represents this run data object.
        """
        d = {
            "attributes": self.attributes,
            "data": self.data
        }
        if self.has_error():
            if isinstance(self.recorded_error, RecordedInternalError):
                d["internal_error"] = {
                    "message": self.recorded_error.message
                }
            if isinstance(self.recorded_error, RecordedProgramError):
                d["error"] = {
                    "message": self.recorded_error.message,
                    "return_code": self.recorded_error.ret_code,
                    "output": self.recorded_error.out,
                    "error_output": self.recorded_error.err
                }
        return d

    def __str__(self) -> str:
        return repr(self.attributes)

    def description(self) -> str:
        """ Description of this instance based on the attributes """
        if "description" in self.attributes and self.attributes["description"] is not None:
            return self.attributes["description"]
        return ", ".join("{}={}".format(key, self.attributes[key]) for key in self.attributes)

    def exclude_properties(self, properties: t.List[str]) -> 'RunData':
        """
        Creates a new run data instance without the passed properties.

        :param properties: excluded properties
        :return: new run data instance
        """
        data = {}
        for prop in self.data:
            if prop not in properties:
                data[prop] = self.data[prop]
        return self.clone(data=data)

    def include_properties(self, properties: t.List[str]) -> 'RunData':
        """
        Creates a new run data instance with only the passed properties.

        :param properties: included properties, can be regular expressions
        :return: new run data instance
        """
        data = {}
        for prop in self.data:
            if prop in properties or any(re.fullmatch(p, prop) for p in properties):
                data[prop] = self.data[prop]
        return self.clone(data=data)

    def exclude_invalid(self) -> t.Tuple[t.Optional['RunData'], t.List[str]]:
        """
        Exclude properties that only have NaNs as measurements.

        :return: (new run data instance or None if all properties are excluded or the current if nothing changed,
                  excluded properties)
        """
        data = {}
        excluded = []
        nan = float("nan")
        for prop in self.data:
            if not all(x == nan for x in self.data[prop]):
                data[prop] = self.data[prop]
            else:
                excluded.append(prop)
        excluded = sorted(excluded)
        if not excluded:
            return self, []
        if len(data) > 0:
            return self.clone(data=data), excluded
        return None, excluded

    def long_properties(self, long_versions: t.Dict[str, str]) -> 'RunData':
        """
        Replace the short properties names with their long version from the passed dictionary.

        :param long_versions: long versions of some properties
        :return: new run data instance (or current instance if nothing changed)
        """
        if not long_versions:
            return self
        data = {}
        for prop in self.data:
            longer_prop = prop
            if prop in long_versions:
                longer_prop = long_versions[prop]
            data[longer_prop] = self.data[prop]
        return self.clone(data=data)

    def has_error(self) -> bool:
        return self.recorded_error is not None

    def get_single_properties(self) -> t.Dict[str, 'SingleProperty']:
        """
        Returns single property stat objects per property that support statistical analysis
        (e.g. mean, standard deviation, …)

        :return: stat object per property
        """
        from temci.report.stats import Single
        return Single(self).properties class Settings(metaclass=Singleton):
    """
    Manages the Settings.
    The settings keys and sub keys are combined by a slash, e.g. "report/in".

    The current settings are:

    .. code: yaml


    """

    config_file_name = "temci.yaml"  # type: str
    """ Default name of the configuration files """
    type_scheme = Dict({
        "settings": Str() // Description("Additional settings file")
                        // Default(config_file_name if os.path.exists(config_file_name) else "")
                        // CompletionHint(zsh=YAML_FILE_COMPLETION_HINT),
        "config": Str() // Description("Alias for settings")
                    // Default(config_file_name if os.path.exists(config_file_name) else "")
                    // CompletionHint(zsh=YAML_FILE_COMPLETION_HINT),
        "tmp_dir": Str() // Default("/tmp/temci") // Description("Used temporary directory"),
        "log_level": ExactEither("debug", "info", "warn", "error", "quiet") // Default("info")
                     // Description("Logging level"),
        "stats": Dict({
            "properties": ListOrTuple(Str()) // Default(["all"])
                        // CompletionHint(zsh="(" + " ".join(["__ov-time", "cache-misses", "cycles", "task-clock",
                                                              "instructions", "branch-misses", "cache-references",
                                                              "all"])
                                              + ")")
                        // Description("Properties to use for reporting and null hypothesis tests, "
                                       "can be regular expressions"),
            "uncertainty_range": Tuple(Float(lambda x: x >= 0), Float(lambda x: x >= 0)) // Default([0.05, 0.15])
                        // Description("Range of p values that allow no conclusion.")
        }, unknown_keys=True),
        "report": Dict({
            #  "reporter": Str() // Default("console") // Description(),
            "in": Either(Str(), ListOrTuple(Str())) // Default("run_output.yaml")
                  // Description("Files that contain the benchmarking results")
                  // CompletionHint(zsh=YAML_FILE_COMPLETION_HINT),
            "excluded_properties": ListOrTuple(Str()) // Default(["__ov-time"])
                    // Description("Properties that aren't shown in the report."),
            "exclude_invalid": BoolOrNone() // Default(True)
                    // Description("Exclude all data sets that contain only NaNs."),
            "long_properties": BoolOrNone() // Default(False)
                    // Description("Replace the property names in reports with longer more descriptive versions?"),
            "xkcd_like_plots": BoolOrNone() // Default(False)
                    // Description("Produce xkcd like plots (requires the humor sans font to be installed)"),
            "number": FNumber.settings_format,
            "included_blocks": ListOrTuple(Str()) // Default(["all"])
                               // Description("List of included run blocks (all: include all), "
                                              "identified by their description or tag attribute, "
                                              "can be regular expressions"),
        }, unknown_keys=True),
        "run": Dict({
            "discarded_runs": NaturalNumber() // Description("First n runs that are discarded") // Default(1),
            "min_runs": NaturalNumber() // Default(20) // Description("Minimum number of benchmarking runs"),
            "max_runs": NaturalNumber() // Default(100) // Description("Maximum number of benchmarking runs"),
            "max_runs_per_tag": Dict(unknown_keys=True, key_type=Str() // Description("Tag"), value_type=NaturalNumber() // Description("Max runs"))
                                 // Default({}) // Description("Maximum runs per tag (block attribute 'tag'), min('max_runs', 'per_tag') is used"),
            "min_runs_per_tag": Dict(unknown_keys=True, key_type=Str() // Description("Tag"),
                                     value_type=NaturalNumber() // Description("Min runs"))
                                // Default({}) // Description(
                                "Minimum runs per tag (block attribute 'tag'), max('min_runs', 'per_tag') is used"),
            "runs_per_tag": Dict(unknown_keys=True, key_type=Str() // Description("Tag"),
                                     value_type=NaturalNumber() // Description("Runs"))
                                // Default({}) // Description(
                                 "Runs per tag (block attribute 'tag'), max('runs', 'per_tag') is used"),
            "runs": Int(lambda x: x >= -1) // Default(-1) // Description("if != -1 sets max and min runs to its value"),
            "max_time": ValidTimeSpan() // Default("-1") // Description("Maximum time the whole benchmarking should take, "
                                                                        "-1 == no timeout, supports normal time span expressions"), # in seconds
            "max_block_time": ValidTimeSpan() // Default("-1") // Description("Maximum time one run block should take, "
                                                                              "-1 == no timeout, supports normal time span expressions"),
            "run_block_size": PositiveInt() // Default(1)
                              // Description("Number of benchmarking runs that are done together"),
            "in": Str() // Default("input.exec.yaml")
                  // Description("Input file with the program blocks to benchmark")
                  // CompletionHint(zsh=YAML_FILE_COMPLETION_HINT),
            "out": Str() // Default("run_output.yaml") // Description("Output file for the benchmarking results")
                    // CompletionHint(zsh=YAML_FILE_COMPLETION_HINT),
            "store_often": Bool() // Default(False)
                           // Description("Store the result file after each set of blocks is benchmarked"),
            "exec_plugins": Dict({

            }),
            "included_blocks" : ListOrTuple(Str()) // Default(["all"])
                              // Description("List of included run blocks (all: include all), "
                                             "or their tag attribute "
                                             "or their number in the file (starting with 0), "
                                             "can be regular expressions"),
            "cpuset": Dict({
                "active": Bool() // Description("Use cpuset functionality?") // Default(False),
                "base_core_number": ValidCPUCoreNumber()
                                    // Description("Number of cpu cores for the base (remaining part of the) system") // Default(1),
                "parallel": Int(lambda x: x >= -1) // Description("0: benchmark sequential, "
                                                      "> 0: benchmark parallel with n instances, "
                                                      "-1: determine n automatically") // Default(0),
                "sub_core_number": ValidCPUCoreNumber() // Description("Number of cpu cores per parallel running program.")
                                   // Default(1),
                "temci_in_base_set": Bool() // Default(True)
                                     // Description("place temci in the same cpu set as the rest of the system?")
            }),
            "disable_hyper_threading": Bool() // Default(False)
                                       // Description("Disable the hyper threaded cores. Good for cpu bound programs."),
            "show_report": Bool() // Default(True)
                // Description("Print console report if log_level=info"),
            "append": Bool() // Default(False)
                      // Description("Append to the output file instead of overwriting by adding new run data blocks"),
            "shuffle": Bool() // Default(True) // Description("Randomize the order in which the program blocks are "
                                                              "benchmarked."),
            "send_mail": Str() // Default("")
                         // Description("If not empty, recipient of a mail after the benchmarking finished."),
            "discard_all_data_for_block_on_error": Bool() // Default(False)
                         // Description("Discard all run data for the failing program on error"),
            "record_errors_in_file": Bool() // Default(True)
                         // Description("Record the caught errors in the run_output file"),
            "no_build": Bool() // Default(False)
                         // Description("Do not build if build configs are present, only works if the working directory "
                                        "of the blocks does not change"),
            "only_build": Bool() // Default(False) // Description("Only build"),
            "abort_after_build_error": Bool() // Default(True)
                                    // Description("Abort after the first failing build"),
            "watch": Bool() // Default(False) // Description("Show the report continuously"),
            "watch_every": PositiveInt() // Default(1)
                            // Description("Update the screen nth run (less updates are better for benchmarks)")
        }),
        "build": Dict({
            "in": Str() // Default("build.yaml") // Description("Input file with the program blocks to build")
                // CompletionHint(zsh=YAML_FILE_COMPLETION_HINT),
            "out": Str() // Default("run_config.yaml") // Description("Resulting run config file"),
            "threads": PositiveInt() // Default(1) // Description("Number of simultaneous builds for a specific "
                                                                  "program block, only makes sense when build_config/number > 1, "
                                                                  "and if the build commands create a different binary every "
                                                                  "time they are run"),
            "rand": Obsolete("Removed builder randomization", "0.8") // Description("Obsolete randomization configuration")
        }, unknown_keys=True),
        "package": Obsolete("Removed temci package", "0.8"),
        "env": Dict({"USER": Str(), "PATH": Str()}, unknown_keys=True)
               // Default({"USER": "", "PATH": ""})
               // Description("Environment variables for the benchmarked programs, includes the user used for "
                              "generated files"),
        "sudo": Bool() // Default(False) // Description("Acquire sudo privileges and run benchmark programs with "
                                                        "non-sudo user. Only supported on the command line.")
    }, unknown_keys=True)  # type: Dict
    """ Type scheme of the settings """

    def __init__(self):
        """
        Initializes a Settings singleton object and thereby loads the Settings files.
        It loads the settings files from the app folder (config.yaml) and
        the current working directory (temci.yaml) if they exist.

        :raises: SettingsError if some of the settings aren't in the format described via the type_scheme class property
        """
        self.prefs = copy.deepcopy(self.type_scheme.get_default())  # type: t.Dict[str, t.Any]
        """ The set sonfigurations """
        res = self._validate_settings_dict(self.prefs, "default settings")
        if not res:
            raise SettingsError(str(res))
        self._setup()

    def load_files(self):
        """ Loads the configuration files from the current and the config directory """
        self.load_from_config_dir()
        self.load_from_current_dir()
        self._setup()

    def _setup(self):
        """
        Simple setup method that checks if basic directories exist and creates them if necessary.
        """
        if not os.path.exists(self.prefs["tmp_dir"]):
            os.mkdir(self.prefs["tmp_dir"])
        log_level = self["log_level"]
        logging.Logger.disabled = log_level == "quiet"
        logger = logging.getLogger()
        mapping = {
            "debug": logging.DEBUG,
            "info": logging.INFO,
            "warn": logging.WARNING,
            "error": logging.ERROR,
            "quiet": logging.ERROR
        }
        logger.setLevel(mapping[log_level])
        self._update_doc()
        self.apply_override_actions()

    def _update_doc(self):
        """
        Update the class documentation
        """
        if sphinx_doc():
            self.__doc__ = self.__doc__.split(".. code: yaml")[0] + """.. code: yaml 

        """ + "\n        ".join(self.type_scheme.get_default_yaml().split("\n"))

    def reset(self):
        """
        Resets the current settings to the defaults.
        """
        self.prefs = copy.deepcopy(self.type_scheme.get_default())

    def _validate_settings_dict(self, data: t.Dict[str, t.Any], description: str = None):
        """
        Check whether the passed dictionary matches the settings type scheme.

        :param data: passed dictionary
        :param description: short description of the passed dictionary
        :return: True like object if valid, else string like object which is the error message
        """
        return verbose_isinstance(data, self.type_scheme, description or "Settings")

    def load_file(self, file: str):
        """
        Loads the configuration from the configuration YAML file.

        :param file: path to the file
        :raises: SettingsError if the settings file is incorrect or doesn't exist
        """
        self.prefs = self.type_scheme.get_default()
        tmp = copy.deepcopy(self.prefs)
        try:
            with open(file, 'r') as stream:
                map = yaml.safe_load(stream.read().replace("!!python/tuple", ""))

                def func(key, path, value):
                    if value is not None or self.get_type_scheme(path).check(value):
                        self._set_default(path, value)
                        self._set(path, value)

                recursive_exec_for_leafs(map, func)
        except (yaml.YAMLError, IOError) as err:
            self.prefs = tmp
            raise SettingsError(str(err))
        res = self._validate_settings_dict(self.prefs, "settings with ones from file '{}'".format(file))
        if not res:
            self.prefs = tmp
            raise SettingsError(str(res))
        self._setup()

    def load_from_dict(self, config_dict: t.Dict[str, t.Any]):
        """
        Load the configuration from the passed dictionary.

        :param config_dict: passed configuration dictionary
        """
        self.prefs = self.type_scheme.get_default()
        tmp = copy.deepcopy(self.prefs)

        def func(key, path, value):
            self._set_default(path, value)

        recursive_exec_for_leafs(config_dict, func)
        res = self._validate_settings_dict(self.prefs, "settings with ones config dict")
        if not res:
            self.prefs = tmp
            raise SettingsError(str(res))
        self._setup()

    def load_from_dir(self, dir: str):
        """
        Load the configuration from the configuration file inside the passed directory.

        :param dir: path of the directory
        """
        self.load_file(os.path.join(dir, "config.yaml"))

    def load_from_config_dir(self):
        """
        Load the config file from the application directory (e.g. in the users home folder) if it exists.
        """
        conf = os.path.join(click.get_app_dir("temci"), "config.yaml")
        if os.path.exists(conf) and os.path.isfile(conf):
            self.load_file(conf)

    def load_from_current_dir(self):
        """
        Load the configuration from the `configuration file in the current working directory if it exists.
        """
        if os.path.exists(self.config_file_name) and os.path.isfile(self.config_file_name):
            self.load_file(self.config_file_name)

    def get(self, key: t.Union[str, t.List[str]]) -> t.Any:
        """
        Get the setting with the given key.

        :param key: name of the setting
        :return: value of the setting
        :raises: SettingsError if the setting doesn't exist
        """
        if self.is_obsolete(key):
            raise SettingsError("Using obsolete setting {!r}: {}".format(key, self.obsoleteness_reason(key)))
        path = key.split("/") if isinstance(key, str) else key
        if not self.validate_key_path(path):
            raise SettingsError("No such setting {}".format(key))
        data = self.prefs
        for sub in path:
            data = data[sub]
        return data

    def __getitem__(self, key: str) -> t.Any:
        """
        Alias for self.get(self, key).
        """
        return self.get(key)

    def _set(self, path: t.List[str], value):
        """
        Set the setting at the passed path.

        :param path: passed key path
        :param value: new value
        """
        if self.is_obsolete(path):
            return
        tmp_pref = self.prefs
        tmp_type = self.type_scheme
        for key in path[0:-1]:
            if key not in tmp_pref:
                tmp_pref[key] = {}
                tmp_type[key] = Dict(unknown_keys=True, key_type=Str())
            tmp_pref = tmp_pref[key]
            tmp_type = tmp_type[key]
        tmp_pref[path[-1]] = value
        if path[-1] not in tmp_type.data:
            tmp_type[path[-1]] = Any() // Default(value)
        if (path == ["config"] or path == ["settings"]) and value != "":
            self.load_file(value)
        self._update_doc()

    def validate(self):
        """
        Validate this settings object

        :raises: SettingsError if the setting isn't valid
        """
        self._validate_settings_dict(self.prefs)

    def set(self, key: str, value, validate: bool = True, setup: bool = True):
        """
        Sets the setting key to the passed new value

        :param key: settings key
        :param value: new value
        :param validate: validate after the setting operation
        :param setup: call the setup function
        :raises: SettingsError if the setting isn't valid
        """
        tmp = copy.deepcopy(self.prefs)
        path = key.split("/")
        self._set(path, value)
        if validate:
            res = self._validate_settings_dict(self.prefs, "settings with new setting ({}={!r})".format(key, value))
            if not res:
                self.prefs = tmp
                raise SettingsError(str(res))
        if setup:
            self._setup()

    def __setitem__(self, key: str, value):
        """
        Alias for self.set(key, value).
        """
        self.set(key, value)

    def validate_key_path(self, path: t.List[str]) -> bool:
        """
        Validates a path into in to the settings trees,
        :param path: list of sub keys
        :return: Is this key path valid?
        """
        if self.is_obsolete(path):
            return True
        tmp = self.prefs
        for item in path:
            if item not in tmp:
                return False
            tmp = tmp[item]
        return True

    def has_key(self, key: str) -> bool:
        """ Does the passed key exist? """
        return self.validate_key_path(key.split("/"))

    def _set_default(self, path: t.List[str], value):
        """
        Set the default value of the setting with the passed path

        :param path: passed key path
        :param value: new default value
        """
        self.modify_type_scheme("/".join(path), lambda t: t // Default(value))
        self._update_doc()

    def modify_setting(self, key: str, type_scheme: Type):
        """
        Modifies the setting with the given key and adds it if it doesn't exist.

        :param key: key of the setting
        :param type_scheme: Type of the setting
        :param default_value: default value of the setting
        :raises: SettingsError if the settings domain (the key without the last element) doesn't exist
        :raises: TypeError if the default value doesn't adhere the type scheme
        """
        if self.is_obsolete(key):
            logging.info("Using obsolete setting {!r}: {}".format(key, self.obsoleteness_reason(key)))
            return
        path = key.split("/")
        domain = "/".join(path[:-1])
        if len(path) > 1 and not self.validate_key_path(path[:-1]) \
                and not isinstance(self.get(domain), dict):
            raise SettingsError("Setting domain {} doesn't exist".format(domain))
        tmp_typ = self.type_scheme
        tmp_prefs = self.prefs
        for subkey in path[:-1]:
            tmp_typ = tmp_typ[subkey]
            tmp_prefs = tmp_prefs[subkey]
        tmp_typ[path[-1]] = type_scheme
        if path[-1] in tmp_prefs:
            if type_scheme.typecheck_default:
                typecheck(tmp_prefs[path[-1]], type_scheme)
            tmp_typ[path[-1]] = type_scheme
        else:
            tmp_prefs[path[-1]] = type_scheme.get_default()

    def get_type_scheme(self, key: t.Union[str, t.List[str]]) -> Type:
        """
        Returns the type scheme of the given key.

        :param key: given key
        :return: type scheme
        :raises: SettingsError if the setting with the given key doesn't exist
        """
        key = key.split("/") if isinstance(key, str) else key
        if not self.validate_key_path(key):
            raise SettingsError("Setting {} doesn't exist".format("/".join(key)))
        tmp_typ = self.type_scheme
        for subkey in key:
            tmp_typ = tmp_typ[subkey]
        return tmp_typ

    def modify_type_scheme(self, key: str, modificator: t.Callable[[Type], Type]):
        """
        Modifies the type scheme of the given key via a modificator function.

        :param key: given key
        :param modificator: gets the type scheme and returns its modified version
        :raises: SettingsError if the setting with the given key doesn't exist
        """
        if self.is_obsolete(key):
            return
        if not self.validate_key_path(key.split("/")):
            raise SettingsError("Setting {} doesn't exist".format(key))
        tmp_typ = self.type_scheme
        subkeys = key.split("/")
        for subkey in subkeys[:-1]:
            tmp_typ = tmp_typ[subkey]
        tmp_typ[subkeys[-1]] = modificator(tmp_typ[subkeys[-1]])
        assert isinstance(tmp_typ[subkeys[-1]], Type)

    def default(self, value: t.Optional[t.Any], key: str):
        """
        Returns the passed value if isn't None else the settings value under the passed key.

        :param value: passed value
        :param key: passed settings key
        """
        if value is None:
            return self[key]
        typecheck(value, self.get_type_scheme(key))
        return value

    def store_into_file(self, file_name: str, comment_out_defaults: bool = False):
        """
        Stores the current settings into a yaml file with comments.

        :param file_name: name of the resulting file
        :param comment_out_defaults: comment out the default values
        """
        with open(file_name, "w") as f:
            print(self.type_scheme.get_default_yaml(defaults=self.prefs,
                                                    comment_out_defaults=comment_out_defaults), file=f)

    def has_log_level(self, level: str) -> bool:
        """
        Is the current log level the passed level?

        :param level: passed level (in ["error", "warn", "info", "debug"])
        """
        levels = ["error", "warn", "info", "debug"]
        return levels.index(level) <= levels.index(self["log_level"])

    def is_obsolete(self, key: t.Union[str, t.List[str]]) -> bool:
        """
        Is the setting with the passed key obsolete?

        :param key: key or key path
        :return: obsolete setting?
        """
        return self.obsoleteness_reason(key) is not None

    def obsoleteness_reason(self, key: t.Union[str, t.List[str]]) -> t.Optional[Obsolete]:
        """
        Returns the obsolete type object for obsolete settings

        :param key: key or path
        :return: object that contains information on the obsoleteness or None
        """
        path = key.split("/") if isinstance(key, str) else key
        tmp_type = self.type_scheme
        for subkey in path[:-1]:
            if tmp_type.is_obsolete(subkey):
                return tmp_type.obsoleteness_reason(subkey)
            if subkey not in tmp_type:
                return None
            if isinstance(tmp_type[subkey], Obsolete):
                return tmp_type[subkey]
            tmp_type = tmp_type[subkey]
        if path[-1] in tmp_type and isinstance(tmp_type[path[-1]], Obsolete):
            return tmp_type[subkey]
        return None

    def apply_override_actions(self):
        """ Applies actions like overriding max_runs with runs """
        if self["run/runs"] > -1:
            self.set("run/max_runs", self["run/runs"], setup=False)
            self.set("run/min_runs", self["run/runs"], setup=False) class ReportProcessor:
    def __init__(self, stats_helper: RunDataStatsHelper = None):
    def report(self):
import copy
import random
import traceback
import temci.run.run_driver_plugin
import time, logging, humanfriendly, sys, math, os, click
import typing as t
import yaml
                import click
                            import click._termui_impl
from temci.build.build_processor import BuildProcessor
from temci.build.builder import Builder, BuildError
from temci.utils.curses import Screen
from temci.utils.sudo_utils import chown
from temci.utils.util import join_strs, in_standalone_mode, parse_timespan
from temci.utils.mail import send_mail
from temci.utils.typecheck import *
from temci.run.run_worker_pool import RunWorkerPool, ParallelRunWorkerPool, AbstractRunWorkerPool
from temci.run.run_driver import RunProgramBlock, BenchmarkingResultBlock, RunDriverRegistry, ExecRunDriver, \
    is_perf_available, filter_runs, log_program_error
from temci.report.rundata import RunDataStatsHelper, RunData
from temci.utils.settings import Settings
from temci.report.report_processor import ReporterRegistry
            try:
                if os.path.exists(Settings()["run/out"]):
                    with open(Settings()["run/out"], "r") as f:
                        run_data = yaml.safe_load(f)
                self.stats_helper = RunDataStatsHelper.init_from_dicts(run_data, external=True)
                for run in runs:
                    self.stats_helper.runs.append(RunData(attributes=run["attributes"]))
            except:
                self.teardown()
                raise
        else:
            self.stats_helper = RunDataStatsHelper.init_from_dicts(copy.deepcopy(runs),
                                                                   included_blocks=Settings()["run/included_blocks"])
        #if Settings()["run/remote"]:
        #    self.pool = RemoteRunWorkerPool(Settings()["run/remote"], Settings()["run/remote_port"])
            if os.path.exists(Settings()["run/out"]):
                os.remove(Settings()["run/out"])
        self.start_time = time.time()  # type: float
        """ Unix time stamp of the start of the benchmarking """
        self.end_time = -1  # type: float
        """ Unix time stamp of the point in time that the benchmarking can at most reach """
        try:
            max_time = parse_timespan(Settings()["run/max_time"])
            if max_time > -1:
                self.end_time = self.start_time + max_time
        except:
            self.teardown()
            raise
        self.pool = None  # type: AbstractRunWorkerPool
        """ Used run worker pool that abstracts the benchmarking """
        if Settings()["run/cpuset/parallel"] == 0:
            self.pool = RunWorkerPool(end_time=self.end_time)
        else:
            self.pool = ParallelRunWorkerPool(end_time=self.end_time)
        self.run_block_size = Settings()["run/run_block_size"]  # type: int
        """ Number of benchmarking runs that are done together """
        self.discarded_runs = Settings()["run/discarded_runs"]  # type: int
        """ First n runs that are discarded """
        self.max_runs = Settings()["run/max_runs"]  # type: int
        """ Maximum number of benchmarking runs """
        self.min_runs = Settings()["run/min_runs"]  # type: int
        """ Minimum number of benchmarking runs """
        if self.min_runs > self.max_runs:
            logging.warning("min_runs ({}) is bigger than max_runs ({}), therefore they are swapped."
                            .format(self.min_runs, self.max_runs))
            tmp = self.min_runs
            self.min_runs = self.max_runs
            self.max_runs = tmp

        self.shuffle = Settings()["run/shuffle"]  # type: bool
        """ Randomize the order in which the program blocks are benchmarked. """
        self.fixed_runs = Settings()["run/runs"] != -1  # type: bool
        """ Do a fixed number of benchmarking runs? """
        if self.fixed_runs:
            self.min_runs = self.max_runs = self.min_runs = Settings()["run/runs"]
        self.store_often = Settings()["run/store_often"]  # type: bool
        """ Store the result file after each set of blocks is benchmarked """
        self.block_run_count = 0  # type: int
        """ Number of benchmarked blocks """
        self.erroneous_run_blocks = []  # type: t.List[t.Tuple[int, BenchmarkingResultBlock]]
        """ List of all failing run blocks (id and results till failing) """
        self.discard_all_data_for_block_on_error = Settings()["run/discard_all_data_for_block_on_error"]
        self.no_build = Settings()["run/no_build"]
        self.only_build = Settings()["run/only_build"]
        self.abort_after_build_error = Settings()["run/abort_after_build_error"]

    def _finished(self) -> bool:
        if not self.pool.has_time_left():
            return True
        if self.fixed_runs:
            return self.block_run_count >= self.max_runs
        return (len(self.stats_helper.get_program_ids_to_bench()) == 0 \
               or not self._can_run_next_block()) and self.maximum_of_min_runs() <= self.block_run_count

    def maximum_of_min_runs(self) -> int:
        return max(list(block.min_runs for block in self.run_blocks) + [self.min_runs])

    def maximum_of_max_runs(self) -> int:
        return max(list(block.max_runs for block in self.run_blocks) + [self.max_runs])

    def _can_run_next_block(self) -> bool:
        if not in_standalone_mode:
            estimated_time = self.stats_helper.estimate_time_for_next_round(self.run_block_size,
                                                                            all=self.block_run_count < self.min_runs)
            to_bench_count = len(self.stats_helper.get_program_ids_to_bench())
            if -1 < self.end_time < round(time.time() + estimated_time):
                logging.warning("Ran too long ({}) and is therefore now aborted. "
                                "{} program blocks should've been benchmarked again."
                                .format(humanfriendly.format_timespan(time.time() + estimated_time - self.start_time),
                                        to_bench_count))
                return False
        if self.block_run_count >= self.maximum_of_max_runs() and self.block_run_count >= self.maximum_of_min_runs():
            logging.warning("Benchmarked program blocks too often and aborted therefore now.")
            return False
        return True

    def build(self):
        """
        Build before benchmarking, essentially calls `temci build` where necessary and modifies the run configs
        """
        if self.no_build:
            return
        to_build = [(i, conf) for i, conf in enumerate(self.runs) if "build_config" in conf]
        if len(to_build) == 0:
            return
        logging.info("Start building {} block(s)".format(len(to_build)))
        for i, block in to_build:
            if "working_dir" not in block["build_config"]:
                block["build_config"]["working_dir"] = self.run_blocks[i].data["cwd"]
            try:
                block = BuildProcessor.preprocess_build_blocks([block])[0]
                logging.info("Build {}".format(self.run_blocks[i].description()))
                block_builder = Builder(self.run_blocks[i].description(),
                                        block["build_config"]["working_dir"],
                                        block["build_config"]["cmd"], block["build_config"]["revision"],
                                        block["build_config"]["number"],
                                        block["build_config"]["base_dir"], block["build_config"]["branch"])
                working_dirs = block_builder.build()
                block["cwds"] = working_dirs
                self.run_blocks[i].data["cwds"] = working_dirs

            except BuildError as err: