import argparse
from types import NoneType

import imp
import yaml
from yaml.scanner import ScannerError

TYPE = 'type'
LIST = 'list'
DESCRIPTION = 'description'
REQUIRED = 'required'
DEFAULT = 'default'
ALLOWED = 'allowed'
VALUES_DSC = 'values_description'
ONE_OF = 'one of'
SCHEMA = 'schema'
EXAMPLES = 'examples'
ANYOF = 'anyof'
NO_DSC = '(no description)'
VALIDATOR = 'validator'

class TextBlock(object):
    def __init__(self, text, tab_replacement='  ', ending=''):
        """

        :type text: str
        """
        self.text = str(text).replace('\t', tab_replacement)
        self.lines = self.text.splitlines()
        self.width = max([len(line) for line in self.lines] + [0])
        self.padded_width = self.width + 2
        self.height = len(self.lines)

    def get_line(self, item, raise_index_error=False, default=''):
        try:
            return self.lines[item]
        except IndexError:
            if raise_index_error:
                raise
            else:
                return default

    def get_line_justified(self, item, fillchar=' ', raise_index_error=False, default=''):
        return self.get_line(item, raise_index_error, default).ljust(self.width, fillchar)

    def __str__(self):
        return self.text

def to_text_block(method):
    def decorated(content):
        if not isinstance(content, TextBlock):
            return method(TextBlock(content))
        else:
            return method(TextBlock)

    return decorated

class RSTRenderer(object):
    def with_escape(method):
        def escaped(content):
            return method(RSTRenderer.escape(content))

        return escaped

    @staticmethod
    def any_of_table(blocks):
        """

        :type blocks: list of TextBlock
        """
        HEADER = 'any of'
        cnt = len(blocks)
        # no need table for single content
        if cnt < 2:
            return blocks[0] if blocks else ''
        # width = widths of contents + separators
        width = max((len(HEADER), sum([c.padded_width for c in blocks]))) + (cnt + 1)
        height = max([c.height for c in blocks])
        # rows separators
        top_bar = '+{}+'.format('-' * (width - 2))
        header_bar = '+{}+'.format('+'.join(['=' * c.padded_width for c in blocks]))
        bottom_bar = '+{}+'.format('+'.join(['-' * c.padded_width for c in blocks]))

        header = '|{}|'.format(HEADER.center(width - 2))
        body = '\n'.join(
            ['| {} |'.format(' | '.join([c.get_line_justified(i) for c in blocks])) for i in range(height)])
        return '\n'.join([top_bar,
                          header,
                          header_bar,
                          body,
                          bottom_bar])

    @staticmethod
    def preserve_indents(block):
        """

        :type block: TextBlock
        """
        return '\n'.join(['| {}'.format(line) for line in block.lines])

    @staticmethod
    def bold(content):
        """
        :type content: str
        :return: str
        """
        return '\n'.join(['**{}**'.format(line) for line in content.splitlines()])

    @staticmethod
    def title(content, new_line_replacement=' ', tab_replacement='  '):
        """
        Underlines content with '='. New lines and tabs will be replaced
        :param str content:
        :param str new_line_replacement:
        :param str tab_replacement:
        :return: unicode
        """
        prepared_content = content.strip().replace('\n', new_line_replacement).replace('\t', tab_replacement)
        return u'{}\n{}'.format(prepared_content, '=' * len(prepared_content))

    @staticmethod
    def subtitle(content, new_line_replacement=' ', tab_replacement='  '):
        prepared_content = content.strip().replace('\n', new_line_replacement).replace('\t', tab_replacement)
        return u'{}\n{}'.format(prepared_content, '-' * len(prepared_content))

    @staticmethod
    @with_escape
    @to_text_block
    def italic(block):
        """

        :type block: TextBlock
        """
        return '\n'.join(['*{}*'.format(line) for line in block.lines])

    @staticmethod
    @to_text_block
    def mono(block):
        """

        :type block: TextBlock
        """
        return '\n'.join(['``{}``'.format(line) for line in block.lines])

    @classmethod
    def bullet_list(cls, blocks):
        """

        :type blocks: list of TextBlock
        :rtype: TextBlock
        """
        return TextBlock('\n'.join([cls._list_item(block) for block in blocks]))

    @staticmethod
    def _list_item(block):
        """

        :type block: TextBlock
        """
        return '- ' + '\n  '.join(block.lines)

    @staticmethod
    def def_list(items, sort=True, newlines=True):
        def format_value(value):
            if isinstance(value, (int, bool, NoneType)):
                return format_value(str(value))
            if isinstance(value, str):
                return '\n '.join(value.splitlines())
            elif isinstance(value, TextBlock):
                return '\n '.join(value.lines)
            elif isinstance(value, dict):
                return '\n '.join(RSTRenderer.def_list(value, sort, newlines).splitlines())
            elif isinstance(value, list):
                return '\n '.join(RSTRenderer.bullet_list([TextBlock(item) for item in value]).lines)
            else:
                raise ValueError('Unsupported value type: {}\n{}'.format(type(value), value))

        sort = sorted if sort else lambda x: x
        template = '{}\n {}' if newlines else ':{}: {}'
        return '\n' + '\n'.join([template.format(k.replace('\n', ' '),
                                                 format_value(v).strip())
                                 for k, v in sort(items.items())]) if items else ''

    @staticmethod
    def field_list(items, sort=True, newlines=True):
        """

        :param bool newlines: add newlines between names and values
        :param bool sort: sort items alphabetically by key
        :type items: dict
        :rtype: TextBlock
        """

        def format_value(value):
            if isinstance(value, (int, bool, NoneType)):
                return format_value(str(value))
            if isinstance(value, (str, unicode)):
                return '\n '.join(value.splitlines())
            elif isinstance(value, TextBlock):
                return '\n '.join(value.lines)
            elif isinstance(value, dict):
                return '\n '.join(RSTRenderer.field_list(value, sort, newlines).splitlines())
            elif isinstance(value, list):
                return '\n '.join(RSTRenderer.bullet_list([TextBlock(item) for item in value]).lines)
            else:
                raise ValueError('Unsupported value type: {}\n{}'.format(type(value), value))

        sort = sorted if sort else lambda x: x
        template = ':{}:\n {}' if newlines else ':{}: {}'
        return '\n' + '\n'.join([template.format(k.replace('\n', ' '),
                                                 format_value(v).strip())
                                 for k, v in sort(items.items())]) if items else ''

    @staticmethod
    def dict_list_structure(items, sort_dict=True):
        if isinstance(items, str):
            return TextBlock(items)
        elif isinstance(items, int):
            return TextBlock(str(items))
        elif isinstance(items, list):
            return RSTRenderer.bullet_list([RSTRenderer.dict_list_structure(item) for item in items])
        elif isinstance(items, dict):
            return RSTRenderer.field_list({k: RSTRenderer.dict_list_structure(v) for k, v in items.items()}, sort_dict)

    @staticmethod
    def escape(content):
        """
        :type content: str
        """
        return content.replace('-', r'\-')

    del with_escape

def render_body(renderer, option_kwargs, exclude_keys, special_keys=None):
    """

    :type option_kwargs: dict
    :type exclude_keys: list
    :type special_keys: dict
    """
    common_formatters = {
        EXAMPLES: lambda examples: renderer.def_list({renderer.mono(example): annotation for example, annotation in examples.items()})
    }

    def default_fmt(x):
        return x

    special_keys = special_keys or {}
    special_part = '\n'.join([special_handler(renderer, option_kwargs[special_key])
                              for special_key, special_handler in special_keys.items()
                              if special_key in option_kwargs])
    common_part = renderer.field_list({k: common_formatters.get(k, default_fmt)(v) for k, v in option_kwargs.items()
                                       if k not in exclude_keys + special_keys.keys()})

    return '\n'.join([_ for _ in [common_part, special_part] if _])

def render_values_description(renderer, option_kwargs):
    values_description_dict = {
        value: option_kwargs[VALUES_DSC].get(value, '') for value in option_kwargs[ALLOWED]
    } \
        if ALLOWED in option_kwargs \
        else \
        option_kwargs[VALUES_DSC]
    values_description = renderer.field_list(
        {renderer.mono(value): dsc for value, dsc in values_description_dict.items()},
        newlines=False
    )
    return renderer.field_list({ONE_OF: values_description})

def allowed(renderer, values):
    return renderer.field_list({ONE_OF: '[{}]'.format(', '.join([renderer.mono(value) for value in values]))},
                               newlines=False)

class OptionFormatter(object):
    def __init__(self, option_schema):
        """

        :type option_schema: dict
        """
        self.option_name, self.option_kwargs = option_schema.items()[0]
        # print(option_name, option_kwargs)
        self.formatter = self.__guess_formatter()

    def format_dsc(self, renderer):
        dsc = self.option_kwargs.get(DESCRIPTION, NO_DSC).strip('. ')
        if DEFAULT in self.option_kwargs:
            default_value = self.option_kwargs.get(DEFAULT)
            if default_value == '':
                default_value = '""'
            return ' '.join([renderer.italic('- {}. Default:'.format(dsc)),
                             renderer.mono(default_value)])
        elif REQUIRED in self.option_kwargs:
            return renderer.italic('- {}.'.format(dsc)) +\
                ' ' +\
                renderer.bold('Required.')
        else:
            return renderer.italic('- {}.'.format(dsc))

    def scalar_formatter(self, renderer, header=True):
        hdr = renderer.subtitle(renderer.mono(self.option_name) + ' ' + '({})'.format(self.option_kwargs.get(TYPE))) \
            if header else ''
        dsc = self.format_dsc(renderer)
        body = render_body(renderer, self.option_kwargs, [VALIDATOR, TYPE, DESCRIPTION, DEFAULT, REQUIRED], {'allowed': allowed})
        return '\n'.join([_ for _ in [hdr, dsc, body] if _])

    def scalar_with_values_description(self, renderer, header=True):
        hdr = renderer.subtitle(renderer.mono(self.option_name) + ' ' + '({})'.format(self.option_kwargs.get(TYPE))) \
            if header else ''
        dsc = self.format_dsc(renderer)
        body = render_body(renderer, self.option_kwargs, [VALIDATOR, TYPE, DESCRIPTION, DEFAULT, REQUIRED, ALLOWED, VALUES_DSC])
        values_description_block = render_values_description(renderer, self.option_kwargs)
        return '\n'.join([_ for _ in [hdr, dsc, body, values_description_block] if _])

    def dict_formatter(self, renderer, header=True):
        hdr = renderer.subtitle(renderer.mono(self.option_name) + ' ' + '({})'.format(self.option_kwargs.get(TYPE))) \
            if header else ''
        dsc = self.format_dsc(renderer)

        dict_schema = self.option_kwargs[SCHEMA]

        schema_block = renderer.field_list({
            '{} ({})'.format(renderer.mono(key), dict_schema[key].get(TYPE, 'anyof')): get_formatter({key: value})(renderer, header=False)
            for key, value in dict_schema.items()})
        body = render_body(renderer, self.option_kwargs, [VALIDATOR, TYPE, DESCRIPTION, DEFAULT, REQUIRED, SCHEMA])
        return '\n'.join([_ for _ in [hdr, dsc, schema_block, body] if _])

    def anyof_formatter(self, renderer, header=True):
        types = [case[TYPE] for case in self.option_kwargs[ANYOF] if TYPE in case]
        hdr = renderer.subtitle(renderer.mono(self.option_name) + ' ' + '({})'.format(' or '.join(types))) \
            if header else ''
        dsc = self.format_dsc(renderer)
        values_description_block = render_values_description(renderer, self.option_kwargs) \
            if VALUES_DSC in self.option_kwargs else ''
        body = render_body(renderer, self.option_kwargs, [VALIDATOR, TYPE, DESCRIPTION, DEFAULT, REQUIRED, ANYOF, VALUES_DSC])

        return '\n'.join([_ for _ in [hdr, dsc, values_description_block, body] if _])

    def list_formatter(self, renderer, header=True):
        schema = self.option_kwargs[SCHEMA]
        hdr = renderer.subtitle(renderer.mono(self.option_name) + ' '
                                + '({} of {})'.format(self.option_kwargs.get(TYPE, LIST), schema.get(TYPE, '')))
        dsc = self.format_dsc(renderer)
        body = render_body(renderer, self.option_kwargs, [VALIDATOR, TYPE, DEFAULT, REQUIRED, DESCRIPTION, SCHEMA])
        if set(schema.keys()) - {TYPE}:
            schema_block = renderer.field_list({
                '[list_element] ({})'.format(schema.get(TYPE, '')):
                    get_formatter({'list_element': schema})(renderer, header=False)
            })
            return '\n'.join([_ for _ in [hdr, dsc, schema_block, body] if _])
        else:
            return '\n'.join([_ for _ in [hdr, dsc, body] if _])

    def __guess_formatter(self):
        if ANYOF in self.option_kwargs:
            return self.anyof_formatter
        elif SCHEMA in self.option_kwargs:
            return self.list_formatter if self.option_kwargs.get(TYPE) == LIST else self.dict_formatter
        elif VALUES_DSC in self.option_kwargs:
            return self.scalar_with_values_description
        else:
            return self.scalar_formatter

def get_formatter(option_schema):
    """

    :type option_schema: dict
    """
    return OptionFormatter(option_schema).formatter

def format_option(option_schema, renderer):
    return get_formatter(option_schema)(renderer)

def format_schema(schema, renderer, title=None):
    """

    :param dict schema: Cerberus config schema
    :type renderer: RSTRenderer
    """
    body = '\n\n'.join(
        sorted([format_option({option_name: option_schema}, renderer) for option_name, option_schema in schema.items()]))

    if title:
        title = renderer.title(title)
        return title + '\n\n' + body
    else:
        return body

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('schema', help='Path to schema file')
    parser.add_argument('-o', '--output_filename', default='output.rst', help='Name for the output rst document')
    parser.add_argument('--title', default=None, help='Document title')
    parser.add_argument('-a', '--append', action='store_true', help='Don\'t overwrite output file')
    args = parser.parse_args()

    schema_path = args.schema
    output_filename = args.output_filename
    title = args.title
    append = args.append

    try:
        with open(schema_path) as f:
            schema = yaml.load(f)
    except ScannerError:
        schema_module = imp.load_source('schema', schema_path)
        schema = schema_module.OPTIONS
    document = format_schema(schema, RSTRenderer(), title)

    if append:
        with open(output_filename, 'a') as f:
            f.write('\n\n')
            f.write(document)
    else:
        with open(output_filename, 'w') as f:
            f.write(document)

if __name__ == '__main__':
    main()


""" jmeter load generator support """
import datetime
import logging
import os
import signal
import subprocess
import time
import socket
import re
import shlex

from pkg_resources import resource_string

from .reader import JMeterReader
from ..Console import Plugin as ConsolePlugin
from ..Console import screen as ConsoleScreen
from ...common.interfaces import AggregateResultListener, AbstractInfoWidget, GeneratorPlugin

logger = logging.getLogger(__name__)

class Plugin(GeneratorPlugin):
    """ JMeter tank plugin """
    SECTION = 'jmeter'
    SHUTDOWN_TEST = 'Shutdown'
    STOP_TEST_NOW = 'Stop Test'
    DISCOVER_PORT_PATTERN = r'Waiting for possible .* message on port (?P<port>\d+)'

    def __init__(self, core, cfg, name):
        super(Plugin, self).__init__(core, cfg, name)
        self.args = None
        self.original_jmx = None
        self.jtl_file = None
        self.ext_log = None
        self.ext_levels = ['none', 'errors', 'all']
        self.ext_log_file = None
        self.jmx = None
        self.user_args = None
        self.jmeter_path = None
        self.jmeter_ver = None
        self.jmeter_log = None
        self.start_time = time.time()
        self.jmeter_buffer_size = None
        self.jmeter_udp_port = None
        self.shutdown_timeout = None

    @staticmethod
    def get_key():
        return __file__

    def get_available_options(self):
        return [
            "jmx", "args", "jmeter_path", "buffer_size", "buffered_seconds",
            "exclude_markers", "shutdown_timeout"
        ]

    def configure(self):
        self.original_jmx = self.get_option("jmx")
        self.core.add_artifact_file(self.original_jmx, True)
        self.jtl_file = self.core.mkstemp('.jtl', 'jmeter_')
        self.core.add_artifact_file(self.jtl_file)
        self.user_args = self.get_option("args", '')
        self.jmeter_path = self.get_option('jmeter_path')
        self.jmeter_log = self.core.mkstemp('.log', 'jmeter_')
        self.jmeter_ver = self.get_option('jmeter_ver')
        self.ext_log = self.get_option('extended_log', self.get_option('ext_log'))

        if self.ext_log != 'none':
            self.ext_log_file = self.core.mkstemp('.jtl', 'jmeter_ext_')
            self.core.add_artifact_file(self.ext_log_file)

        self.core.add_artifact_file(self.jmeter_log, True)
        self.exclude_markers = set(self.get_option('exclude_markers', []))
        self.jmx = self.__add_jmeter_components(
            self.original_jmx, self.jtl_file, self.get_option('variables'))
        self.core.add_artifact_file(self.jmx)

        jmeter_stderr_file = self.core.mkstemp(".log", "jmeter_stdout_stderr_")
        self.core.add_artifact_file(jmeter_stderr_file)
        self.process_stderr = open(jmeter_stderr_file, 'w')
        self.shutdown_timeout = self.get_option('shutdown_timeout', 3)

        self.affinity = self.get_option('affinity', '')

    def get_reader(self):
        if self.reader is None:
            self.reader = JMeterReader(self.jtl_file)
        return self.reader

    def get_stats_reader(self):
        if self.stats_reader is None:
            self.stats_reader = self.reader.stats_reader
        return self.stats_reader

    def prepare_test(self):
        self.args = [
            self.jmeter_path, "-n", "-t", self.jmx, '-j', self.jmeter_log,
            '-Jjmeter.save.saveservice.default_delimiter=\\t',
            '-Jjmeter.save.saveservice.connect_time=true'
        ]
        self.args += shlex.split(self.user_args)

        if self.affinity:
            self.core.__setup_affinity(self.affinity, args=self.args)

        try:
            console = self.core.get_plugin_of_type(ConsolePlugin)
        except Exception as ex:
            logger.debug("Console not found: %s", ex)
            console = None

        if console:
            widget = JMeterInfoWidget(self)
            console.add_info_widget(widget)
            self.core.job.aggregator.add_result_listener(widget)

    def start_test(self):
        logger.info(
            "Starting %s with arguments: %s", self.jmeter_path, self.args)
        try:
            self.process = subprocess.Popen(
                self.args,
                executable=self.jmeter_path,
                preexec_fn=os.setsid,
                close_fds=True,
                stdout=self.process_stderr,
                stderr=self.process_stderr)
        except OSError:
            logger.debug(
                "Unable to start JMeter process. Args: %s, Executable: %s",
                self.args,
                self.jmeter_path,
                exc_info=True)
            raise RuntimeError(
                "Unable to access to JMeter executable file or it does not exist: %s"
                % self.jmeter_path)
        self.start_time = time.time()
        self.jmeter_udp_port = self.__discover_jmeter_udp_port()

    def is_test_finished(self):
        retcode = self.process.poll()
        aggregator = self.core.job.aggregator
        if not aggregator.reader.jmeter_finished and retcode is not None:
            logger.info(
                "JMeter process finished with exit code: %s, waiting for aggregator",
                retcode)
            self.retries = 0
            aggregator.reader.jmeter_finished = True
            return -1
        elif aggregator.reader.jmeter_finished is True:
            if aggregator.reader.agg_finished:
                return retcode
            else:
                logger.info("Waiting for aggregator to finish")
                return -1
        else:
            return -1

    def end_test(self, retcode):
        if self.process:
            gracefully_shutdown = self.__graceful_shutdown()
            if not gracefully_shutdown:
                self.__kill_jmeter()
        if self.process_stderr:
            self.process_stderr.close()
        self.core.add_artifact_file(self.jmeter_log)
        return retcode

    def __discover_jmeter_udp_port(self):
        """Searching for line in jmeter.log such as
        Waiting for possible shutdown message on port 4445
        """
        r = re.compile(self.DISCOVER_PORT_PATTERN)
        with open(self.process_stderr.name, 'r') as f:
            cnt = 0
            while self.process.pid and cnt < 10:
                line = f.readline()
                m = r.match(line)
                if m is None:
                    cnt += 1
                    time.sleep(1)
                else:
                    port = int(m.group('port'))
                    return port
            else:
                logger.warning('JMeter UDP port wasn\'t discovered')
                return None

    def __kill_jmeter(self):
        logger.info(
            "Terminating jmeter process group with PID %s",
            self.process.pid)
        try:
            os.killpg(self.process.pid, signal.SIGTERM)
        except OSError as exc:
            logger.debug("Seems JMeter exited itself: %s", exc)
            # Utils.log_stdout_stderr(logger, self.process.stdout, self.process.stderr, "jmeter")

    def __add_jmeter_components(self, jmx, jtl, variables):
        """ Genius idea by Alexey Lavrenyuk """
        logger.debug("Original JMX: %s", os.path.realpath(jmx))
        with open(jmx, 'r') as src_jmx:
            source_lines = src_jmx.readlines()

        try:
            # In new Jmeter version (3.2 as example) WorkBench's plugin checkbox enabled by default
            # It totally crashes Yandex tank injection and raises XML Parse Exception
            closing = source_lines.pop(-1)
            if "WorkBenchGui" in source_lines[-5]:
                logger.info("WorkBench checkbox enabled...bypassing")
                last_string_count = 6
            else:
                last_string_count = 2
            while last_string_count > 0:
                closing = source_lines.pop(-1) + closing
                last_string_count -= 1
            logger.debug("Closing statement: %s", closing)
        except Exception as exc:
            raise RuntimeError("Failed to find the end of JMX XML: %s" % exc)

        udv_tpl = resource_string(__name__, 'config/jmeter_var_template.xml')
        udv_set = []
        for var_name, var_value in variables.iteritems():
            udv_set.append(udv_tpl % (var_name, var_name, var_value))
        udv = "\n".join(udv_set)

        if self.jmeter_ver >= 2.13:
            save_connect = '<connectTime>true</connectTime>'
        else:
            save_connect = ''

        if self.ext_log in ['errors', 'all']:
            level_map = {'errors': 'true', 'all': 'false'}
            tpl_resource = 'jmeter_writer_ext.xml'
            tpl_args = {
                'jtl': self.jtl_file,
                'udv': udv,
                'ext_log': self.ext_log_file,
                'ext_level': level_map[self.ext_log],
                'save_connect': save_connect
            }
        else:
            tpl_resource = 'jmeter_writer.xml'
            tpl_args = {
                'jtl': self.jtl_file,
                'udv': udv,
                'save_connect': save_connect
            }

        tpl = resource_string(__name__, 'config/' + tpl_resource)

        try:
            new_jmx = self.core.mkstemp(
                '.jmx', 'modified_', os.path.dirname(os.path.realpath(jmx)))
        except OSError as exc:
            logger.debug("Can't create modified jmx near original: %s", exc)
            new_jmx = self.core.mkstemp('.jmx', 'modified_')
        logger.debug("Modified JMX: %s", new_jmx)
        with open(new_jmx, "wb") as fh:
            fh.write(''.join(source_lines))
            fh.write(tpl % tpl_args)
            fh.write(closing)
        return new_jmx

    def __graceful_shutdown(self):
        if self.jmeter_udp_port is None:
            return False
        shutdown_test_started = time.time()
        while time.time() - shutdown_test_started < self.shutdown_timeout:
            self.__send_udp_message(self.SHUTDOWN_TEST)
            if self.process.poll() is not None:
                return True
            else:
                time.sleep(1)
        self.log.info('Graceful shutdown failed after %s' % str(time.time() - shutdown_test_started))

        stop_test_started = time.time()
        while time.time() - stop_test_started < self.shutdown_timeout:
            self.__send_udp_message(self.STOP_TEST_NOW)
            if self.process.poll() is not None:
                return True
            else:
                time.sleep(1)
        self.log.info('Graceful stop failed after %s' % time.time() - stop_test_started)
        return False

    def __send_udp_message(self, message):
        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        sock.sendto(message, ('localhost', self.jmeter_udp_port))

class JMeterInfoWidget(AbstractInfoWidget, AggregateResultListener):
    """ Right panel widget with JMeter test info """

    def __init__(self, jmeter):
        AbstractInfoWidget.__init__(self)
        self.krutilka = ConsoleScreen.krutilka()
        self.jmeter = jmeter
        self.active_threads = 0
        self.RPS = 0

    def get_index(self):
        return 0

    def on_aggregated_data(self, data, stats):
        self.active_threads = stats['metrics']['instances']
        self.RPS = data['overall']['interval_real']['len']

    def render(self, screen):
        jmeter = " JMeter Test %s" % self.krutilka.next()
        space = screen.right_panel_width - len(jmeter) - 1
        left_spaces = space / 2
        right_spaces = space / 2

        dur_seconds = int(time.time()) - int(self.jmeter.start_time)
        duration = str(datetime.timedelta(seconds=dur_seconds))

        template = screen.markup.BG_MAGENTA + '~' * left_spaces + jmeter + ' '
        template += '~' * right_spaces + screen.markup.RESET + "\n"
        template += "     Test Plan: %s\n"
        template += "      Duration: %s\n"
        template += "Active Threads: %s\n"
        template += "   Responses/s: %s"
        data = (
            os.path.basename(self.jmeter.original_jmx), duration,
            self.active_threads, self.RPS)

        return template % data


import errno
import collections
import logging
import os.path
import subprocess
import time

from ...common.interfaces import AbstractPlugin, GeneratorPlugin, AggregateResultListener, AbstractInfoWidget, \
    StatsReader
from ...common.util import FileScanner
from ..Console import Plugin as ConsolePlugin
from ..Phantom import PhantomReader

_INFO = collections.namedtuple(
    "Info",
    "address, port, instances, ammo_count, loop_count, duration, steps, stat_log, rps_schedule, ammo_file"
)
_LOGGER = logging.getLogger(__name__)

_PROCESS_KILL_TIMEOUT = 10  # Kill running process after specified number of seconds
_OUTPUT_WAIT_TIMEOUT = 10  # Output files should be found after specified number of seconds

class Plugin(GeneratorPlugin):
    """Simple executor of shooting process with phantom compatible output"""

    SECTION = 'shootexec'

    def __init__(self, core, cfg, name):
        AbstractPlugin.__init__(self, core, cfg, name)
        self.stats_reader = None
        self.reader = None
        self.__process = None
        self.__stderr_file = None
        self.__processed_ammo_count = 0
        self.__start_time = 0

    @staticmethod
    def get_key():
        return __file__

    def get_available_options(self):
        return ["cmd", "output_path", "stats_path"]

    def configure(self):
        self.__cmd = self.get_option("cmd")

        self.__output_path = self.get_option("output_path")
        self.core.add_artifact_file(self.__output_path)

        self.__stats_path = self.get_option("stats_path")
        if self.__stats_path:
            self.core.add_artifact_file(self.__stats_path)

    def get_reader(self):
        if self.reader is None:
            # Touch output_path because PhantomReader wants to open it
            with open(self.__output_path, "w"):
                pass
            self.reader = PhantomReader(self.__output_path)
        return self.reader

    def get_stats_reader(self):
        if self.stats_reader is None:
            self.stats_reader = _FileStatsReader(self.__stats_path) if self.__stats_path else _DummyStatsReader()
        return self.stats_reader

    def prepare_test(self):
        stderr_path = self.core.mkstemp(".log", "shootexec_stdout_stderr_")
        self.__stderr_file = open(stderr_path, 'w')

        _LOGGER.debug("Linking sample reader to aggregator. Reading samples from %s", self.__output_path)

        self.__start_time = time.time()
        self.core.job.aggregator.add_result_listener(self)

        try:
            console = self.core.get_plugin_of_type(ConsolePlugin)
        except Exception as ex:
            _LOGGER.debug("Console not found: %s", ex)
            console = None

        if console:
            widget = _InfoWidget(self)
            console.add_info_widget(widget)
            self.core.job.aggregator.add_result_listener(widget)

    def start_test(self):
        _LOGGER.info("Starting shooting process: '%s'", self.__cmd)
        self.__process = subprocess.Popen(
            self.__cmd,
            shell=True,
            stderr=self.__stderr_file,
            stdout=self.__stderr_file,
            close_fds=True
        )

        # Ensure that all expected output files are ready to use
        _LOGGER.info("Waiting until output files are ready")
        waitfor = time.time() + _OUTPUT_WAIT_TIMEOUT
        while time.time() < waitfor:
            output_path_is_ready = os.path.isfile(self.__output_path)
            stats_path_is_ready = (not self.__stats_path or os.path.isfile(self.__stats_path))
            if output_path_is_ready and stats_path_is_ready:
                break
            time.sleep(0.1)
        else:
            raise Exception("Failed to wait until output resources are ready: output={}, stats={}".format(
                output_path_is_ready,
                stats_path_is_ready
            ))
        _LOGGER.info("Shooting proces is ready to use")

    def is_test_finished(self):
        retcode = self.__process.poll()
        if retcode is not None:
            _LOGGER.info("Shooting process done its work with exit code: %s", retcode)
            return abs(retcode)
        else:
            return -1

    def end_test(self, retcode):
        if self.__process and self.__process.poll() is None:
            _LOGGER.warn("Terminating shooting process with PID %s", self.__process.pid)
            self.__terminate()
        else:
            _LOGGER.debug("Seems shooting process finished OK")
        return retcode

    def post_process(self, retcode):
        return retcode

    def on_aggregated_data(self, data, stats):
        self.__processed_ammo_count += data["overall"]["interval_real"]["len"]
        _LOGGER.debug("Processed ammo count: %s", self.__processed_ammo_count)

    def get_info(self):
        """ returns info object """
        return _INFO(
            "",
            "",
            "0",
            "0",
            "0",
            time.time() - self.__start_time,
            None,
            "",
            "",
            ""
        )

    def __terminate(self):
        """Gracefull termination of running process"""

        if self.__stderr_file:
            self.__stderr_file.close()

        if not self.__process:
            return

        waitfor = time.time() + _PROCESS_KILL_TIMEOUT
        while time.time() < waitfor:
            try:
                self.__process.terminate()
            except EnvironmentError as e:
                if e.errno != errno.ESRCH:
                    _LOGGER.warning("Failed to terminate process '{}': {}".format(self.__cmd, e))
                return
            time.sleep(0.1)

        try:
            self.__process.kill()
        except EnvironmentError as e:
            if e.errno != errno.ESRCH:
                _LOGGER.warning("Failed to kill process '{}': {}".format(self.__cmd, e))
            return

class _InfoWidget(AbstractInfoWidget, AggregateResultListener):
    """
    Widget with information about current run state
    """

    def get_index(self):
        return 2

    def __init__(self, sender):
        AbstractInfoWidget.__init__(self)
        self.owner = sender

    def render(self, screen):
        return ""

    def on_aggregated_data(self, data, stats):
        pass

class _FileStatsReader(FileScanner, StatsReader):
    """
    Read shooting stats line by line

    Line format is 'timestamp\trps\tinstances'
    """

    def __init__(self, *args, **kwargs):
        super(_FileStatsReader, self).__init__(*args, **kwargs)
        self.__last_ts = 0

    def _read_data(self, lines):
        """
        Parse lines and return stats
        """

        results = []
        for line in lines:
            timestamp, rps, instances = line.split("\t")
            curr_ts = int(float(timestamp))  # We allow floats here, but tank expects only seconds
            if self.__last_ts < curr_ts:
                self.__last_ts = curr_ts
                results.append(self.stats_item(self.__last_ts, float(rps), float(instances)))
        return results

class _DummyStatsReader(StatsReader):
    """
    Dummy stats reader for shooters without stats file
    """

    def __init__(self):
        self.__closed = False
        self.__last_ts = 0

    def __iter__(self):
        while not self.__closed:
            cur_ts = int(time.time())
            if cur_ts > self.__last_ts:
                yield [self.stats_item(cur_ts, 0, 0)]
                self.__last_ts = cur_ts
            else:
                yield []

    def close(self):
        self.__closed = True


""" Autostop facility """
# pylint: disable=C0301
import logging
import os.path

from . import criterions as cr
from . import cumulative_criterions as cum_cr
from ..Console import Plugin as ConsolePlugin
from ...common.interfaces import AbstractPlugin, AggregateResultListener, AbstractInfoWidget

logger = logging.getLogger(__name__)

class Plugin(AbstractPlugin, AggregateResultListener):
    """ Plugin that accepts criterion classes and triggers autostop """
    SECTION = 'autostop'

    def __init__(self, core, cfg, name):
        AbstractPlugin.__init__(self, core, cfg, name)
        AggregateResultListener.__init__(self)

        self.cause_criterion = None
        self._criterions = {}
        self.custom_criterions = []
        self.counting = []
        self._stop_report_path = ''

    @staticmethod
    def get_key():
        return __file__

    def get_counting(self):
        """ get criterions that are activated """
        return self.counting

    def add_counting(self, obj):
        """ add criterion that activated """
        self.counting += [obj]

    def add_criterion_class(self, criterion_class):
        """ add new criterion class """
        self.custom_criterions += [criterion_class]

    def get_available_options(self):
        return ["autostop", "report_file"]

    def configure(self):
        aggregator = self.core.job.aggregator
        aggregator.add_result_listener(self)

        self._stop_report_path = os.path.join(
            self.core.artifacts_dir,
            self.get_option("report_file", 'autostop_report.txt'))

        self.add_criterion_class(cr.AvgTimeCriterion)
        self.add_criterion_class(cr.NetCodesCriterion)
        self.add_criterion_class(cr.HTTPCodesCriterion)
        self.add_criterion_class(cr.QuantileCriterion)
        self.add_criterion_class(cr.SteadyCumulativeQuantilesCriterion)
        self.add_criterion_class(cr.TimeLimitCriterion)
        self.add_criterion_class(cum_cr.TotalFracTimeCriterion)
        self.add_criterion_class(cum_cr.TotalHTTPCodesCriterion)
        self.add_criterion_class(cum_cr.TotalNetCodesCriterion)
        self.add_criterion_class(cum_cr.TotalNegativeHTTPCodesCriterion)
        self.add_criterion_class(cum_cr.TotalNegativeNetCodesCriterion)
        self.add_criterion_class(cum_cr.TotalHTTPTrendCriterion)
        self.add_criterion_class(cum_cr.QuantileOfSaturationCriterion)

    def prepare_test(self):
        criterions = self.get_option("autostop")
        for criterion_str in criterions:
            if not criterion_str:
                continue
            self.log.debug("Criterion string: %s", criterion_str)
            self._criterions[criterion_str] = self.__create_criterion(
                criterion_str)

        self.log.debug("Criterion objects: %s", self._criterions)

        try:
            console = self.core.get_plugin_of_type(ConsolePlugin)
        except Exception as ex:  # pylint: disable=W0703
            self.log.debug("Console not found: %s", ex)
            console = None

        if console:
            console.add_info_widget(AutostopWidget(self))

    def is_test_finished(self):
        if self.cause_criterion:
            self.log.warning(
                "Autostop criterion requested test stop: %s",
                self.cause_criterion.explain())
            return self.cause_criterion.get_rc()
        else:
            return -1

    def __create_criterion(self, criterion_str):
        """ instantiate criterion from config string """
        parsed = criterion_str.split("(")
        type_str = parsed[0].strip().lower()
        parsed[1] = parsed[1].split(")")[0].strip()

        for criterion_class in self.custom_criterions:
            if criterion_class.get_type_string() == type_str:
                return criterion_class(self, parsed[1])
        raise ValueError(
            "Unsupported autostop criterion type: %s" % criterion_str)

    def on_aggregated_data(self, data, stat):
        self.counting = []
        if not self.cause_criterion:
            for criterion_text, criterion in self._criterions.iteritems():
                if criterion.notify(data, stat):
                    self.log.warning(
                        "Autostop criterion requested test stop: %s", criterion_text)
                    self.cause_criterion = criterion
                    open(self._stop_report_path, 'w').write(criterion_text)
                    self.core.add_artifact_file(self._stop_report_path)

class AutostopWidget(AbstractInfoWidget):
    """ widget that displays counting criterions """

    def __init__(self, sender):
        AbstractInfoWidget.__init__(self)
        self.owner = sender

    def get_index(self):
        return 25

    def render(self, screen):
        res = []
        candidates = self.owner.get_counting()
        for candidate in candidates:
            text, perc = candidate.widget_explain()
            if perc >= 0.95:
                res += [screen.markup.RED_DARK + text + screen.markup.RESET]
            elif perc >= 0.8:
                res += [screen.markup.RED + text + screen.markup.RESET]
            elif perc >= 0.5:
                res += [screen.markup.YELLOW + text + screen.markup.RESET]
            else:
                res += [text]

        if res:
            return "Autostop:\n  " + ("\n  ".join(res))
        else:
            return ''


from xml.etree import ElementTree as etree
import os.path
import getpass
import logging
import tempfile
from future.utils import iteritems
from ..Telegraf.decoder import decoder
import sys
if sys.version_info[0] < 3:
    import ConfigParser
else:
    import configparser as ConfigParser

logger = logging.getLogger(__name__)

class ConfigManager(object):
    """
    Config reader and parser helper.
    XML support
    """

    @staticmethod
    def parse_xml(config):
        if os.path.exists(config):
            return etree.parse(config)
        else:
            return etree.fromstring(config)

    def getconfig(self, filename, target_hint):
        """Prepare config data."""
        try:
            tree = self.parse_xml(filename)
        except IOError as exc:
            logger.error("Error loading config: %s", exc)
            raise RuntimeError("Can't read monitoring config %s" % filename)
        hosts = tree.findall('Host')
        config = []
        for host in hosts:
            host_config = self.get_host_config(host, target_hint)
            config.append(host_config)
        return config

    def get_host_config(self, host, target_hint):
        defaults = {
            "CPU": {
                "name": '[inputs.cpu]',
                "percpu": 'false',
                "fielddrop": '["time_*", "usage_guest_nice"]'
            },
            "Memory": {
                "name": '[inputs.mem]',
                "fielddrop":
                '["active", "inactive", "total", "used_per*", "avail*"]',
            },
            "Disk": {
                "name": '[inputs.diskio]',
                "devices": '[{devices}]'.format(
                    devices=",".join(
                        ['"vda%s","sda%s"' % (num, num) for num in range(6)])),
            },
            "Net": {
                "name": '[inputs.net]',
                "interfaces": '[{interfaces}]'.format(
                    interfaces=",".join(
                        ['"eth%s"' % (num) for num in range(6)])),
                "fielddrop":
                '["icmp*", "ip*", "udplite*", "tcp*", "udp*", "drop*", "err*"]',
            },
            "Nstat": {
                "name": '[inputs.nstat]',
                "fieldpass": '["TcpRetransSegs"]',
            },
            "Netstat": {
                "name": '[inputs.netstat]',
            },
            "NetResponse": {
                "name": '[inputs.net_response]',
                "protocol": '"tcp"',
                "address": '":80"',
                "timeout": '"1s"'
            },
            "System": {
                "name": '[inputs.system]',
                "fielddrop": '["n_users", "n_cpus", "uptime*"]',
            },
            "Kernel": {
                "name": '[inputs.kernel]',
                "fielddrop": '["boot_time"]',
            },
            "KernelVmstat": {
                "name": '[inputs.kernel_vmstat]',
                "fieldpass": '["pgfault", "pgmajfault"]',
            }
        }
        defaults_enabled = ['CPU', 'Memory', 'Disk', 'Net', 'System', 'Kernel']
        defaults_boolean = [
            'percpu', 'round_interval', 'fielddrop', 'fieldpass', 'interfaces',
            'devices'
        ]
        hostname = host.get('address').lower()
        if hostname == '[target]':
            if not target_hint:
                raise ValueError(
                    "Can't use `[target]` keyword with no target parameter specified"
                )
            logger.debug("Using target hint: %s", target_hint)
            hostname = target_hint.lower()
        custom = []
        startups = []
        shutdowns = []
        sources = []
        telegrafraw = []
        # agent defaults
        host_config = {}
        for metric in host:
            if str(metric.tag) in defaults.keys():
                for key in defaults[metric.tag]:
                    if key != 'name' and key not in defaults_boolean:
                        value = metric.get(key, None)
                        if value:
                            defaults[metric.tag][key] = "'{value}'".format(
                                value=value)
                    elif key in defaults_boolean:
                        value = metric.get(key, None)
                        if value:
                            defaults[metric.tag][key] = "{value}".format(
                                value=value)
                host_config[metric.tag] = defaults[metric.tag]
            # custom metrics
            if (str(metric.tag)).lower() == 'custom':
                isdiff = metric.get('diff', 0)
                cmd = {
                    'cmd': metric.text,
                    'label': metric.get('label'),
                    'diff': isdiff
                }
                custom.append(cmd)
            elif (str(metric.tag)).lower() == 'startup':
                startups.append(metric.text)
            elif (str(metric.tag)).lower() == 'shutdown':
                shutdowns.append(metric.text)
            elif (str(metric.tag)).lower() == 'source':
                sources.append(metric.text)
            elif (str(metric.tag)).lower() == 'telegrafraw':
                telegrafraw.append(metric.text)
        if len(host_config) == 0:
            logger.info('Empty host config, using defaults')
            for section in defaults_enabled:
                host_config[section] = defaults[section]
        result = {
            'host_config': host_config,
            'port': int(host.get('port', 22)),
            'python': host.get('python', '/usr/bin/env python2'),
            'interval': host.get('interval', 1),
            'username': host.get('username', getpass.getuser()),
            'telegraf': host.get('telegraf', '/usr/bin/telegraf'),
            'comment': host.get('comment', ''),
            'custom': custom,
            'host': hostname,
            'startup': startups,
            'shutdown': shutdowns,
            'source': sources,
            'telegrafraw': telegrafraw
        }
        logger.info("Telegraf Result config %s", result)
        return result

class AgentConfig(object):
    """ Agent config generator helper """

    def __init__(self, config, old_style_configs):
        self.host = config['host']
        self.custom = config['custom']
        self.startups = config['startup']
        self.shutdowns = config['shutdown']
        self.sources = config['source']
        self.interval = config['interval']
        self.comment = config['comment']
        self.telegrafraw = config['telegrafraw']
        self.host_config = config['host_config']
        self.old_style_configs = old_style_configs

    def create_startup_config(self):
        """ Startup and shutdown commands config
        Used by agent.py on the target

        """
        cfg_path = "agent_startup_{}.cfg".format(self.host)
        if os.path.isfile(cfg_path):
            logger.info(
                'Found agent startup config file in working directory with the same name as created for host %s.\n'
                'Creating new one via tempfile. This will affect predictable filenames for agent artefacts',
                self.host)
            handle, cfg_path = tempfile.mkstemp('.cfg', 'agent_')
            os.close(handle)
        try:
            config = ConfigParser.RawConfigParser()
            # FIXME incinerate such a string formatting inside a method call
            # T_T
            config.add_section('startup')
            [
                config.set('startup', "cmd%s" % idx, cmd)
                for idx, cmd in enumerate(self.startups)
            ]
            config.add_section('shutdown')
            [
                config.set('shutdown', "cmd%s" % idx, cmd)
                for idx, cmd in enumerate(self.shutdowns)
            ]
            config.add_section('source')
            [
                config.set('source', "file%s" % idx, path)
                for idx, path in enumerate(self.sources)
            ]
            with open(cfg_path, 'w') as fds:
                config.write(fds)

        except Exception as exc:
            logger.error(
                'Error trying to create monitoring startups config. Malformed? %s',
                exc,
                exc_info=True)
        return cfg_path

    def create_custom_exec_script(self):
        """ bash script w/ custom commands inside
        inspired by half a night trying to avoid escaping bash special characters

        """
        cfg_path = "agent_customs_{}.cfg".format(self.host)
        if os.path.isfile(cfg_path):
            logger.info(
                'Found agent custom execs config file in working directory with the same name as created for host %s.\n'
                'Creating new one via tempfile. This will affect predictable filenames for agent artefacts',
                self.host)
            handle, cfg_path = tempfile.mkstemp('.sh', 'agent_customs_')
            os.close(handle)

        cmds = ""
        for idx, cmd in enumerate(self.custom):
            cmds += "-{idx}) {cmd};;\n".format(idx=idx, cmd=cmd['cmd'])
        customs_script = """
        #!/bin/sh
        while :
        do
            case "$1" in
            {cmds}
            *) break;;
            esac
            shift
        done
        """.format(cmds=cmds)

        with open(cfg_path, 'w') as fds:
            fds.write(customs_script)
        return cfg_path

    def create_collector_config(self, workdir):
        """ Telegraf collector config,
        toml format

        """
        cfg_path = "agent_collector_{}.cfg".format(self.host)
        if os.path.isfile(cfg_path):
            logger.info(
                'Found agent config file in working directory with the same name as created for host %s.\n'
                'Creating new one via tempfile. This will affect predictable filenames for agent artefacts',
                self.host)
            handle, cfg_path = tempfile.mkstemp('.cfg', 'agent_collector_')
            os.close(handle)

        self.monitoring_data_output = "{remote_folder}/monitoring.rawdata".format(
            remote_folder=workdir)

        defaults_old_enabled = ['CPU', 'Memory', 'Disk', 'Net', 'System']

        try:
            config = ConfigParser.RawConfigParser()

            config.add_section("global_tags")
            config.add_section("agent")
            config.set(
                "agent",
                "interval",
                "'{interval}s'".format(interval=self.interval))
            config.set("agent", "round_interval", "true")
            config.set("agent", "flush_interval", "'1s'")
            config.set("agent", "collection_jitter", "'0s'")
            config.set("agent", "flush_jitter", "'1s'")

            for section in self.host_config.keys():
                # telegraf-style config
                if not self.old_style_configs:
                    config.add_section(
                        "{section_name}".format(
                            section_name=self.host_config[section]['name']))
                    for key, value in iteritems(self.host_config[section]):
                        if key != 'name':
                            config.set(
                                "{section_name}".format(
                                    section_name=self.host_config[section][
                                        'name']),
                                "{key}".format(key=key),
                                "{value}".format(value=value))
                # monitoring-style config
                else:
                    if section in defaults_old_enabled:
                        config.add_section(
                            "{section_name}".format(
                                section_name=self.host_config[section]['name']))
                        for key, value in iteritems(self.host_config[section]):
                            if key in [
                                    'fielddrop', 'fieldpass', 'percpu',
                                    'devices', 'interfaces'
                            ]:
                                config.set(
                                    "{section_name}".format(
                                        section_name=self.host_config[section][
                                            'name']),
                                    "{key}".format(key=key),
                                    "{value}".format(value=value))

            # outputs
            config.add_section("[outputs.file]")
            config.set(
                "[outputs.file]",
                "files",
                "['{config}']".format(config=self.monitoring_data_output))
            config.set("[outputs.file]", "data_format", "'json'")

            with open(cfg_path, 'w') as fds:
                config.write(fds)

            # dirty hack, this allow to avoid bash escape quoting, we're pushing shell script w/ arguments
            # index of argument is index of custom metric in our config
            inputs = ""
            for idx, cmd in enumerate(self.custom):
                inputs += "[[inputs.exec]]\n"
                inputs += "commands = ['/bin/sh {workdir}/agent_customs.sh -{idx}']\n".format(
                    workdir=workdir, idx=idx)
                inputs += "data_format = 'value'\n"
                inputs += "data_type = 'float'\n"
                inputs += "name_prefix = '{}_'\n\n".format(cmd.get('label'))
                if cmd['diff']:
                    decoder.diff_metrics['custom'].append(
                        decoder.find_common_names(cmd.get('label')))

            with open(cfg_path, 'a') as fds:
                fds.write(inputs)

            # telegraf raw configuration into xml
            telegraf_raw = ""
            for element in self.telegrafraw:
                telegraf_raw += element

            with open(cfg_path, 'a') as fds:
                fds.write(telegraf_raw)

        except Exception as exc:
            logger.error(
                'Error trying to create monitoring config. Malformed? %s',
                exc,
                exc_info=True)
        return cfg_path


''' Module to check system resources at load generator'''

import logging
import time

import psutil
from ...common.util import expand_to_seconds
from ...common.interfaces import AbstractPlugin

from netort.process import execute

class Plugin(AbstractPlugin):
    '''Plugin to check system resources'''
    SECTION = "rcheck"

    @staticmethod
    def get_key():
        return __file__

    def __init__(self, core, cfg, name):
        '''         Constructor        '''
        AbstractPlugin.__init__(self, core, cfg, name)
        self.interval = "10s"
        self.disk_limit = 2048  # 2 GB
        self.mem_limit = 512  # 0.5 GB
        self.last_check = 0

    def get_available_options(self):
        return ["interval", "disk_limit", "mem_limit"]

    def configure(self):
        self.interval = expand_to_seconds(
            self.get_option("interval", self.interval))
        self.disk_limit = int(self.get_option("disk_limit", self.disk_limit))
        self.mem_limit = int(self.get_option("mem_limit", self.mem_limit))

    def prepare_test(self):
        self.log.info("Checking tank resources...")
        self.__check_disk()
        self.__check_mem()

    def is_test_finished(self):
        self.log.debug("Checking tank resources...")
        if time.time() - self.last_check < self.interval:
            return -1
        self.__check_disk()
        self.__check_mem()
        self.last_check = time.time()
        return -1

    def __check_disk(self):
        ''' raise exception on disk space exceeded '''
        cmd = "sh -c \"df --no-sync -m -P -l -x fuse -x tmpfs -x devtmpfs -x davfs -x nfs "
        cmd += self.core.artifacts_base_dir
        cmd += " | tail -n 1 | awk '{print \$4}' \""
        res = execute(cmd, True, 0.1, True)
        logging.debug("Result: %s", res)
        if not len(res[1]):
            self.log.debug("No disk usage info: %s", res[2])
            return
        disk_free = res[1]
        self.log.debug(
            "Disk free space: %s/%s", disk_free.strip(), self.disk_limit)
        if int(disk_free.strip()) < self.disk_limit:
            raise RuntimeError(
                "Not enough local resources: disk space less than %sMB in %s: %sMB"
                % (
                    self.disk_limit, self.core.artifacts_base_dir,
                    int(disk_free.strip())))

    def __check_mem(self):
        ''' raise exception on RAM exceeded '''
        mem_free = psutil.virtual_memory().available / 2**20
        self.log.debug("Memory free: %s/%s", mem_free, self.mem_limit)
        if mem_free < self.mem_limit:
            raise RuntimeError(
                "Not enough resources: free memory less "
                "than %sMB: %sMB" % (self.mem_limit, mem_free))


# -*- coding: utf-8 -*-
''' Classes to build full console screen '''
import fcntl
import logging
import os
import struct
import termios
import time
import bisect
from collections import defaultdict
import pandas as pd

from ...common import util

def get_terminal_size():
    '''
    Gets width and height of terminal viewport
    '''
    default_size = (30, 120)
    env = os.environ

    def ioctl_gwinsz(file_d):
        '''
        Helper to get console size
        '''
        try:
            sizes = struct.unpack(
                'hh', fcntl.ioctl(file_d, termios.TIOCGWINSZ, '1234'))
        except Exception:
            sizes = default_size
        return sizes

    sizes = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2)
    if not sizes:
        try:
            file_d = os.open(os.ctermid(), os.O_RDONLY)
            sizes = ioctl_gwinsz(file_d)
            os.close(file_d.fileno())
        except Exception:
            pass
    if not sizes:
        try:
            sizes = (env['LINES'], env['COLUMNS'])
        except Exception:
            sizes = default_size
    return int(sizes[1]), int(sizes[0])

def safe_div(summ, count):
    if count == 0:
        return 0
    else:
        return float(summ) / count

def str_len(n):
    return len(str(n))

def avg_from_dict(src):
    result = {}
    count = src['count']
    for k, v in src.iteritems():
        if k != 'count':
            result[k] = safe_div(v, count)
    return result

def krutilka():
    pos = 0
    chars = "|/-\\"
    while True:
        yield chars[pos]
        pos += 1
        if pos >= len(chars):
            pos = 0

def try_color(old, new, markup):
    order = [
        markup.WHITE, markup.GREEN, markup.CYAN,
        markup.YELLOW, markup.MAGENTA, markup.RED]
    if not old:
        return new
    else:
        if order.index(old) > order.index(new):
            return old
        else:
            return new

def combine_codes(tag_data, markup):
    net_err, http_err = 0, 0
    color = ''
    net_codes = tag_data['net_code']['count']
    for code, count in sorted(net_codes.iteritems()):
        if count > 0:
            if int(code) == 0:
                continue
            elif int(code) == 314:
                color = try_color(color, markup.MAGENTA, markup)
                net_err += count
            else:
                color = try_color(color, markup.RED, markup)
                net_err += count
    http_codes = tag_data['proto_code']['count']
    for code, count in sorted(http_codes.iteritems()):
        if count > 0:
            if 100 <= int(code) <= 299:
                color = try_color(color, markup.GREEN, markup)
            elif 300 <= int(code) <= 399:
                color = try_color(color, markup.CYAN, markup)
            elif 400 <= int(code) <= 499:
                color = try_color(color, markup.YELLOW, markup)
                http_err += count
            elif 500 <= int(code) <= 599:
                color = try_color(color, markup.RED, markup)
                http_err += count
            else:
                color = try_color(color, markup.MAGENTA, markup)
                http_err += count
    return (net_err, http_err, color)

class TableFormatter(object):
    def __init__(self, template, delimiters, reshape_delay=5):
        self.log = logging.getLogger(__name__)
        self.template = template
        self.delimiters = delimiters
        self.default_final = '{{{:}:>{len}}}'
        self.last_reshaped = time.time()
        self.old_shape = {}
        self.reshape_delay = reshape_delay

    def __delimiter_gen(self):
        for d in self.delimiters:
            yield d
        while True:
            yield self.delimiters[-1]

    def __prepare(self, data):
        prepared = []
        shape = {}
        for line in data:
            new = {}
            for f in line:
                if f in self.template:
                    new[f] = self.template[f]['tpl'].format(line[f])
                    if f not in shape:
                        shape[f] = len(new[f])
                    else:
                        shape[f] = max(shape[f], len(new[f]))
            prepared.append(new)
        return (prepared, shape)

    def __update_shape(self, shape):
        def change_shape():
            self.last_reshaped = time.time()
            self.old_shape = shape
        if set(shape.keys()) != set(self.old_shape.keys()):
            change_shape()
            return shape
        else:
            for f in shape:
                if shape[f] > self.old_shape[f]:
                    change_shape()
                    return shape
                elif shape[f] < self.old_shape[f]:
                    if time.time() > (self.last_reshaped + self.reshape_delay):
                        change_shape()
                        return shape
        return self.old_shape

    def render_table(self, data, fields):
        prepared, shape = self.__prepare(data)
        headers = {}
        for f in shape:
            if 'header' in self.template[f]:
                headers[f] = self.template[f]['header']
                shape[f] = max(shape[f], len(headers[f]))
            else:
                headers[f] = ''
        shape = self.__update_shape(shape)
        has_headers = any(headers.values())
        delimiter_gen = self.__delimiter_gen()
        row_tpl = ''
        for num, field in enumerate(fields):
            if 'final' in self.template[field]:
                final = self.template[field]['final']
            else:
                final = self.default_final
            row_tpl += final.format(field, len=shape[field])
            if num < len(fields) - 1:
                row_tpl += delimiter_gen.next()
        result = []
        if has_headers:
            result.append(
                (row_tpl.format(**headers),))
        for line in prepared:
            result.append(
                (row_tpl.format(**line),))
        return result

class Sparkline(object):
    def __init__(self, window):
        self.log = logging.getLogger(__name__)
        self.data = {}
        self.window = window
        self.active_seconds = []
        self.ticks = '_▁▂▃▄▅▆▇'.decode('utf-8')

    def recalc_active(self, ts):
        if not self.active_seconds:
            self.active_seconds.append(ts)
            self.data[ts] = {}
        if ts not in self.active_seconds:
            if ts > max(self.active_seconds):
                for i in range(max(self.active_seconds) + 1, ts + 1):
                    self.active_seconds.append(i)
                    self.active_seconds.sort()
                    self.data[i] = {}
        while len(self.active_seconds) > self.window:
            self.active_seconds.pop(0)
        for sec in self.data.keys():
            if sec not in self.active_seconds:
                self.data.pop(sec)

    def get_key_data(self, key):
        result = []
        if not self.active_seconds:
            return None
        for sec in self.active_seconds:
            if key in self.data[sec]:
                result.append(self.data[sec][key])
            else:
                result.append(('', 0))
        return result

    def add(self, ts, key, value, color=''):
        if ts not in self.data:
            self.recalc_active(ts)
        if ts < min(self.active_seconds):
            self.log.warning('Sparkline got outdated second %s, oldest in list %s', ts, min(self.active_seconds))
            return
        value = max(value, 0)
        self.data[ts][key] = (color, value)

    def get_sparkline(self, key, baseline='zero', spark_len='auto', align='right'):
        if spark_len == 'auto':
            spark_len = self.window
        elif spark_len <= 0:
            return ''
        key_data = self.get_key_data(key)
        if not key_data:
            return ''
        active_data = key_data[-spark_len:]
        result = []
        if active_data:
            values = [i[1] for i in active_data]
            if baseline == 'zero':
                min_val = 0
                step = float(max(values)) / len(self.ticks)
            elif baseline == 'min':
                min_val = min(values)
                step = float(max(values) - min_val) / len(self.ticks)
            ranges = [step * i for i in range(len(self.ticks) + 1)]
            for color, value in active_data:
                if value <= 0:
                    tick = ' '
                else:
                    rank = bisect.bisect_left(ranges, value) - 1
                    rank = max(rank, 0)
                    rank = min(rank, len(self.ticks) - 1)
                    tick = self.ticks[rank]
                result.append(color)
                result.append(tick)
        space = ' ' * (spark_len - len(active_data))
        if align == 'right':
            result = [space] + result
        elif align == 'left':
            result = result + [space]
        return result

class Screen(object):
    '''     Console screen renderer class    '''
    RIGHT_PANEL_SEPARATOR = ' . '

    def __init__(self, info_panel_width, markup_provider, **kwargs):
        self.log = logging.getLogger(__name__)
        self.info_panel_percent = int(info_panel_width)
        self.info_widgets = {}
        self.markup = markup_provider
        self.term_height = 60
        self.term_width = 120
        self.right_panel_width = 10
        self.left_panel_width = self.term_width - self.right_panel_width - len(
            self.RIGHT_PANEL_SEPARATOR)
        cases_args = dict(
            [(k, v)
                for k, v in kwargs.iteritems()
                if k in ['cases_sort_by', 'cases_max_spark', 'max_case_len']]
        )
        times_args = {'times_max_spark': kwargs['times_max_spark']}
        sizes_args = {'sizes_max_spark': kwargs['sizes_max_spark']}

        codes_block = VerticalBlock(CurrentHTTPBlock(self), CurrentNetBlock(self), self)
        times_block = VerticalBlock(AnswSizesBlock(self, **sizes_args), AvgTimesBlock(self, **times_args), self)
        top_block = VerticalBlock(codes_block, times_block, self)
        general_block = HorizontalBlock(PercentilesBlock(self), top_block, self)
        overall_block = VerticalBlock(RPSBlock(self), general_block, self)
        final_block = VerticalBlock(overall_block, CasesBlock(self, **cases_args), self)

        self.left_panel = final_block

    def __get_right_line(self, widget_output):
        '''        Gets next line for right panel        '''
        right_line = ''
        if widget_output:
            right_line = widget_output.pop(0)
            if len(right_line) > self.right_panel_width:
                right_line_plain = self.markup.clean_markup(right_line)
                if len(right_line_plain) > self.right_panel_width:
                    right_line = right_line[:self.right_panel_width] + self.markup.RESET
        return right_line

    def __truncate(self, line_arr, max_width):
        '''  Cut tuple of line chunks according to it's wisible lenght  '''
        def is_space(chunk):
            return all([True if i == ' ' else False for i in chunk])

        def is_empty(chunks, markups):
            result = []
            for chunk in chunks:
                if chunk in markups:
                    result.append(True)
                elif is_space(chunk):
                    result.append(True)
                else:
                    result.append(False)
            return all(result)
        left = max_width
        result = ''
        markups = self.markup.get_markup_vars()
        for num, chunk in enumerate(line_arr):
            if chunk in markups:
                result += chunk
            else:
                if left > 0:
                    if len(chunk) <= left:
                        result += chunk
                        left -= len(chunk)
                    else:
                        leftover = (chunk[left:],) + line_arr[num + 1:]
                        was_cut = not is_empty(leftover, markups)
                        if was_cut:
                            result += chunk[:left - 1] + self.markup.RESET + u'\u2026'
                        else:
                            result += chunk[:left]
                        left = 0
        return result

    def __render_left_panel(self):
        ''' Render left blocks '''
        self.log.debug("Rendering left blocks")
        left_block = self.left_panel
        left_block.render()
        blank_space = self.left_panel_width - left_block.width

        lines = []
        pre_space = ' ' * int(blank_space / 2)
        if not left_block.lines:
            lines = [(''), (self.markup.RED + 'BROKEN LEFT PANEL' + self.markup.RESET)]
        else:
            while self.left_panel.lines:
                src_line = self.left_panel.lines.pop(0)
                line = pre_space + self.__truncate(src_line, self.left_panel_width)
                post_space = ' ' * (self.left_panel_width - len(self.markup.clean_markup(line)))
                line += post_space + self.markup.RESET
                lines.append(line)
        return lines

    def render_screen(self):
        '''        Main method to render screen view        '''
        self.term_width, self.term_height = get_terminal_size()
        self.log.debug(
            "Terminal size: %sx%s", self.term_width, self.term_height)
        self.right_panel_width = int(
            (self.term_width - len(self.RIGHT_PANEL_SEPARATOR))
            * (float(self.info_panel_percent) / 100)) - 1
        if self.right_panel_width > 0:
            self.left_panel_width = self.term_width - \
                self.right_panel_width - len(self.RIGHT_PANEL_SEPARATOR) - 2
        else:
            self.right_panel_width = 0
            self.left_panel_width = self.term_width - 1
        self.log.debug(
            "Left/right panels width: %s/%s", self.left_panel_width,
            self.right_panel_width)

        widget_output = []
        if self.right_panel_width:
            widget_output = []
            self.log.debug("There are %d info widgets" % len(self.info_widgets))
            for index, widget in sorted(
                    self.info_widgets.iteritems(),
                    key=lambda item: (item[1].get_index(), item[0])):
                self.log.debug("Rendering info widget #%s: %s", index, widget)
                widget_out = widget.render(self).strip()
                if widget_out:
                    widget_output += widget_out.split("\n")
                    widget_output += [""]

        left_lines = self.__render_left_panel()

        self.log.debug("Composing final screen output")
        output = []
        for line_no in range(1, self.term_height):
            line = " "

            if line_no > 1 and left_lines:
                left_line = left_lines.pop(0)
                left_line_plain = self.markup.clean_markup(left_line)

                left_line += (
                    ' ' * (self.left_panel_width - len(left_line_plain)))
                line += left_line
            else:
                line += ' ' * self.left_panel_width
            if self.right_panel_width:
                line += self.markup.RESET
                line += self.markup.WHITE
                line += self.RIGHT_PANEL_SEPARATOR
                line += self.markup.RESET
                right_line = self.__get_right_line(widget_output)
                line += right_line

            output.append(line)
        return self.markup.new_line.join(output) + self.markup.new_line

    def add_info_widget(self, widget):
        '''
        Add widget string to right panel of the screen
        '''
        index = widget.get_index()
        while index in self.info_widgets.keys():
            index += 1
        self.info_widgets[widget.get_index()] = widget

    def add_second_data(self, data):
        '''
        Notification method about new aggregator data
        '''
        self.left_panel.add_second(data)

class AbstractBlock:
    '''
    Parent class for all left panel blocks
    '''

    def __init__(self, screen):
        self.log = logging.getLogger(__name__)
        self.lines = []
        self.width = 0
        self.screen = screen

    def add_second(self, data):
        '''
        Notification about new aggregate data
        '''
        pass

    def fill_rectangle(self, prepared):
        '''  Right-pad lines of block to equal width  '''
        result = []
        width = max([self.clean_len(line) for line in prepared])
        for line in prepared:
            spacer = ' ' * (width - self.clean_len(line))
            result.append(line + (self.screen.markup.RESET, spacer))
        return (width, result)

    def clean_len(self, line):
        '''  Calculate wisible length of string  '''
        if isinstance(line, basestring):
            return len(self.screen.markup.clean_markup(line))
        elif isinstance(line, tuple) or isinstance(line, list):
            markups = self.screen.markup.get_markup_vars()
            length = 0
            for i in line:
                if i not in markups:
                    length += len(i)
            return length

    def render(self):
        '''
        Render method, fills .lines and .width properties with rendered data
        '''
        raise RuntimeError("Abstract method needs to be overridden")

class HorizontalBlock(AbstractBlock):
    '''
    Block to merge two other blocks horizontaly
    '''

    def __init__(self, left_block, right_block, screen):
        AbstractBlock.__init__(self, screen)
        self.left = left_block
        self.right = right_block
        self.separator = '  . '

    def render(self, expected_width=None):
        if not expected_width:
            expected_width = self.screen.left_panel_width

        def get_line(source, num):
            if num >= len(source.lines):
                return (' ' * source.width,)
            else:
                source_line = source.lines[n]
                spacer = ' ' * (source.width - self.clean_len(source_line))
                return source_line + (spacer,)
        self.left.render(expected_width=expected_width)
        right_width_limit = expected_width - self.left.width - len(self.separator)
        self.right.render(expected_width=right_width_limit)
        self.height = max(len(self.left.lines), len(self.right.lines))
        self.width = self.left.width + self.right.width + len(self.separator)

        self.lines = []

        for n in range(self.height):
            self.lines.append(
                get_line(self.left, n) + (self.separator,) + get_line(self.right, n)
            )
        self.lines.append((' ' * self.width,))

    def add_second(self, data):
        self.left.add_second(data)
        self.right.add_second(data)

class VerticalBlock(AbstractBlock):
    '''
    Block to merge two other blocks vertically
    '''

    def __init__(self, top_block, bottom_block, screen):
        AbstractBlock.__init__(self, screen)
        self.top = top_block
        self.bottom = bottom_block

    def render(self, expected_width=None):
        if not expected_width:
            expected_width = self.screen.left_panel_width

        self.top.render(expected_width=expected_width)
        self.bottom.render(expected_width=expected_width)
        self.width = max(self.top.width, self.bottom.width)

        self.lines = []
        for line in self.top.lines:
            spacer = ' ' * (self.width - self.top.width)
            self.lines.append(line + (spacer,))

        if self.top.lines and self.bottom.lines:
            spacer = ' ' * self.width
            self.lines.append((spacer,))

        for line in self.bottom.lines:
            spacer = ' ' * (self.width - self.bottom.width)
            self.lines.append(line + (spacer,))

    def add_second(self, data):
        self.top.add_second(data)
        self.bottom.add_second(data)

class RPSBlock(AbstractBlock):
    ''' Actual RPS sparkline '''

    def __init__(self, screen):
        AbstractBlock.__init__(self, screen)
        self.begin_tpl = 'Data delay: {delay}s, RPS: {rps:>3,} '
        self.sparkline = Sparkline(180)
        self.last_count = 0
        self.last_second = None

    def add_second(self, data):
        count = data['overall']['interval_real']['len']
        self.last_count = count
        ts = data['ts']
        self.last_second = ts
        self.sparkline.add(ts, 'rps', count)

    def render(self, expected_width=None):
        if self.last_second:
            delay = int(time.time() - self.last_second)
        else:
            delay = ' - '
        line_start = self.begin_tpl.format(rps=self.last_count, delay=delay)
        spark_len = expected_width - len(line_start) - 2

        spark = self.sparkline.get_sparkline('rps', spark_len=spark_len)
        prepared = [(self.screen.markup.WHITE, line_start, ' ') + tuple(spark) + (self.screen.markup.RESET,)]
        self.width, self.lines = self.fill_rectangle(prepared)

class PercentilesBlock(AbstractBlock):
    ''' Aggregated percentiles '''

    def __init__(self, screen):
        AbstractBlock.__init__(self, screen)
        self.title = 'Percentiles (all/last 1m/last), ms:'
        self.overall = None
        self.width = 10
        self.last_ts = None
        self.last_min = {}
        self.quantiles = [10, 20, 30, 40, 50, 60, 70, 75, 80, 85, 90, 95, 99, 99.5, 100]
        template = {
            'quantile': {'tpl': '{:>.1f}%'},
            'all':      {'tpl': '{:>,.1f}'},  # noqa: E241
            'last_1m':  {'tpl': '{:>,.1f}'},  # noqa: E241
            'last':     {'tpl': '{:>,.1f}'}   # noqa: E241
        }
        delimiters = [' <  ', '  ']
        self.formatter = TableFormatter(template, delimiters)

    def add_second(self, data):
        incoming_hist = data['overall']['interval_real']['hist']
        ts = data['ts']
        self.precise_quantiles = {
            q: float(v) / 1000
            for q, v in zip(
                data["overall"]["interval_real"]["q"]["q"],
                data["overall"]["interval_real"]["q"]["value"])
        }
        dist = pd.Series(incoming_hist['data'], index=incoming_hist['bins'])
        if self.overall is None:
            self.overall = dist
        else:
            self.overall = self.overall.add(dist, fill_value=0)
        for second in self.last_min.keys():
            if ts - second > 60:
                self.last_min.pop(second)
        self.last_min[ts] = dist

    def __calc_percentiles(self):
        def hist_to_quant(histogram, quant):
            cumulative = histogram.cumsum()
            total = cumulative.max()
            positions = cumulative.searchsorted([float(i) / 100 * total for i in quant])
            quant_times = [cumulative.index[i] / 1000. for i in positions]
            return quant_times

        all_times = hist_to_quant(self.overall, self.quantiles)
        last_data = self.last_min[max(self.last_min.keys())]
        last_times = hist_to_quant(last_data, self.quantiles)
        # Check if we have precise data for last second quantiles instead of binned histogram
        for position, q in enumerate(self.quantiles):
            if q in self.precise_quantiles:
                last_times[position] = self.precise_quantiles[q]
        # Replace binned values with precise, if lower quantile bin happens to be
        # greater than upper quantile precise values
        for position in reversed(range(1, len(last_times))):
            if last_times[position - 1] > last_times[position]:
                last_times[position - 1] = last_times[position]
        last_1m = pd.Series()
        for ts, data in self.last_min.iteritems():
            if last_1m.empty:
                last_1m = data
            else:
                last_1m = last_1m.add(data, fill_value=0)
        last_1m_times = hist_to_quant(last_1m, self.quantiles)
        quant_times = reversed(
            zip(self.quantiles, all_times, last_1m_times, last_times)
        )
        data = []
        for q, all_time, last_1m, last_time in quant_times:
            data.append({
                'quantile': q,
                'all': all_time,
                'last_1m': last_1m,
                'last': last_time
            })
        return data

    def render(self, expected_width=None):
        prepared = [(self.screen.markup.WHITE, self.title)]
        if self.overall is None:
            prepared.append(('',))
        else:
            data = self.__calc_percentiles()
            prepared += self.formatter.render_table(data, ['quantile', 'all', 'last_1m', 'last'])
        self.width, self.lines = self.fill_rectangle(prepared)

class CurrentHTTPBlock(AbstractBlock):
    ''' Http codes with highlight'''

    def __init__(self, screen):
        AbstractBlock.__init__(self, screen)
        self.overall_dist = defaultdict(int)
        self.title = 'HTTP codes: '
        self.total_count = 0
        template = {
            'count':       {'tpl': '{:>,}'},  # noqa: E241
            'last':        {'tpl': '+{:>,}'},  # noqa: E241
            'percent':     {'tpl': '{:>.2f}%'},  # noqa: E241
            'code':        {'tpl': '{:>3}', 'final': '{{{:}:<{len}}}'},  # noqa: E241
            'description': {'tpl': '{:<10}', 'final': '{{{:}:<{len}}}'}
        }
        delimiters = [' ', '  ', ' : ', ' ']
        self.formatter = TableFormatter(template, delimiters)

    def add_second(self, data):
        self.last_dist = data["overall"]["proto_code"]["count"]
        for code, count in self.last_dist.iteritems():
            self.total_count += count
            self.overall_dist[code] += count

    def __code_color(self, code):
        colors = {(200, 299): self.screen.markup.GREEN,
                  (300, 399): self.screen.markup.CYAN,
                  (400, 499): self.screen.markup.YELLOW,
                  (500, 599): self.screen.markup.RED}
        if code in self.last_dist.keys():
            for left, right in colors:
                if left <= int(code) <= right:
                    return colors[(left, right)]
            return self.screen.markup.MAGENTA
        else:
            return ''

    def __code_descr(self, code):
        if int(code) in util.HTTP:
            return util.HTTP[int(code)]
        else:
            return 'N/A'

    def render(self, expected_width=None):
        prepared = [(self.screen.markup.WHITE, self.title)]
        if not self.overall_dist:
            prepared.append(('',))
        else:
            data = []
            for code, count in sorted(self.overall_dist.iteritems()):
                if code in self.last_dist:
                    last_count = self.last_dist[code]
                else:
                    last_count = 0
                data.append({
                    'count': count,
                    'last': last_count,
                    'percent': 100 * safe_div(count, self.total_count),
                    'code': code,
                    'description': self.__code_descr(code)
                })
            table = self.formatter.render_table(data, ['count', 'last', 'percent', 'code', 'description'])
            for num, line in enumerate(data):
                color = self.__code_color(line['code'])
                prepared.append((color, table[num][0]))
        self.width, self.lines = self.fill_rectangle(prepared)

class CurrentNetBlock(AbstractBlock):
    ''' NET codes with highlight'''

    def __init__(self, screen):
        AbstractBlock.__init__(self, screen)
        self.overall_dist = defaultdict(int)
        self.title = 'Net codes:'
        self.total_count = 0
        template = {
            'count':       {'tpl': '{:>,}'},  # noqa: E241
            'last':        {'tpl': '+{:>,}'},  # noqa: E241
            'percent':     {'tpl': '{:>.2f}%'},  # noqa: E241
            'code':        {'tpl': '{:>2}', 'final': '{{{:}:<{len}}}'},  # noqa: E241
            'description': {'tpl': '{:<10}', 'final': '{{{:}:<{len}}}'}
        }
        delimiters = [' ', '  ', ' : ', ' ']
        self.formatter = TableFormatter(template, delimiters)

    def add_second(self, data):
        self.last_dist = data["overall"]["net_code"]["count"]

        for code, count in self.last_dist.iteritems():
            self.total_count += count
            self.overall_dist[code] += count

    def __code_descr(self, code):
        if int(code) in util.NET:
            return util.NET[int(code)]
        else:
            return 'N/A'

    def __code_color(self, code):
        if code in self.last_dist.keys():
            if int(code) == 0:
                return self.screen.markup.GREEN
            elif int(code) == 314:
                return self.screen.markup.MAGENTA
            else:
                return self.screen.markup.RED
        else:
            return ''

    def render(self, expected_width=None):
        prepared = [(self.screen.markup.WHITE, self.title)]
        if not self.overall_dist:
            prepared.append(('',))
        else:
            data = []
            for code, count in sorted(self.overall_dist.iteritems()):
                if code in self.last_dist:
                    last_count = self.last_dist[code]
                else:
                    last_count = 0
                data.append({
                    'count': count,
                    'last': last_count,
                    'percent': 100 * safe_div(count, self.total_count),
                    'code': code,
                    'description': self.__code_descr(code)
                })
            table = self.formatter.render_table(data, ['count', 'last', 'percent', 'code', 'description'])
            for num, line in enumerate(data):
                color = self.__code_color(line['code'])
                prepared.append((color, table[num][0]))
        self.width, self.lines = self.fill_rectangle(prepared)

class AnswSizesBlock(AbstractBlock):
    ''' Answer and response sizes, if available '''

    def __init__(self, screen, sizes_max_spark=120):
        AbstractBlock.__init__(self, screen)
        self.sparkline = Sparkline(sizes_max_spark)
        self.overall = {'count': 0, 'Response': 0, 'Request': 0}
        self.last = {'count': 0, 'Response': 0, 'Request': 0}
        self.title = 'Average Sizes (all/last), bytes:'
        template = {
            'name': {'tpl': '{:>}'},
            'avg': {'tpl': '{:>,.1f}'},
            'last_avg': {'tpl': '{:>,.1f}'}
        }
        delimiters = [': ', ' / ']
        self.formatter = TableFormatter(template, delimiters)

    def add_second(self, data):
        self.last['count'] = data["overall"]["interval_real"]["len"]
        self.last['Request'] = data["overall"]["size_out"]["total"]
        self.last['Response'] = data["overall"]["size_in"]["total"]

        self.overall['count'] += self.last['count']
        self.overall['Request'] += self.last['Request']
        self.overall['Response'] += self.last['Response']
        ts = data['ts']
        for direction in ['Request', 'Response']:
            self.sparkline.add(ts, direction, self.last[direction] / self.last['count'])

    def render(self, expected_width=None):
        prepared = [(self.screen.markup.WHITE, self.title)]
        if self.overall['count']:
            overall_avg = avg_from_dict(self.overall)
            last_avg = avg_from_dict(self.last)
            data = []
            for direction in ['Request', 'Response']:
                data.append({
                    'name': direction,
                    'avg': overall_avg[direction],
                    'last_avg': last_avg[direction]
                })
            table = self.formatter.render_table(data, ['name', 'avg', 'last_avg'])
            for num, direction in enumerate(['Request', 'Response']):
                spark_len = expected_width - self.clean_len(table[0]) - 3
                spark = self.sparkline.get_sparkline(direction, spark_len=spark_len)
                prepared.append(table[num] + ('  ',) + tuple(spark))
        else:
            self.lines.append(('',))
            self.lines.append(('',))
        self.width, self.lines = self.fill_rectangle(prepared)

class AvgTimesBlock(AbstractBlock):
    ''' Average times breakdown '''

    def __init__(self, screen, times_max_spark=120):
        AbstractBlock.__init__(self, screen)
        self.sparkline = Sparkline(times_max_spark)
        self.fraction_keys = [
            'interval_real', 'connect_time', 'send_time', 'latency', 'receive_time']
        self.fraction_names = {
            'interval_real': 'Overall',
            'connect_time':  'Connect',  # noqa: E241
            'send_time':     'Send',  # noqa: E241
            'latency':       'Latency',  # noqa: E241
            'receive_time':  'Receive'}  # noqa: E241
        self.overall = dict([(k, 0) for k in self.fraction_keys])
        self.overall['count'] = 0
        self.last = dict([(k, 0) for k in self.fraction_keys])
        self.last['count'] = 0
        self.title = 'Average Times (all/last), ms:'
        template = {
            'name': {'tpl': '{:>}'},
            'avg': {'tpl': '{:>,.2f}'},
            'last_avg': {'tpl': '{:>,.2f}'}
        }
        delimiters = [': ', ' / ']
        self.formatter = TableFormatter(template, delimiters)

    def add_second(self, data):
        self.last = {}
        self.last['count'] = data["overall"]["interval_real"]["len"]
        self.overall['count'] += self.last['count']
        ts = data["ts"]
        for fraction in self.fraction_keys:
            self.last[fraction] = float(data["overall"][fraction]["total"]) / 1000
            self.overall[fraction] += self.last[fraction]
            self.sparkline.add(ts, fraction, self.last[fraction] / self.last['count'])

    def render(self, expected_width=None):
        prepared = [(self.screen.markup.WHITE, self.title)]
        if self.overall['count']:
            overall_avg = avg_from_dict(self.overall)
            last_avg = avg_from_dict(self.last)
            data = []
            for fraction in self.fraction_keys:
                data.append({
                    'name': self.fraction_names[fraction],
                    'avg': overall_avg[fraction],
                    'last_avg': last_avg[fraction]
                })
            table = self.formatter.render_table(data, ['name', 'avg', 'last_avg'])
            for num, fraction in enumerate(self.fraction_keys):
                spark_len = expected_width - self.clean_len(table[0]) - 3
                spark = self.sparkline.get_sparkline(fraction, spark_len=spark_len)
                prepared.append(table[num] + ('  ',) + tuple(spark))
        else:
            for fraction in self.fraction_keys:
                prepared.append(('-',))
        self.width, self.lines = self.fill_rectangle(prepared)

class CasesBlock(AbstractBlock):
    '''     Cases info    '''

    def __init__(self, screen, cases_sort_by='http_err', cases_max_spark=60, reorder_delay=5, max_case_len=32):
        AbstractBlock.__init__(self, screen)
        self.cumulative_cases = {}
        self.last_cases = {}
        self.title = 'Cumulative Cases Info:'
        self.max_case_len = max_case_len
        self.cases_order = []
        self.reorder_delay = reorder_delay
        self.sparkline = Sparkline(cases_max_spark)
        self.cases_sort_by = cases_sort_by
        self.last_reordered = time.time()
        self.field_order = ['name', 'count', 'percent', 'last', 'net_err', 'http_err', 'avg', 'last_avg']

        template = {
            'name':     {'tpl': u'{:>}:',   'header': 'name', 'final': u'{{{:}:>{len}}}'},  # noqa: E241
            'count':    {'tpl': '{:>,}',    'header': 'count'},   # noqa: E241
            'last':     {'tpl': '+{:>,}',   'header': 'last'},    # noqa: E241
            'percent':  {'tpl': '{:>.2f}%', 'header': '%'},       # noqa: E241
            'net_err':  {'tpl': '{:>,}',    'header': 'net_e'},   # noqa: E241
            'http_err': {'tpl': '{:>,}',    'header': 'http_e'},  # noqa: E241
            'avg':      {'tpl': '{:>,.1f}', 'header': 'avg ms'},  # noqa: E241
            'last_avg': {'tpl': '{:>,.1f}', 'header': 'last ms'}
        }
        delimiters = [' ']
        self.formatter = TableFormatter(template, delimiters)

    def add_second(self, data):
        def prepare_data(tag_data, display_name):
            count = tag_data["interval_real"]["len"]
            time = tag_data["interval_real"]["total"] / 1000
            net_err, http_err, spark_color = combine_codes(tag_data, self.screen.markup)
            if spark_color == self.screen.markup.GREEN:
                text_color = self.screen.markup.WHITE
            else:
                text_color = spark_color
            return (spark_color, {
                'count': count, 'net_err': net_err, 'http_err': http_err,
                'time': time, 'color': text_color, 'display_name': display_name})

        ts = data["ts"]
        overall = data["overall"]
        self.last_cases = {}
        spark_color, self.last_cases[0] = prepare_data(overall, u'OVERALL')
        self.sparkline.add(ts, 0, self.last_cases[0]['count'], color=spark_color)

        tagged = data["tagged"]
        for tag_name, tag_data in tagged.iteritems():
            # decode symbols to utf-8 in order to support cyrillic symbols in cases
            name = tag_name.decode('utf-8')
            spark_color, self.last_cases[name] = prepare_data(tag_data, name)
            self.sparkline.add(ts, name, self.last_cases[name]['count'], color=spark_color)
        for name in self.last_cases:
            if name not in self.cumulative_cases:
                self.cumulative_cases[name] = {}
                for k in ['count', 'net_err', 'http_err', 'time', 'display_name']:
                    self.cumulative_cases[name][k] = self.last_cases[name][k]
            else:
                for k in ['count', 'net_err', 'http_err', 'time']:
                    self.cumulative_cases[name][k] += self.last_cases[name][k]

    def __cut_name(self, name):
        if len(name) > self.max_case_len:
            return name[:self.max_case_len - 1] + u'\u2026'
        else:
            return name

    def __reorder_cases(self):
        sorted_cases = sorted(self.cumulative_cases.iteritems(),
                              key=lambda (k, v): (-1 * v[self.cases_sort_by], k))
        new_order = [case for (case, data) in sorted_cases]
        now = time.time()
        if now - self.reorder_delay > self.last_reordered:
            self.cases_order = new_order
            self.last_reordered = now
        else:
            if len(new_order) > len(self.cases_order):
                for case in new_order:
                    if case not in self.cases_order:
                        self.cases_order.append(case)

    def render(self, expected_width=None):
        prepared = [(self.screen.markup.WHITE, self.title)]

        if 0 in self.cumulative_cases:  # 0 used as special name for OVERALL to avoid name collision
            total_count = self.cumulative_cases[0]['count']

            self.__reorder_cases()
            data = []
            for name in self.cases_order:
                case_data = self.cumulative_cases[name]
                if name in self.last_cases:
                    last = self.last_cases[name]
                else:
                    last = {'count': 0, 'net_err': 0, 'http_err': 0, 'time': 0, 'color': '', 'display_name': name}
                data.append({
                    'full_name': name,
                    'name': self.__cut_name(case_data['display_name']),
                    'count': case_data['count'],
                    'percent': 100 * safe_div(case_data['count'], total_count),
                    'last': last['count'],
                    'net_err': case_data['net_err'],
                    'http_err': case_data['http_err'],
                    'avg': safe_div(case_data['time'], case_data['count']),
                    'last_avg': safe_div(last['time'], last['count'])
                })
            table = self.formatter.render_table(data, self.field_order)
            prepared.append(table[0])  # first line is table header
            for num, line in enumerate(data):
                full_name = line['full_name']
                if full_name in self.last_cases:
                    color = self.last_cases[full_name]['color']
                else:
                    color = ''
                spark_len = expected_width - self.clean_len(table[0]) - 3
                spark = self.sparkline.get_sparkline(full_name, spark_len=spark_len)
                prepared.append((color,) + table[num + 1] + ('  ',) + tuple(spark))

        for _ in range(3 - len(self.cumulative_cases)):
            prepared.append(('',))

        self.width, self.lines = self.fill_rectangle(prepared)