'''
Load Plan generators
'''
import re
from itertools import chain, groupby
from builtins import range
from . import info
from .util import parse_duration, solve_quadratic, proper_round

class Const(object):
    '''
    Load plan with constant load
    '''

    def __init__(self, rps, duration):
        self.rps = float(rps)
        self.duration = duration

    def __iter__(self):
        if self.rps == 0:
            return iter([])
        interval = 1000.0 / self.rps
        return (
            int(i * interval)
            for i in range(0, int(self.rps * self.duration / 1000)))

    def rps_at(self, t):
        '''Return rps for second t'''
        if 0 <= t <= self.duration:
            return self.rps
        else:
            return 0

    def get_duration(self):
        '''Return step duration'''
        return self.duration

    def __len__(self):
        '''Return total ammo count'''
        return int(self.duration / 1000 * self.rps)

    def get_rps_list(self):
        return [(int(self.rps), self.duration / 1000)]

    def __repr__(self):
        return 'const(%s, %s)' % (self.rps, self.duration / 1000)

class Line(object):
    '''Load plan with linear load'''

    def __init__(self, minrps, maxrps, duration):
        """

        :param minrps:
        :param maxrps:
        :param duration: milliseconds
        """
        self.minrps = float(minrps)
        self.maxrps = float(maxrps)
        self.duration = duration / 1000.0
        self.slope = (self.maxrps - self.minrps) / self.duration

    def ts(self, n):
        """
        :param n: number of charge
        :return: when to shoot nth charge, milliseconds
        """
        try:
            root1, root2 = solve_quadratic(self.slope / 2.0, self.minrps, -n)
        except ZeroDivisionError:
            root2 = float(n) / self.minrps
        return int(root2 * 1000)

    def __iter__(self):
        """

        :return: timestamps for each charge
        """
        return (self.ts(n) for n in range(0, self.__len__()))

    def rps_at(self, t):
        '''Return rps for second t'''
        if 0 <= t <= self.duration:
            return self.minrps + \
                float(self.maxrps - self.minrps) * t / self.duration
        else:
            return 0

    def get_duration(self):
        '''Return load duration in seconds'''
        return int(self.duration * 1000)

    def __len__(self):
        '''Return total ammo count'''
        return int((self.maxrps + self.minrps) / 2.0 * self.duration)

    def get_float_rps_list(self):
        '''
        get list of constant load parts (we have no constant load at all, but tank will think so),
        with parts durations (float)
        '''
        int_rps = range(int(self.minrps), int(self.maxrps) + 1)
        step_duration = float(self.duration) / len(int_rps)
        rps_list = [(rps, int(step_duration)) for rps in int_rps]
        return rps_list

    def get_rps_list(self):
        """
        get list of each second's rps
        :returns: list of tuples (rps, duration of corresponding rps in seconds)
        :rtype: list
        """
        seconds = range(0, int(self.duration) + 1)
        rps_groups = groupby([proper_round(self.rps_at(t)) for t in seconds],
                             lambda x: x)
        rps_list = [(rps, len(list(rpl))) for rps, rpl in rps_groups]
        return rps_list

class Composite(object):
    '''Load plan with multiple steps'''

    def __init__(self, steps):
        self.steps = steps

    def __iter__(self):
        base = 0
        for step in self.steps:
            for ts in step:
                yield ts + base
            base += step.get_duration()

    def get_duration(self):
        '''Return total duration'''
        return sum(step.get_duration() for step in self.steps)

    def __len__(self):
        '''Return total ammo count'''
        return int(sum(step.__len__() for step in self.steps))

    def get_rps_list(self):
        return list(
            chain.from_iterable(step.get_rps_list() for step in self.steps))

class Stairway(Composite):
    def __init__(self, minrps, maxrps, increment, step_duration):
        self.duration = step_duration
        if maxrps < minrps:
            increment = -increment
        n_steps = int((maxrps - minrps) / increment)
        steps = [
            Const(minrps + i * increment, step_duration)
            for i in range(0, n_steps + 1)
        ]
        if increment > 0:
            if (minrps + n_steps * increment) < maxrps:
                steps.append(Const(maxrps, step_duration))
        elif increment < 0:
            if (minrps + n_steps * increment) > maxrps:
                steps.append(Const(maxrps, step_duration))
        super(Stairway, self).__init__(steps)

class StepFactory(object):
    DURATION_RE = r'([0-9.]+d)?([0-9.]+h)?([0-9.]+m)?([0-9.]+s?)?'

    @classmethod
    def line(cls, params):
        template = re.compile(r'([0-9.]+),\s*([0-9.]+),\s*({})\)'.format(cls.DURATION_RE))
        minrps, maxrps, duration = template.search(params).groups()[:3]
        return Line(float(minrps), float(maxrps), parse_duration(duration))

    @classmethod
    def const(cls, params):
        template = re.compile(r'([0-9.]+),\s*({})\)'.format(cls.DURATION_RE))
        rps, duration = template.search(params).groups()[:2]
        return Const(float(rps), parse_duration(duration))

    @classmethod
    def stairway(cls, params):
        template = re.compile(
            r'([0-9.]+),\s*([0-9.]+),\s*([0-9.]+),\s*({})\)'.format(cls.DURATION_RE))
        minrps, maxrps, increment, duration = template.search(params).groups()[:4]
        return Stairway(
            float(minrps),
            float(maxrps), float(increment), parse_duration(duration))

    @staticmethod
    def produce(step_config):
        _plans = {
            'line': StepFactory.line,
            'const': StepFactory.const,
            'step': StepFactory.stairway,
        }
        load_type, params = step_config.split('(')
        load_type = load_type.strip()
        if load_type in _plans:
            return _plans[load_type](params)
        else:
            raise NotImplementedError(
                'No such load type implemented: "%s"' % load_type)

def create(rps_schedule):
    """
    Create Load Plan as defined in schedule. Publish info about its duration.
    """
    if len(rps_schedule) > 1:
        lp = Composite(
            [StepFactory.produce(step_config) for step_config in rps_schedule])
    else:
        lp = StepFactory.produce(rps_schedule[0])
    info.status.publish('duration', lp.get_duration() / 1000)
    info.status.publish('steps', lp.get_rps_list())
    info.status.lp_len = len(lp)
    return lp


'''
Contains shellexec plugin
'''
from netort.process import execute
from ...common.interfaces import AbstractPlugin

class Plugin(AbstractPlugin):
    """
    ShellExec plugin
    allows executing shell scripts before/after test
    """
    SECTION = 'shellexec'

    def __init__(self, core, cfg, name):
        AbstractPlugin.__init__(self, core, cfg, name)
        self.catch_out = False
        self.end = None
        self.poll = None
        self.prepare = None
        self.start = None
        self.postprocess = None

    @staticmethod
    def get_key():
        return __file__

    def get_available_options(self):
        return ["prepare", "start", "end", "poll", "post_process", "catch_out"]

    def configure(self):
        self.catch_out = True if self.get_option("catch_out", False) else False
        self.prepare = self.get_option("prepare", '')
        self.start = self.get_option("start", '')
        self.end = self.get_option("end", '')
        self.poll = self.get_option("poll", '')
        self.postprocess = self.get_option("post_process", '')

    def prepare_test(self):
        if self.prepare:
            self.execute(self.prepare)

    def start_test(self):
        if self.start:
            self.execute(self.start)

    def is_test_finished(self):
        if self.poll:
            self.log.info("Executing: %s", self.poll)
            retcode = execute(
                self.poll,
                shell=True,
                poll_period=0.1,
                catch_out=self.catch_out)[0]
            if retcode:
                self.log.warn(
                    "Non-zero exit code, interrupting test: %s", retcode)
                return retcode
        return -1

    def end_test(self, retcode):
        if self.end:
            self.execute(self.end)
        return retcode

    def post_process(self, retcode):
        if self.postprocess:
            self.execute(self.postprocess)
        return retcode

    def execute(self, cmd):
        """
        Execute and check exit code
        """
        self.log.info("Executing: %s", cmd)
        retcode = execute(
            cmd, shell=True, poll_period=0.1, catch_out=self.catch_out)[0]
        if retcode:
            raise RuntimeError("Subprocess returned %s" % retcode)
        return retcode


import time

def uts(dt):
    return int(time.mktime(dt.timetuple()))

class Decoder(object):
    """
    Decode metrics incoming from tank into points for InfluxDB client

    Parameters
    ----------
    parent_tags : dict
        common per-test tags
    tank_tag : basestring
        tank identifier tag
    uuid : basestring
        test id tag
    labeled : bool
        detailed stats for each label
    histograms : bool
        response time histograms measurements
    """
    def __init__(self, tank_tag, uuid, parent_tags, labeled, histograms):
        self.labeled = labeled
        initial_tags = {
            "tank": tank_tag,
            "uuid": uuid
        }
        initial_tags.update(parent_tags)
        self.tags = initial_tags
        self.histograms = histograms

    def set_uuid(self, id_):
        self.tags['uuid'] = id_

    def decode_monitoring(self, data):
        """
        The reason why we have two separate methods for monitoring
        and aggregates is a strong difference in incoming data.
        """
        points = list()
        for second_data in data:
            for host, host_data in second_data["data"].iteritems():
                points.append(
                    self.__make_points(
                        "monitoring",
                        {"host": host, "comment": host_data.get("comment")},
                        second_data["timestamp"],
                        {
                            metric: value
                            for metric, value in host_data["metrics"].iteritems()
                        }
                    )
                )
        return points

    def decode_aggregates(self, aggregated_data, gun_stats, prefix):
        ts = aggregated_data["ts"]
        points = list()
        # stats overall w/ __OVERALL__ label
        points += self.__make_points_for_label(
            ts,
            aggregated_data["overall"],
            "__OVERALL__",
            prefix,
            gun_stats
        )
        # detailed stats per tag
        if self.labeled:
            for label, aggregated_data_by_tag in aggregated_data["tagged"].items():
                points += self.__make_points_for_label(
                    ts,
                    aggregated_data_by_tag,
                    label,
                    prefix,
                    gun_stats
                )
        return points

    def __make_points_for_label(self, ts, data, label, prefix, gun_stats):
        """x
        Make a set of points for `this` label

        overall_quantiles, overall_meta, net_codes, proto_codes, histograms
        """
        label_points = list()

        label_points.extend(
            (
                # overall quantiles for label
                self.__make_points(
                    prefix + "overall_quantiles",
                    {"label": label},
                    ts,
                    self.__make_quantile_fields(data)
                ),
                # overall meta (gun status) for label
                self.__make_points(
                    prefix + "overall_meta",
                    {"label": label},
                    ts,
                    self.__make_overall_meta_fields(data, gun_stats)
                ),
                # net codes for label
                self.__make_points(
                    prefix + "net_codes",
                    {"label": label},
                    ts,
                    self.__make_netcodes_fields(data)
                ),
                # proto codes for label
                self.__make_points(
                    prefix + "proto_codes",
                    {"label": label},
                    ts,
                    self.__make_protocodes_fields(data)
                )
            )
        )
        # histograms, one row for each bin
        if self.histograms:
            for bin_, count in zip(data["interval_real"]["hist"]["bins"],
                                   data["interval_real"]["hist"]["data"]):
                label_points.append(
                    self.__make_points(
                        prefix + "histograms",
                        {"label": label},
                        ts,
                        {"bin": bin_, "count": count}
                    )
                )
        return label_points

    @staticmethod
    def __make_quantile_fields(data):
        return {
            'q' + str(q): value / 1000.0
            for q, value in zip(data["interval_real"]["q"]["q"],
                                data["interval_real"]["q"]["value"])
        }

    @staticmethod
    def __make_overall_meta_fields(data, stats):
        return {
            "active_threads": stats["metrics"]["instances"],
            "RPS": data["interval_real"]["len"],
            "planned_requests": float(stats["metrics"]["reqps"]),
        }

    @staticmethod
    def __make_netcodes_fields(data):
        return {
            str(code): int(cnt)
            for code, cnt in data["net_code"]["count"].items()
        }

    @staticmethod
    def __make_protocodes_fields(data):
        return {
            str(code): int(cnt)
            for code, cnt in data["proto_code"]["count"].items()
        }

    def __make_points(self, measurement, additional_tags, ts, fields):
        """
        Parameters
        ----------
        measurement : string
            measurement type (e.g. monitoring, overall_meta, net_codes, proto_codes, overall_quantiles)
        additional_tags : dict
            custom additional tags for this points
        ts : integer
            timestamp
        fields : dict
            influxdb columns

        Returns
        -------
        dict
            points for InfluxDB client
        """
        tags = self.tags.copy()
        tags.update(additional_tags)
        return {
            "measurement": measurement,
            "tags": tags,
            "time": int(ts),
            "fields": fields,
        }


import logging

class AbstractPlugin(object):
    """ Plugin interface
    Parent class for all plugins """

    SECTION = 'DEFAULT'

    @staticmethod
    def get_key():
        """ Get dictionary key for plugin,
        should point to __file__ magic constant """
        raise TypeError("Abstract method needs to be overridden")

    # TODO: do we realy need cfg_updater here?
    def __init__(self, core, cfg, name):
        """

        :param name:
        :type core: yandextank.core.TankCore
        :type cfg: dict
        """
        super(AbstractPlugin, self).__init__()
        self._cleanup_actions = []
        self.log = logging.getLogger(__name__)
        self.core = core
        self.cfg = cfg
        self.cfg_section_name = name
        self.interrupted = self.core.interrupted

    def set_option(self, option, value):
        self.cfg[option] = value

    def configure(self):
        """ A stage to read config values and instantiate objects """
        pass

    def prepare_test(self):
        """        Test preparation tasks        """
        pass

    def start_test(self):
        """        Launch test process        """
        pass

    def is_test_finished(self):
        """
        Polling call, if result differs from -1 then test end
        will be triggered
        """
        return -1

    def add_cleanup(self, action):
        """
        :type action: function
        """
        assert callable(action)
        self._cleanup_actions.append(action)

    def cleanup(self):
        for action in reversed(self._cleanup_actions):
            try:
                action()
            except Exception:
                logging.error('Exception occurred during plugin cleanup {}'.format(self.__module__), exc_info=True)

    def end_test(self, retcode):
        """
        Stop processes launched at 'start_test',
        change return code if necessary
        """
        return retcode

    def post_process(self, retcode):
        """ Post-process test data """
        return retcode

    def get_option(self, option_name, default_value=None):
        """ Wrapper to get option from plugins' section """
        return self.cfg.get(option_name, default_value)

    def get_available_options(self):
        """ returns array containing known options for plugin """
        return []

    def get_multiline_option(self, option_name, default_value=None):
        if default_value is not None:
            default = ' '.join(default_value)
        else:
            default = None
        value = self.get_option(option_name, default)
        if value:
            return (' '.join(value.split("\n"))).split(' ')
        else:
            return ()

    def publish(self, key, value):
        """publish value to status"""
        self.log.debug(
            "Publishing status: %s/%s: %s", self.__class__.__name__, key, value)
        self.core.publish(self.__class__.__name__, key, value)

    def close(self):
        """
        Release allocated resources here.
        Warning: don't do any logic or potentially dangerous operations
        """
        pass

class MonitoringDataListener(object):
    """ Monitoring listener interface
    parent class for Monitoring data listeners"""

    def __init__(self):
        pass

    def monitoring_data(self, data):
        """Notification about new monitoring data lines"""
        raise NotImplementedError("Abstract method needs to be overridden")

class AggregateResultListener(object):
    """ Listener interface
    parent class for Aggregate results listeners"""

    def on_aggregated_data(self, data, stats):
        """
        notification about new aggregated data and stats

        data contains aggregated metrics and stats contain non-aggregated
        metrics from gun (like instances count, for example)

        data and stats are cached and synchronized by timestamp. Stat items
        are holded until corresponding data item is received and vice versa.
        """
        raise NotImplementedError("Abstract method should be overridden")

class AbstractInfoWidget(object):
    """ InfoWidgets interface
    parent class for all InfoWidgets"""

    def __init__(self):
        pass

    def render(self, screen):
        raise NotImplementedError("Abstract method should be overridden")

    def on_aggregated_data(self, data, stats):
        raise NotImplementedError("Abstract method should be overridden")

    def get_index(self):
        """ get vertical priority index """
        return 0

class AbstractCriterion(object):
    """ Criterions interface,
    parent class for all criterions """
    RC_TIME = 21
    RC_HTTP = 22
    RC_NET = 23
    RC_STEADY = 33

    def __init__(self):
        self.cause_second = None

    @staticmethod
    def count_matched_codes(codes_regex, codes_dict):
        """ helper to aggregate codes by mask """
        total = 0
        for code, count in codes_dict.items():
            if codes_regex.match(str(code)):
                total += count
        return total

    def notify(self, data, stat):
        """ notification about aggregate data goes here """
        raise NotImplementedError("Abstract methods requires overriding")

    def get_rc(self):
        """ get return code for test """
        raise NotImplementedError("Abstract methods requires overriding")

    def explain(self):
        """ long explanation to show after test stop """
        raise NotImplementedError("Abstract methods requires overriding")

    def get_criterion_parameters(self):
        """ returns dict with all criterion parameters """
        raise NotImplementedError("Abstract methods requires overriding")

    def widget_explain(self):
        """ short explanation to display in right panel """
        return self.explain(), 0

    @staticmethod
    def get_type_string():
        """ returns string that used as config name for criterion """
        raise NotImplementedError("Abstract methods requires overriding")

class GeneratorPlugin(AbstractPlugin):
    DEFAULT_INFO = {
        'address': '',
        'port': 80,
        'instances': 1,
        'ammo_file': '',
        'rps_schedule': [],
        'duration': 0,
        'loop_count': 0
    }

    def __init__(self, core, cfg, name):
        super(GeneratorPlugin, self).__init__(core, cfg, name)
        self.stats_reader = None
        self.reader = None
        self.process = None
        self.process_stderr = None
        self.start_time = None
        self.affinity = None
        self.buffered_seconds = 2

    class Info(object):
        def __init__(
                self, address, port, instances, ammo_file, rps_schedule,
                duration, loop_count):
            self.address = address
            self.port = port
            self.instances = instances
            self.ammo_file = ammo_file
            self.rps_schedule = rps_schedule
            self.duration = duration
            self.loop_count = loop_count

    def get_info(self):
        """
        :rtype: GeneratorPlugin.Info
        """
        return self.Info(**self.DEFAULT_INFO)

    def get_reader(self):
        """

        :rtype: collections.Iterable
        """
        pass

    def get_stats_reader(self):
        """

        :rtype: collections.Iterable
        """
        pass

    def end_test(self, retcode):
        pass

class StatsReader(object):
    @staticmethod
    def stats_item(ts, instances, rps):
        return {
            'ts': ts,
            'metrics': {
                'instances': instances,
                'reqps': rps
            }
        }

class MonitoringPlugin(AbstractPlugin):

    def __init__(self, core, cfg, name):
        super(MonitoringPlugin, self).__init__(core, cfg, name)
        self.listeners = []

    def add_listener(self, plugin):
        self.listeners.append(plugin)


import logging
import time
import threading as th
import multiprocessing as mp
from queue import Empty, Full

from ...stepper import StpdReader

logger = logging.getLogger(__name__)

class BFGBase(object):
    """
    A BFG load generator that manages multiple workers as processes and
    threads in each of them and feeds them with tasks
    """

    def __init__(self, gun, instances, stpd_filename, cached_stpd=False,
                 green_threads_per_instance=None):
        logger.info(
            """
BFG using stpd from {stpd_filename}
Instances: {instances}
Gun: {gun.__class__.__name__}
""".format(
                stpd_filename=stpd_filename,
                instances=instances,
                gun=gun, ))
        self.instances = int(instances)
        self.instance_counter = mp.Value('i')
        self.results = mp.Queue(16384)
        self.gun = gun
        self.gun.results = self.results
        self.quit = mp.Event()
        self.task_queue = mp.Queue(1024)
        self.cached_stpd = cached_stpd
        self.stpd_filename = stpd_filename
        self.pool = [
            mp.Process(target=self._worker) for _ in xrange(self.instances)
        ]
        self.feeder = th.Thread(target=self._feed, name="Feeder")
        self.feeder.daemon = True
        self.workers_finished = False
        self.start_time = None
        self.plan = None
        self.green_threads_per_instance = green_threads_per_instance

    def start(self):
        self.start_time = time.time()
        for process in self.pool:
            process.daemon = True
            process.start()
        self.feeder.start()

    def running(self):
        """
        True while there are alive workers out there. Tank
        will quit when this would become False
        """
        return not self.workers_finished

    def stop(self):
        """
        Say the workers to finish their jobs and quit.
        """
        self.quit.set()
        # yapf:disable
        while sorted([
                self.pool[i].is_alive()
                for i in xrange(len(self.pool))])[-1]:
            time.sleep(1)
        # yapf:enable
        try:
            while not self.task_queue.empty():
                self.task_queue.get(timeout=0.1)
            self.task_queue.close()
            self.feeder.join()
        except Exception as ex:
            logger.info(ex)

    def _feed(self):
        """
        A feeder that runs in distinct thread in main process.
        """
        self.plan = StpdReader(self.stpd_filename)
        if self.cached_stpd:
            self.plan = list(self.plan)
        for task in self.plan:
            if self.quit.is_set():
                logger.info("Stop feeding: gonna quit")
                return
            # try putting a task to a queue unless there is a quit flag
            # or all workers have exited
            while True:
                try:
                    self.task_queue.put(task, timeout=1)
                    break
                except Full:
                    if self.quit.is_set() or self.workers_finished:
                        return
                    else:
                        continue
        workers_count = self.instances
        logger.info(
            "Feeded all data. Publishing %d killer tasks" % (workers_count))
        retry_delay = 1
        for _ in range(5):
            try:
                [
                    self.task_queue.put(None, timeout=1)
                    for _ in xrange(0, workers_count)
                ]
                break
            except Full:
                logger.debug(
                    "Couldn't post killer tasks"
                    " because queue is full. Retrying in %ss", retry_delay)
                time.sleep(retry_delay)
                retry_delay *= 2

        try:
            logger.info("Waiting for workers")
            map(lambda x: x.join(), self.pool)
            logger.info("All workers exited.")
            self.workers_finished = True
        except (KeyboardInterrupt, SystemExit):
            self.task_queue.close()
            self.results.close()
            self.quit.set()
            logger.info("Going to quit. Waiting for workers")
            map(lambda x: x.join(), self.pool)
            self.workers_finished = True

class BFGMultiprocessing(BFGBase):
    """
    Default worker type, creates process per worker,
    every process executes requests synchronously inside.
    """
    def _worker(self):
        """
        A worker that does actual jobs
        """
        logger.debug("Init shooter process")
        try:
            self.gun.setup()
        except Exception:
            logger.exception("Couldn't initialize gun. Exit shooter process")
            return
        while not self.quit.is_set():
            try:
                task = self.task_queue.get(timeout=1)
                if not task:
                    logger.debug("Got killer task.")
                    break
                timestamp, missile, marker = task
                planned_time = self.start_time + (timestamp / 1000.0)
                delay = planned_time - time.time()
                if delay > 0:
                    time.sleep(delay)

                try:
                    with self.instance_counter.get_lock():
                        self.instance_counter.value += 1
                    self.gun.shoot(missile, marker)
                finally:
                    with self.instance_counter.get_lock():
                        self.instance_counter.value -= 1

            except (KeyboardInterrupt, SystemExit):
                break
            except Empty:
                if self.quit.is_set():
                    logger.debug("Empty queue. Exiting process")
                    return
            except Full:
                logger.warning("Couldn't put to result queue because it's full")
            except Exception:
                logger.exception("Bfg shoot exception")

        try:
            self.gun.teardown()
        except Exception:
            logger.exception("Couldn't finalize gun. Exit shooter process")
            return
        logger.debug("Exit shooter process")

class BFGGreen(BFGBase):
    """
    Green version of the worker. Starts `self.instances` processes,
    each of process has a pool of `self.green_threads_per_instance` green threads.
    """
    def _worker(self):
        from gevent import monkey, spawn
        from gevent.queue import Queue as GreenQueue
        # NOTE: Patching everything will conflict with multiprocessing
        monkey.patch_all(thread=False, select=False)

        logger.debug("Init shooter process")
        try:
            self.gun.setup()
        except Exception:
            logger.exception("Couldn't initialize gun. Exit shooter process")
            return

        self.green_queue = GreenQueue(self.green_threads_per_instance)
        self.green_pool = [spawn(self._green_worker) for _ in xrange(0, self.green_threads_per_instance)]

        # Keep track of tasks sent to greenlets. If all greenlets are busy -
        # don't pull more tasks from the main queue, let other workers do that.
        self._free_threads_count = self.green_threads_per_instance

        while not self.quit.is_set():
            while not self.task_queue.empty() and self._free_threads_count:
                try:
                    task = self.task_queue.get_nowait()
                except Empty:
                    continue

                self._free_threads_count -= 1

                if not task:
                    logger.debug("Got killer task.")
                    self.quit.set()
                    break

                self.green_queue.put(task)

            time.sleep(0.1)

        map(lambda g: g.join(), self.green_pool)

        try:
            self.gun.teardown()
        except Exception:
            logger.exception("Couldn't finalize gun. Exit shooter process")
            return
        logger.debug("Exit shooter process")

    def _green_worker(self):
        """
        A worker that does actual jobs
        """
        while not self.quit.is_set():
            try:
                task = self.green_queue.get(timeout=1)

                timestamp, missile, marker = task
                planned_time = self.start_time + (timestamp / 1000.0)
                delay = planned_time - time.time()

                if delay > 0:
                    time.sleep(delay)

                try:
                    with self.instance_counter.get_lock():
                        self.instance_counter.value += 1

                    self.gun.shoot(missile, marker)
                finally:
                    with self.instance_counter.get_lock():
                        self.instance_counter.value -= 1

                    self._free_threads_count += 1

            except (KeyboardInterrupt, SystemExit):
                break
            except Empty:
                continue
            except Full:
                logger.warning("Couldn't put to result queue because it's full")
            except Exception:
                logger.exception("Bfg shoot exception")


# -*- coding: utf-8 -*-
""" Provides class to run TankCore from python """
import logging
import os
import sys
import time
import traceback
import fnmatch
from pkg_resources import resource_filename

from ..core import tankcore

class ApiWorker:
    """    Worker class that runs tank core via python   """

    def __init__(self):
        self.core = tankcore.TankCore()
        self.baseconfigs_location = '/etc/yandex-tank'
        self.log = logging.getLogger(__name__)

    def init_logging(self, log_filename="tank.log"):
        """ Set up logging """
        logger = logging.getLogger('')
        self.log_filename = log_filename
        self.core.add_artifact_file(self.log_filename)

        file_handler = logging.FileHandler(self.log_filename)
        file_handler.setLevel(logging.DEBUG)
        file_handler.setFormatter(
            logging.Formatter(
                "%(asctime)s [%(levelname)s] %(name)s %(message)s"))
        logger.addHandler(file_handler)
        console_handler = logging.StreamHandler(sys.stdout)
        stderr_hdl = logging.StreamHandler(sys.stderr)

        # fmt_verbose = logging.Formatter(
        #     "%(asctime)s [%(levelname)s] %(name)s %(message)s")
        fmt_regular = logging.Formatter(
            "%(asctime)s %(levelname)s: %(message)s", "%H:%M:%S")

        console_handler.setLevel(logging.INFO)
        console_handler.setFormatter(fmt_regular)
        stderr_hdl.setFormatter(fmt_regular)

        f_err = SingleLevelFilter(logging.ERROR, True)
        f_warn = SingleLevelFilter(logging.WARNING, True)
        f_crit = SingleLevelFilter(logging.CRITICAL, True)
        console_handler.addFilter(f_err)
        console_handler.addFilter(f_warn)
        console_handler.addFilter(f_crit)
        logger.addHandler(console_handler)

        f_info = SingleLevelFilter(logging.INFO, True)
        f_debug = SingleLevelFilter(logging.DEBUG, True)
        stderr_hdl.addFilter(f_info)
        stderr_hdl.addFilter(f_debug)
        logger.addHandler(stderr_hdl)

    def __add_user_options(self):
        """ override config options with user specified options"""
        if self.options.get('user_options', None):
            self.core.apply_shorthand_options(self.options['user_options'])

    def configure(self, options):
        """ Make preparations before running Tank """
        self.options = options
        if self.options.get('lock_dir', None):
            self.core.set_option(self.core.SECTION, "lock_dir", self.options['lock_dir'])
        if self.options.get('ignore_lock', None):
            self.core.set_option(self.core.SECTION, 'ignore_lock', self.options['ignore_lock'])

        while True:
            try:
                self.core.get_lock()
                break
            except Exception as exc:
                if self.options.get('lock_fail', None):
                    raise RuntimeError("Lock file present, cannot continue")
                self.log.info(
                    "Couldn't get lock. Will retry in 5 seconds... (%s)",
                    str(exc))
                time.sleep(5)

        configs = self.get_default_configs()
        if self.options.get('config', None):
            configs.append(self.options['config'])
        self.core.load_configs(configs)
        self.__add_user_options()
        self.core.load_plugins()

        if self.options.get('ignore_lock', None):
            self.core.set_option(self.core.SECTION, self.IGNORE_LOCKS, "1")

    def perform_test(self):
        """ Run the test and wait for finish """
        self.log.info("Performing test...")
        retcode = 1
        try:
            self.core.plugins_configure()
            self.core.plugins_prepare_test()
            if self.options.get('manual_start', None):
                self.log.info(
                    "Manual start option specified, waiting for user actions")
                raw_input("Press Enter key to start test")

            self.core.plugins_start_test()
            retcode = self.core.wait_for_finish()
            retcode = self.core.plugins_end_test(retcode)
            retcode = self.core.plugins_post_process(retcode)
        except KeyboardInterrupt as ex:
            self.log.info(
                "Do not press Ctrl+C again, the test will be broken otherwise")
            self.log.debug(
                "Caught KeyboardInterrupt: %s", traceback.format_exc(ex))
            try:
                retcode = self.__graceful_shutdown()
            except KeyboardInterrupt as ex:
                self.log.debug(
                    "Caught KeyboardInterrupt again: %s",
                    traceback.format_exc(ex))
                self.log.info(
                    "User insists on exiting, aborting graceful shutdown...")
                retcode = 1

        except Exception as ex:
            self.log.info("Exception: %s", traceback.format_exc(ex))
            self.log.error("%s", ex)
            retcode = self.__graceful_shutdown()
            self.core.release_lock()

        self.log.info("Done performing test with code %s", retcode)
        return retcode

    def get_default_configs(self):
        """ returns default configs list, from /etc, home dir and package_data"""
        # initialize basic defaults
        configs = [resource_filename(__name__, 'config/00-base.ini')]
        try:
            conf_files = sorted(os.listdir(self.baseconfigs_location))
            for filename in conf_files:
                if fnmatch.fnmatch(filename, '*.ini'):
                    configs += [
                        os.path.realpath(
                            self.baseconfigs_location + os.sep + filename)
                    ]
        except OSError:
            self.log.warn(
                self.baseconfigs_location + ' is not accessible to get configs list')

        configs += [os.path.expanduser('~/.yandex-tank')]
        return configs

    def __graceful_shutdown(self):
        """ call shutdown routines """
        retcode = 1
        self.log.info("Trying to shutdown gracefully...")
        retcode = self.core.plugins_end_test(retcode)
        retcode = self.core.plugins_post_process(retcode)
        self.log.info("Done graceful shutdown")
        return retcode

class SingleLevelFilter(logging.Filter):
    """Exclude or approve one msg type at a time.    """

    def __init__(self, passlevel, reject):
        logging.Filter.__init__(self)
        self.passlevel = passlevel
        self.reject = reject

    def filter(self, record):
        if self.reject:
            return record.levelno != self.passlevel
        else:
            return record.levelno == self.passlevel


""" Core module to calculate aggregate data """
import json
import logging
import queue as q

from pkg_resources import resource_string

from .aggregator import Aggregator, DataPoller
from .chopper import TimeChopper
from yandextank.common.interfaces import AggregateResultListener, StatsReader

from netort.data_processing import Drain, Chopper, get_nowait_from_queue

logger = logging.getLogger(__name__)

class LoggingListener(AggregateResultListener):
    """ Log aggregated results """

    def on_aggregated_data(self, data, stats):
        logger.info("Got aggregated sample:\n%s", json.dumps(data, indent=2))
        logger.info("Stats:\n%s", json.dumps(stats, indent=2))

class TankAggregator(object):
    """
    Plugin that manages aggregation and stats collection
    """

    SECTION = 'aggregator'

    @staticmethod
    def get_key():
        return __file__

    def __init__(self, generator):
        # AbstractPlugin.__init__(self, core, cfg)
        """

        :type generator: GeneratorPlugin
        """
        self.generator = generator
        self.listeners = []  # [LoggingListener()]
        self.results = q.Queue()
        self.stats_results = q.Queue()
        self.data_cache = {}
        self.stat_cache = {}
        self.reader = None
        self.stats_reader = None
        self.drain = None
        self.stats_drain = None

    @staticmethod
    def load_config():
        return json.loads(resource_string(__name__, 'config/phout.json').decode('utf8'))

    def start_test(self, poll_period=1):
        self.reader = self.generator.get_reader()
        self.stats_reader = self.generator.get_stats_reader()
        aggregator_config = self.load_config()
        verbose_histogram = True
        if verbose_histogram:
            logger.info("using verbose histogram")
        if self.reader and self.stats_reader:
            pipeline = Aggregator(
                TimeChopper(
                    DataPoller(source=self.reader, poll_period=poll_period),
                    cache_size=3),
                aggregator_config,
                verbose_histogram)
            self.drain = Drain(pipeline, self.results)
            self.drain.start()
            self.stats_drain = Drain(
                Chopper(DataPoller(
                    source=self.stats_reader, poll_period=poll_period)),
                self.stats_results)
            self.stats_drain.start()
        else:
            logger.warning("Generator not found. Generator must provide a reader and a stats_reader interface")

    def _collect_data(self, end=False):
        """
        Collect data, cache it and send to listeners
        """
        data = get_nowait_from_queue(self.results)
        stats = get_nowait_from_queue(self.stats_results)
        logger.debug("Data timestamps: %s" % [d.get('ts') for d in data])
        logger.debug("Stats timestamps: %s" % [d.get('ts') for d in stats])
        for item in data:
            ts = item['ts']
            if ts in self.stat_cache:
                # send items
                data_item = item
                stat_item = self.stat_cache.pop(ts)
                self.__notify_listeners(data_item, stat_item)
            else:
                self.data_cache[ts] = item
        for item in stats:
            ts = item['ts']
            if ts in self.data_cache:
                # send items
                data_item = self.data_cache.pop(ts)
                stat_item = item
                self.__notify_listeners(data_item, stat_item)
            else:
                self.stat_cache[ts] = item
        if end and len(self.data_cache) > 0:
            logger.info('Timestamps without stats:')
            for ts, data_item in sorted(self.data_cache.items(), key=lambda i: i[0]):
                logger.info(ts)
                self.__notify_listeners(data_item, StatsReader.stats_item(ts, 0, 0))

    def is_aggr_finished(self):
        return self.drain._finished.is_set() and self.stats_drain._finished.is_set()

    def is_test_finished(self):
        self._collect_data()
        return -1

    def end_test(self, retcode):
        retcode = self.generator.end_test(retcode)
        if self.stats_reader:
            logger.debug('Closing stats reader')
            self.stats_reader.close()
        if self.drain:
            logger.debug('Waiting for gun drain to finish')
            self.drain.join()
            logger.debug('Waiting for stats drain to finish')
            self.stats_drain.join()
        logger.debug('Collecting remaining data')
        self._collect_data(end=True)
        return retcode

    def add_result_listener(self, listener):
        self.listeners.append(listener)

    def __notify_listeners(self, data, stats):
        """ notify all listeners about aggregate data and stats """
        for listener in self.listeners:
            listener.on_aggregated_data(data, stats)


from uuid import uuid4
from builtins import int

__test_missile = """\
POST /example/search/hello/help/us?param1=50&param2=0&param3=hello HTTP/1.1\r
Connection: close\r
Host: example.org\r
Content-length: 32\r
\r
param1=50&param2=0&param3=hello
"""

def __mark_by_uri(missile):
    return '_'.join(
        missile.split('\n', 1)[0].split(' ', 2)[1].split('?')[0].split('/'))

class __UriMarker(object):
    '''
    Returns a uri marker function with requested limit

    >>> marker = __UriMarker(2)
    >>> marker(__test_missile)
    '_example_search'
    '''

    def __init__(self, limit):
        self.limit = limit

    def __call__(self, missile):
        return '_'.join(
            missile.split('\n', 1)[0].split(' ', 2)[1].split('?')[0].split('/')[
                0:self.limit + 1])

__markers = {
    'uniq': lambda m: uuid4().hex,
    'uri': __mark_by_uri,
}

class __Enumerator(object):
    def __init__(self, marker):
        self.marker = marker
        self.number = int(0)

    def __call__(self, missile):
        marker = "%s#%d" % (self.marker(missile), self.number)
        self.number += 1
        return marker

def get_marker(marker_type, enum_ammo=False):
    '''
    Returns a marker function of the requested marker_type

    >>> marker = get_marker('uniq')(__test_missile)
    >>> type(marker)
    <type 'str'>
    >>> len(marker)
    32

    >>> get_marker('uri')(__test_missile)
    '_example_search_hello_help_us'

    >>> marker = get_marker('non-existent')(__test_missile)
    Traceback (most recent call last):
      ...
    NotImplementedError: No such marker: "non-existent"

    >>> get_marker('3')(__test_missile)
    '_example_search_hello'

    >>> marker = get_marker('3', True)
    >>> marker(__test_missile)
    '_example_search_hello#0'
    >>> marker(__test_missile)
    '_example_search_hello#1'
    '''
    try:
        limit = int(marker_type)
        if limit:
            marker = __UriMarker(limit)
        else:

            def marker(m):
                return ''
    except ValueError:
        if marker_type in __markers:
            marker = __markers[marker_type]
        else:
            raise NotImplementedError('No such marker: "%s"' % marker_type)

    # todo: fix u'False'
    if enum_ammo:
        marker = __Enumerator(marker)
    return marker


import logging
from netort.data_manager import DataSession

from yandextank.plugins.Phantom.reader import string_to_df_microsec
from ...common.interfaces import AbstractPlugin,\
    MonitoringDataListener

logger = logging.getLogger(__name__)  # pylint: disable=C0103

class Plugin(AbstractPlugin, MonitoringDataListener):
    SECTION = 'neuploader'

    def __init__(self, core, cfg, name):
        super(Plugin, self).__init__(core, cfg, name)
        self._is_telegraf = None
        self.clients_cfg = [{'type': 'luna',
                             'api_address': self.cfg.get('api_address'),
                             'db_name': self.cfg.get('db_name')}]

    def configure(self):
        pass

    def start_test(self):
        try:
            self.reader = self.core.job.generator_plugin.get_reader(parser=string_to_df_microsec)
        except TypeError:
            logger.error('Generator plugin does not support NeUploader')
            self.is_test_finished = lambda: -1
            self.reader = []
        else:
            self.data_session = DataSession({'clients': self.clients_cfg})
            self.add_cleanup(self.cleanup)
            self.data_session.update_job({'name': self.cfg.get('test_name')})
            col_map_aggr = {name: 'metric %s' % name for name in
                            ['interval_real', 'connect_time', 'send_time', 'latency',
                             'receive_time', 'interval_event']}
            self.uploader = get_uploader(self.data_session, col_map_aggr, True)

    def cleanup(self):
        uploader_metainfo = self.map_uploader_tags(self.core.status.get('uploader'))
        self.data_session.update_job(uploader_metainfo)
        self.data_session.close()

    def is_test_finished(self):
        df = next(self.reader)
        if df is not None:
            self.uploader(df)
        return -1

    def monitoring_data(self, data_list):
        pass

    def post_process(self, retcode):
        for chunk in self.reader:
            if chunk is not None:
                self.uploader(chunk)
        return retcode

    @property
    def is_telegraf(self):
        return True

    @staticmethod
    def map_uploader_tags(uploader_tags):
        return dict(
            [
                ('component', uploader_tags.get('component')),
                ('description', uploader_tags.get('job_dsc')),
                ('name', uploader_tags.get('job_name')),
                ('person', uploader_tags.get('person')),
                ('task', uploader_tags.get('task')),
                ('version', uploader_tags.get('version')),
                ('lunapark_jobno', uploader_tags.get('job_no'))
            ] + [
                (k, v) for k, v in uploader_tags.get('meta', {}).items()
            ]
        )

def get_uploader(data_session, column_mapping, overall_only=False):
    """
    :type column_mapping: dict
    :type data_session: DataSession
    """
    overall = {col_name: data_session.new_aggregated_metric(name + ' overall')
               for col_name, name in column_mapping.items()}

    def upload_df(df):
        for col_name, metric in overall.items():
            df['value'] = df[col_name]
            metric.put(df)
    return upload_df


""" Provides classes to run TankCore from console environment """
import fnmatch
import glob
import logging
import os
import shutil
import signal
import stat
import sys
import time
import traceback
from ConfigParser import ConfigParser, MissingSectionHeaderError, NoOptionError, NoSectionError
from threading import Thread, Event

import yaml
from netort.resource import manager as resource_manager
from pkg_resources import resource_filename

from yandextank.validator.validator import ValidationError
from .tankcore import TankCore, LockError, Lock
from ..config_converter.converter import convert_ini, convert_single_option

DEFAULT_CONFIG = 'load.yaml'
logger = logging.getLogger()

class RealConsoleMarkup(object):
    '''
    Took colors from here: https://www.siafoo.net/snippet/88
    '''
    WHITE_ON_BLACK = '\033[37;40m'
    TOTAL_RESET = '\033[0m'
    clear = "\x1b[2J\x1b[H"
    new_line = u"\n"

    YELLOW = '\033[1;33m'
    RED = '\033[1;31m'
    RED_DARK = '\033[31;3m'
    RESET = '\033[1;m'
    CYAN = "\033[1;36m"
    GREEN = "\033[1;32m"
    WHITE = "\033[1;37m"
    MAGENTA = '\033[1;35m'
    BG_MAGENTA = '\033[1;45m'
    BG_GREEN = '\033[1;42m'
    BG_BROWN = '\033[1;43m'
    BG_CYAN = '\033[1;46m'

    def clean_markup(self, orig_str):
        ''' clean markup from string '''
        for val in [
            self.YELLOW, self.RED, self.RESET, self.CYAN, self.BG_MAGENTA,
            self.WHITE, self.BG_GREEN, self.GREEN, self.BG_BROWN,
            self.RED_DARK, self.MAGENTA, self.BG_CYAN
        ]:
            orig_str = orig_str.replace(val, '')
        return orig_str

def signal_handler(sig, frame):
    """ required for non-tty python runs to interrupt """
    raise KeyboardInterrupt()

signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)

def load_cfg(cfg_filename):
    """

    :type cfg_filename: str
    """
    if is_ini(cfg_filename):
        return convert_ini(cfg_filename)
    else:
        with open(cfg_filename) as f:
            return yaml.load(f)

def cfg_folder_loader(path):
    """
    :type path: str
    """
    CFG_WILDCARD = '*.yaml'
    return [load_cfg(filename) for filename in sorted(glob.glob(os.path.join(path, CFG_WILDCARD)))]

def load_core_base_cfg():
    return load_cfg(resource_filename(__name__, 'config/00-base.yaml'))

def load_local_base_cfgs():
    return cfg_folder_loader('/etc/yandex-tank')

def parse_options(options):
    """
    :type options: list of str
    :rtype: list of dict
    """
    if options is None:
        return []
    else:
        return [
            convert_single_option(key.strip(), value.strip())
            for key, value
            in [option.split('=', 1) for option in options]
        ]

def apply_shorthand_options(config, options, default_section='DEFAULT'):
    """

    :type config: ConfigParser
    """
    if not options:
        return config
    for option_str in options:
        key, value = option_str.split('=')
        try:
            section, option = key.split('.')
        except ValueError:
            section = default_section
            option = key
        if not config.has_section(section):
            config.add_section(section)
        config.set(section, option, value)
    return config

def load_ini_cfgs(config_files):
    config_filenames = [resource_manager.resource_filename(config) for config in config_files]
    cfg = ConfigParser()
    cfg.read(config_filenames)

    dotted_options = []
    if cfg.has_section('tank'):
        for option, value in cfg.items('tank'):
            if '.' in option:
                dotted_options += [option + '=' + value]
    else:
        cfg.add_section('tank')
    cfg = apply_shorthand_options(cfg, dotted_options)
    cfg.set('tank', 'pid', str(os.getpid()))
    return cfg

def get_default_configs():
    """ returns default configs list, from /etc and home dir """
    # initialize basic defaults
    configs = [resource_filename(__name__, 'config/00-base.ini')]
    baseconfigs_location = '/etc/yandex-tank'
    try:
        conf_files = sorted(os.listdir(baseconfigs_location))
        for filename in conf_files:
            if fnmatch.fnmatch(filename, '*.ini'):
                configs += [
                    os.path.realpath(
                        baseconfigs_location + os.sep + filename)
                ]
    except OSError:
        logger.info(
            baseconfigs_location + ' is not accessible to get configs list')

    configs += [os.path.expanduser('~/.yandex-tank')]
    return configs

def is_ini(cfg_file):
    if cfg_file.endswith('.yaml') or cfg_file.endswith('.json'):
        return False
    try:
        ConfigParser().read(cfg_file)
        return True
    except MissingSectionHeaderError:
        return False

def get_depr_cfg(config_files, no_rc, cmd_options, depr_options):
    try:
        all_config_files = []

        if not no_rc:
            all_config_files = get_default_configs()

        if not config_files:
            if os.path.exists(os.path.realpath('load.ini')):
                all_config_files += [os.path.realpath('load.ini')]
            elif os.path.exists(os.path.realpath('load.conf')):
                # just for old 'lunapark' compatibility
                conf_file = os.path.realpath('load.conf')
                all_config_files += [conf_file]
        else:
            for config_file in config_files:
                all_config_files.append(config_file)

        cfg_ini = load_ini_cfgs([cfg_file for cfg_file in all_config_files if is_ini(cfg_file)])

        # substitute telegraf config
        def patch_ini_config_with_monitoring(ini_config, mon_section_name):
            """
            :type ini_config: ConfigParser
            """
            CONFIG = 'config'
            telegraf_cfg = ini_config.get(mon_section_name, CONFIG)
            if not telegraf_cfg.startswith('<') and not telegraf_cfg.lower() == 'auto':
                with open(resource_manager.resource_filename(telegraf_cfg), 'rb') as telegraf_cfg_file:
                    config_contents = telegraf_cfg_file.read()
                ini_config.set(mon_section_name, CONFIG, config_contents)
            return ini_config

        try:
            cfg_ini = patch_ini_config_with_monitoring(cfg_ini, 'monitoring')
        except (NoSectionError, NoOptionError):
            try:
                patch_ini_config_with_monitoring(cfg_ini, 'telegraf')
            except (NoOptionError, NoSectionError):
                pass

        for section, key, value in depr_options:
            if not cfg_ini.has_section(section):
                cfg_ini.add_section(section)
            cfg_ini.set(section, key, value)
        return apply_shorthand_options(cfg_ini, cmd_options)
    except Exception as ex:
        sys.stderr.write(RealConsoleMarkup.RED)
        sys.stderr.write(RealConsoleMarkup.RESET)
        sys.stderr.write(RealConsoleMarkup.TOTAL_RESET)
        raise ex

def parse_and_check_patches(patches):
    parsed = [yaml.load(p) for p in patches]
    for patch in parsed:
        if not isinstance(patch, dict):
            raise ValidationError('Config patch "{}" should be a dict'.format(patch))
    return parsed

class Cleanup:
    def __init__(self, tankworker):
        """

        :type tankworker: TankWorker
        """
        self._actions = []
        self.tankworker = tankworker

    def add_action(self, name, fn):
        """

        :type fn: function
        :type name: str
        """
        assert callable(fn)
        self._actions.append((name, fn))

    def __enter__(self):
        self.tankworker.init_folder()
        return self.add_action

    def __exit__(self, exc_type, exc_val, exc_tb):
        msgs = []
        if exc_type:
            msg = 'Exception occurred:\n{}: {}\n{}'.format(exc_type, exc_val, '\n'.join(traceback.format_tb(exc_tb)))
            msgs.append(msg)
            logger.error(msg)
        logger.info('Trying to clean up')
        for name, action in reversed(self._actions):
            try:
                action()
            except Exception:
                msg = 'Exception occurred during cleanup action {}'.format(name)
                msgs.append(msg)
                logger.error(msg, exc_info=True)
        self.tankworker.save_status('\n'.join(msgs))
        self.tankworker.core._collect_artifacts()
        return False  # re-raise exception

class Finish:
    def __init__(self, tankworker):
        """
        :type tankworker: TankWorker
        """
        self.worker = tankworker

    def __enter__(self):
        pass

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.worker.status = Status.TEST_FINISHING
        retcode = self.worker.retcode
        if exc_type:
            logger.error('Test interrupted:\n{}: {}\n{}'.format(exc_type, exc_val, exc_tb))
            retcode = 1
        retcode = self.worker.core.plugins_end_test(retcode)
        self.worker.retcode = retcode
        return True  # swallow exception & proceed to post-processing

class Status():
    TEST_POST_PROCESS = 'POST_PROCESS'
    TEST_INITIATED = 'INITIATED'
    TEST_PREPARING = "PREPARING"
    TEST_NOT_FOUND = "NOT_FOUND"
    TEST_RUNNING = "RUNNING"
    TEST_FINISHING = "FINISHING"
    TEST_FINISHED = "FINISHED"

class TankWorker(Thread):
    SECTION = 'core'
    FINISH_FILENAME = 'finish_status.yaml'

    def __init__(self, configs, cli_options=None, cfg_patches=None, cli_args=None, no_local=False,
                 log_handlers=None, wait_lock=True, files=None, ammo_file=None, api_start=False):
        super(TankWorker, self).__init__()
        self.api_start = api_start
        self.wait_lock = wait_lock
        self.log_handlers = log_handlers if log_handlers is not None else []
        self.files = [] if files is None else files
        self.ammo_file = ammo_file

        self.interrupted = Event()
        self.config_list = self._combine_configs(configs, cli_options, cfg_patches, cli_args, no_local)
        self.core = TankCore(self.config_list, self.interrupted)
        self.status = Status.TEST_INITIATED
        self.test_id = self.core.test_id
        self.retcode = None
        self.msg = ''

    @staticmethod
    def _combine_configs(run_cfgs, cli_options=None, cfg_patches=None, cli_args=None, no_local=False):
        if cli_options is None:
            cli_options = []
        if cfg_patches is None:
            cfg_patches = []
        if cli_args is None:
            cli_args = []
        run_cfgs = run_cfgs if len(run_cfgs) > 0 else [DEFAULT_CONFIG]

        if no_local:
            configs = [load_cfg(cfg) for cfg in run_cfgs] + \
                parse_options(cli_options) + \
                parse_and_check_patches(cfg_patches) + \
                cli_args
        else:
            configs = [load_core_base_cfg()] + \
                load_local_base_cfgs() + \
                [load_cfg(cfg) for cfg in run_cfgs] + \
                parse_options(cli_options) + \
                parse_and_check_patches(cfg_patches) + \
                cli_args
        return configs

    def init_folder(self):
        self.folder = self.core.artifacts_dir
        if self.api_start > 0:
            for f in self.files:
                shutil.move(f, self.folder)
            if self.ammo_file:
                shutil.move(self.ammo_file, self.folder)
            os.chdir(self.folder)

    def run(self):
        with Cleanup(self) as add_cleanup:
            lock = self.get_lock()
            add_cleanup('release lock', lock.release)
            self.status = Status.TEST_PREPARING
            self.init_logging(debug=True)
            logger.info('Created a folder for the test. %s' % self.folder)

            self.core.plugins_configure()
            add_cleanup('plugins cleanup', self.core.plugins_cleanup)
            self.core.plugins_prepare_test()
            with Finish(self):
                self.status = Status.TEST_RUNNING
                self.core.plugins_start_test()
                self.retcode = self.core.wait_for_finish()
            self.status = Status.TEST_POST_PROCESS
            self.retcode = self.core.plugins_post_process(self.retcode)
            self.status = Status.TEST_FINISHED

    def stop(self):
        self.interrupted.set()
        self.core.interrupt()

    def get_status(self):
        return {'status_code': self.status,
                'left_time': None,
                'exit_code': self.retcode if self.status is Status.TEST_FINISHED else None,
                'lunapark_id': self.get_lunapark_jobno(),
                'tank_msg': self.msg,
                'lunapark_url': self.get_lunapark_link()}

    def save_status(self, msg):
        self.msg = msg
        self.status = Status.TEST_FINISHED
        with open(os.path.join(self.folder, self.FINISH_FILENAME), 'w') as f:
            yaml.dump(self.get_status(), f)

    def get_lunapark_jobno(self):
        try:
            return str(self.core.status['uploader']['job_no'])
        except KeyError:
            logger.warning('Job number is not available yet')
            return None

    def get_lunapark_link(self):
        try:
            return str(self.core.status['uploader']['web_link'])
        except KeyError:
            logger.warning('Job number is not available yet')
            return None

    def init_logging(self, debug=False):

        filename = os.path.join(self.core.artifacts_dir, 'tank.log')
        open(filename, 'a').close()
        current_file_mode = os.stat(filename).st_mode
        os.chmod(filename, current_file_mode | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)

        logger.handlers = []
        logger.setLevel(logging.DEBUG if debug else logging.INFO)

        file_handler = logging.FileHandler(filename)
        file_handler.setLevel(logging.DEBUG)
        file_handler.setFormatter(logging.Formatter(
            "%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)d\t%(message)s"))
        file_handler.addFilter(TankapiLogFilter())
        logger.addHandler(file_handler)
        logger.info("Log file created")

        for handler in self.log_handlers:
            logger.addHandler(handler)
            logger.info("Logging handler {} added".format(handler))

    def get_lock(self):
        while not self.interrupted.is_set():
            try:
                lock = Lock(self.test_id, self.folder).acquire(self.core.lock_dir,
                                                               self.core.config.get_option(self.SECTION, 'ignore_lock'))
                self.set_msg('')
                break
            except LockError as e:
                self.set_msg(e.message)
                if not self.wait_lock:
                    raise RuntimeError("Lock file present, cannot continue")
                logger.warning(
                    "Couldn't get lock. Will retry in 5 seconds...")
                time.sleep(5)
        else:
            raise KeyboardInterrupt
        return lock

    def set_msg(self, msg):
        self.msg = msg

class TankapiLogFilter(logging.Filter):
    def filter(self, record):
        return record.name != 'tankapi'

class DevNullOpts:
    def __init__(self):
        pass

    log = "/dev/null"


'''
Utilities: parsers, converters, etc.
'''
import re
import logging
import math
from itertools import islice

from .module_exceptions import StepperConfigurationError

logging.getLogger("requests").setLevel(logging.WARNING)

def take(number, iter):
    return list(islice(iter, 0, number))

def parse_duration(duration):
    '''
    Parse duration string, such as '3h2m3s' into milliseconds

    >>> parse_duration('3h2m3s')
    10923000

    >>> parse_duration('0.3s')
    300

    >>> parse_duration('5')
    5000
    '''
    _re_token = re.compile("([0-9.]+)([dhms]?)")

    def parse_token(time, multiplier):
        multipliers = {
            'd': 86400,
            'h': 3600,
            'm': 60,
            's': 1,
        }
        if multiplier:
            if multiplier in multipliers:
                return int(float(time) * multipliers[multiplier] * 1000)
            else:
                raise StepperConfigurationError(
                    'Failed to parse duration: %s' % duration)
        else:
            return int(float(time) * 1000)

    return sum(parse_token(*token) for token in _re_token.findall(duration))

def solve_quadratic(a, b, c):
    '''
    >>> solve_quadratic(1.0, 2.0, 1.0)
    (-1.0, -1.0)
    '''
    discRoot = math.sqrt((b * b) - 4 * a * c)
    root1 = (-b - discRoot) / (2 * a)
    root2 = (-b + discRoot) / (2 * a)
    return (root1, root2)

def s_to_ms(f_sec):
    return int(f_sec * 1000.0)

def proper_round(n):
    """
    rounds float to closest int
    :rtype: int
    :param n: float
    """
    return int(n) + (n / abs(n)) * int(abs(n - int(n)) >= 0.5) if n != 0 else 0


import queue
import hashlib
import logging
import os
import subprocess
import tempfile
import threading
import time
from shutil import copyfile, rmtree

from ...common.util import SecuredShell

from ..Telegraf.config import AgentConfig
from ..Telegraf.reader import MonitoringReader

logger = logging.getLogger(__name__)
logging.getLogger("paramiko.transport").setLevel(logging.WARNING)

def generate_file_md5(filename, blocksize=2 ** 20):
    m = hashlib.md5()
    with open(filename, "rb") as f:
        while True:
            buf = f.read(blocksize)
            if not buf:
                break
            m.update(buf)
    return m.hexdigest()

class LocalhostClient(object):
    """ localhost client setup """
    AGENT_FILENAME = 'agent.py'

    def __init__(self, config, old_style_configs, kill_old):
        # config
        self.kill_old = '--kill-old' if kill_old else ''
        self.python = config['python']
        self.host = "localhost"
        self.telegraf = config['telegraf']
        self.config = AgentConfig(config, old_style_configs)

        # connection
        self.incoming_queue = queue.Queue()
        self.buffer = ""

        self.workdir = None
        self.reader = MonitoringReader(self.incoming_queue)

        self.path = {
            'AGENT_LOCAL_FOLDER': os.path.join(
                os.path.dirname(__file__),
                'agent'),
            'TELEGRAF_LOCAL_PATH': self.telegraf,
        }

    def install(self):
        self.workdir = tempfile.mkdtemp()
        logger.info("Created temp dir %s", self.workdir)
        agent_config = self.config.create_collector_config(self.workdir)
        startup_config = self.config.create_startup_config()
        customs_script = self.config.create_custom_exec_script()
        try:
            copyfile(
                os.path.join(
                    self.path['AGENT_LOCAL_FOLDER'],
                    self.AGENT_FILENAME),
                os.path.join(
                    self.workdir,
                    self.AGENT_FILENAME))
            copyfile(agent_config, os.path.join(self.workdir, 'agent.cfg'))
            copyfile(startup_config, os.path.join(
                self.workdir,
                'agent_startup.cfg'))
            copyfile(
                customs_script,
                os.path.join(
                    self.workdir,
                    'agent_customs.sh'))
            if not os.path.isfile(self.path['TELEGRAF_LOCAL_PATH']):
                logger.error(
                    'Telegraf binary not found at specified path: %s\n'
                    'You can download telegraf binaries here: https://github.com/influxdata/telegraf\n'
                    'or install debian package: `telegraf`',
                    self.path['TELEGRAF_LOCAL_PATH'])
                return None, None, None
        except Exception:
            logger.error("Failed to copy agent to %s on localhost", self.workdir)
            logger.debug("Failed to copy agent to %s on localhost", self.workdir, exc_info=True)
            return None, None, None
        return agent_config, startup_config, customs_script

    @staticmethod
    def popen(args):
        return subprocess.Popen(
            args,
            bufsize=0,
            preexec_fn=os.setsid,
            close_fds=True,
            shell=False,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            stdin=subprocess.PIPE, )

    def start(self):
        """Start local agent"""
        logger.info('Starting agent on localhost')
        args = self.python.split() + [
            os.path.join(
                self.workdir,
                self.AGENT_FILENAME),
            '--telegraf',
            self.path['TELEGRAF_LOCAL_PATH'],
            '--host',
            self.host]
        if self.kill_old:
            args.append(self.kill_old)
        self.session = self.popen(args)
        self.reader_thread = threading.Thread(target=self.read_buffer)
        self.reader_thread.setDaemon(True)
        return self.session

    def read_buffer(self):
        while self.session:
            try:
                chunk = self.session.stdout.read(4096)
                if chunk:
                    parts = chunk.rsplit('\n', 1)
                    if len(parts) > 1:
                        ready_chunk = self.buffer + parts[0] + '\n'
                        self.buffer = parts[1]
                        self.incoming_queue.put(ready_chunk)
                    else:
                        self.buffer += parts[0]
            except ValueError:
                logger.debug(
                    'this exc most likely raised during interpreter shutdown\n'
                    'otherwise something really nasty happend',
                    exc_info=True)

    def uninstall(self):
        """
        Remove agent's files from remote host
        """
        if self.session:
            logger.info('Waiting monitoring data...')
            self.session.terminate()
            self.session.wait()
            self.session = None
        log_filename = "agent_{host}.log".format(host="localhost")
        data_filename = "agent_{host}.rawdata".format(host="localhost")
        try:
            logger.info('Saving monitoring artefacts from localhost')
            copyfile(self.workdir + "/_agent.log", log_filename)
            copyfile(self.workdir + "/monitoring.rawdata", data_filename)
            logger.info('Deleting temp directory: %s', self.workdir)
            rmtree(self.workdir)
        except Exception:
            logger.error("Exception while uninstalling agent", exc_info=True)

        logger.info("Removing agent from: localhost")
        return log_filename, data_filename

class SSHClient(object):
    """remote agent client setup """
    AGENT_FILENAME = 'agent.py'

    def __init__(self, config, old_style_configs, timeout, kill_old):
        # config
        self.kill_old = '--kill-old' if kill_old else ''
        self.host = config['host']
        self.username = config['username']
        self.python = config['python']
        self.port = config['port']
        self.telegraf = config['telegraf']
        self.config = AgentConfig(config, old_style_configs)

        # connection
        self.session = None
        self.ssh = SecuredShell(self.host, self.port, self.username, timeout)
        self.incoming_queue = queue.Queue()
        self.buffer = ""

        self.reader = MonitoringReader(self.incoming_queue)
        handle, cfg_path = tempfile.mkstemp('.cfg', 'agent_')
        os.close(handle)
        self.path = {
            # Destination path on remote host
            'AGENT_REMOTE_FOLDER': '/tmp/',
            # Source path on tank
            'AGENT_LOCAL_FOLDER': os.path.dirname(__file__) + '/agent',
            'TELEGRAF_REMOTE_PATH': '/tmp/telegraf',
            'TELEGRAF_LOCAL_PATH': self.telegraf,
        }
        self.agent_remote_folder = None

    def install(self):
        """Create folder and copy agent and metrics scripts to remote host"""
        logger.info(
            "Installing monitoring agent at %s@%s...",
            self.username,
            self.host)

        # create remote temp dir
        cmd = self.python + ' -c "import tempfile; print tempfile.mkdtemp();"'
        logger.info("Creating temp dir on %s", self.host)
        try:
            out, errors, err_code = self.ssh.execute(cmd)
        except Exception:
            logger.error(
                "Failed to install monitoring agent to %s",
                self.host,
                exc_info=True)
            return None, None, None
        if errors:
            logger.error("[%s] error: '%s'", self.host, errors)
            logger.error("Cancelling agent installation on %s", self.host)
            return None, None, None

        if err_code:
            logger.error(
                "Failed to create remote dir via SSH at %s@%s, code %s: %s" %
                (self.username, self.host, err_code, out.strip()))
            return None, None, None

        remote_dir = out.strip()
        if remote_dir:
            self.path['AGENT_REMOTE_FOLDER'] = remote_dir
            self.agent_remote_folder = remote_dir
        logger.debug(
            "Remote dir at %s:%s", self.host, self.path['AGENT_REMOTE_FOLDER'])

        # create collector config
        agent_config = self.config.create_collector_config(
            self.path['AGENT_REMOTE_FOLDER'])
        startup_config = self.config.create_startup_config()
        customs_script = self.config.create_custom_exec_script()

        # trying to detect os version/architecture and get information about telegraf client
        # DO NOT DELETE indices in string format below. Python 2.6 does not
        # support string formatting without indices
        remote_cmd = 'import os; print os.path.isfile("' + self.path[
            'TELEGRAF_REMOTE_PATH'] + '")'
        cmd = self.python + ' -c \'{cmd}\''.format(cmd=remote_cmd)
        remote_telegraf_exists = "False"
        try:
            out, err, err_code = self.ssh.execute(cmd)
        except Exception:
            logger.error(
                "SSH execute error trying to check telegraf availability on host %s",
                self.host,
                exc_info=True)
        else:
            if err:
                logger.error("[%s] error: '%s'", self.host, errors)
            if out.strip():
                remote_telegraf_exists = out.strip()

        try:
            if remote_telegraf_exists in "True":
                logger.debug('Found telegraf client on %s..', self.host)
            else:
                logger.debug(
                    'Not found telegraf client on %s, trying to install from tank. Copying..',
                    self.host)
                if os.path.isfile(self.path['TELEGRAF_LOCAL_PATH']):
                    self.ssh.send_file(
                        self.path['TELEGRAF_LOCAL_PATH'],
                        self.path['TELEGRAF_REMOTE_PATH'])
                elif os.path.isfile("/usr/bin/telegraf"):
                    self.ssh.send_file(
                        '/usr/bin/telegraf', self.path['TELEGRAF_REMOTE_PATH'])
                else:
                    logger.error(
                        'Telegraf binary not found neither on %s nor on localhost at specified path: %s\n'
                        'You can download telegraf binaries here: https://github.com/influxdata/telegraf\n'
                        'or install debian package: `telegraf`', self.host, self.path['TELEGRAF_LOCAL_PATH'])
                    return None, None, None

            self.ssh.send_file(
                os.path.join(
                    self.path['AGENT_LOCAL_FOLDER'],
                    self.AGENT_FILENAME),
                os.path.join(
                    self.path['AGENT_REMOTE_FOLDER'],
                    self.AGENT_FILENAME))
            self.ssh.send_file(
                agent_config,
                os.path.join(
                    self.path['AGENT_REMOTE_FOLDER'],
                    'agent.cfg'))
            self.ssh.send_file(
                startup_config,
                os.path.join(
                    self.path['AGENT_REMOTE_FOLDER'],
                    'agent_startup.cfg'))
            self.ssh.send_file(
                customs_script,
                os.path.join(
                    self.path['AGENT_REMOTE_FOLDER'],
                    'agent_customs.sh'))

        except Exception:
            logger.error(
                "Failed to install agent on %s", self.host, exc_info=True)
            return None, None, None

        return agent_config, startup_config, customs_script

    def start(self):
        """Start remote agent"""
        logger.info('Starting agent: %s', self.host)
        command = "{python} {agent_path} --telegraf {telegraf_path} --host {host} {kill_old}".format(
            python=self.python,
            agent_path=os.path.join(
                self.path['AGENT_REMOTE_FOLDER'],
                self.AGENT_FILENAME),
            telegraf_path=self.path['TELEGRAF_REMOTE_PATH'],
            host=self.host,
            kill_old=self.kill_old)
        logger.debug('Command to start agent: %s', command)
        self.session = self.ssh.async_session(command)
        self.reader_thread = threading.Thread(target=self.read_buffer)
        self.reader_thread.setDaemon(True)
        return self.session

    def read_buffer(self):
        while self.session:
            chunk = self.session.read_maybe()
            if chunk:
                parts = chunk.rsplit('\n', 1)
                if len(parts) > 1:
                    ready_chunk = self.buffer + parts[0] + '\n'
                    self.buffer = parts[1]
                    self.incoming_queue.put(ready_chunk)
                else:
                    self.buffer += parts[0]
            else:
                time.sleep(1)

    def uninstall(self):
        """
        Remove agent's files from remote host
        """
        log_filename = "agent_{host}.log".format(host=self.host)
        data_filename = "agent_{host}.rawdata".format(host=self.host)

        try:
            if self.session:
                self.session.send("stop\n")
                self.session.close()
                self.session = None
        except BaseException:
            logger.warning(
                'Unable to correctly stop monitoring agent - session is broken. Pay attention to agent log (%s).',
                log_filename,
                exc_info=True)
        else:
            try:
                self.ssh.get_file(
                    os.path.join(
                        self.path['AGENT_REMOTE_FOLDER'],
                        "_agent.log"),
                    log_filename)
                self.ssh.get_file(
                    os.path.join(
                        self.path['AGENT_REMOTE_FOLDER'],
                        "monitoring.rawdata"),
                    data_filename)
                self.ssh.rm_r(self.path['AGENT_REMOTE_FOLDER'])
            except Exception:
                logger.error("Unable to get agent artefacts", exc_info=True)
        self._kill_agent()
        return log_filename, data_filename

    def _kill_agent(self):
        if self.agent_remote_folder:
            cmd = 'pgrep -f {} | xargs kill -9'.format(
                self.agent_remote_folder)
            out, errors, err_code = self.ssh.execute(cmd)
            if errors:
                logger.error(
                    "[%s] error while killing agent: '%s'",
                    self.host,
                    errors)


import logging
import re
from ConfigParser import ConfigParser, ParsingError
from functools import reduce

import pkg_resources
import yaml

from yandextank.common.util import recursive_dict_update
from yandextank.validator.validator import load_plugin_schema, load_yaml_schema

logger = logging.getLogger(__name__)
CORE_SCHEMA = load_yaml_schema(pkg_resources.resource_filename('yandextank.core', 'config/schema.yaml'))['core']['schema']

DEPRECATED_SECTIONS = ['lunaport', 'aggregator']

def old_plugin_mapper(package):
    MAP = {'Overload': 'DataUploader'}
    return MAP.get(package, package)

def parse_package_name(package_path):
    if package_path.startswith("Tank/Plugins/"):
        package = package_path.split('/')[-1].split('.')[0]
    else:
        package = package_path.split('.')[-1].split()[0]
    return old_plugin_mapper(package)

SECTIONS_PATTERNS = {
    'tank': 'core|tank',
    'Aggregator': 'aggregator',
    'Android': 'android',
    'Appium': 'appium',
    'Autostop': 'autostop',
    'BatteryHistorian': 'battery_historian',
    'Bfg': 'bfg|ultimate_gun|http_gun|custom_gun|scenario_gun',
    'Phantom': 'phantom(-.*)?',
    'DataUploader': 'meta|overload|uploader|datauploader',
    'Telegraf': 'telegraf|monitoring',
    'JMeter': 'jmeter',
    'ResourceCheck': 'rcheck',
    'ShellExec': 'shell_?exec',
    'ShootExec': 'shoot_?exec',
    'Console': 'console',
    'TipsAndTricks': 'tips',
    'RCAssert': 'rcassert',
    'JsonReport': 'json_report|jsonreport',
    'Pandora': 'pandora',
    'InfluxUploader': 'influx',

}

class ConversionError(Exception):
    MSG = 'ConversionError:\n{}\nCheck your file format'

    def __init__(self, message=''):
        self.message = self.MSG.format(message)

class OptionsConflict(ConversionError):
    pass

class UnrecognizedSection(ConversionError):
    pass

def guess_plugin(section):
    for plugin, section_name_pattern in SECTIONS_PATTERNS.items():
        if re.match(section_name_pattern, section):
            return plugin
    else:
        raise UnrecognizedSection('Section {} did not match any plugin'.format(section))

def convert_rps_schedule(key, value):
    return {'load_profile': {
        'load_type': 'rps',
        'schedule': value
    }}

def convert_instances_schedule(key, value):
    return {'load_profile': {
        'load_type': 'instances',
        'schedule': value
    }}

def convert_stpd_schedule(key, value):
    return {'load_profile': {
        'load_type': 'stpd_file',
        'schedule': value
    }}

def to_bool(value):
    try:
        return bool(int(value))
    except ValueError:
        return True if 'true' == value.lower() else False

def is_option_deprecated(plugin, option_name):
    DEPRECATED = {
        'Aggregator': [
            'time_periods',
            'precise_cumulative'
        ],
        'DataUploader': [
            'copy_config_to'
        ]
    }
    if option_name in DEPRECATED.get(plugin, []):
        logger.warning('Deprecated option {} in plugin {}, omitting'.format(option_name, plugin))
        return True
    else:
        return False

def check_options(plugin, options):
    CONFLICT_OPTS = {
        'Phantom': [{'rps_schedule', 'instances_schedule', 'stpd_file'}]
    }
    for conflict_options in CONFLICT_OPTS.get(plugin, []):
        intersect = {option[0] for option in options} & conflict_options
        if len(intersect) > 1:
            raise OptionsConflict('Conflicting options: {}: {}'.format(plugin, list(intersect)))
    return plugin, options

def without_deprecated(plugin, options):
    """
    :type options: list of tuple
    """
    return filter(lambda option: not is_option_deprecated(plugin, option[0]), options)

def old_section_name_mapper(name):
    MAP = {
        'monitoring': 'telegraf',
    }
    return MAP.get(name, name)

def rename(name):
    MAP = {
        'meta': 'uploader'
    }
    return MAP.get(name, name)

class Package(object):
    def __init__(self, package_path):
        if package_path.startswith("Tank/Plugins/"):
            self.package = package_path.split('.')[0].replace('Tank/Plugins/', 'yandextank.plugins.')
        else:
            self.package = package_path
        self.plugin_name = old_plugin_mapper(self.package.split('.')[-1])

class UnknownOption(ConversionError):

    def __init__(self, option):
        self.message = 'Unknown option: {}'.format(option)

def empty_to_none(func):
    def new_func(k, v):
        if v in '':
            return {k: None}
        else:
            return func(k, v)
    return new_func

class Option(object):
    TYPE_CASTERS = {
        'boolean': empty_to_none(lambda k, v: {k: to_bool(v)}),
        'integer': empty_to_none(lambda k, v: {k: int(v)}),
        'list': empty_to_none(lambda k, v: {k: [_.strip() for _ in v.strip().split()]}),
        'float': empty_to_none(lambda k, v: {k: float(v)})
    }

    SPECIAL_CONVERTERS = {
        'Phantom': {
            'rps_schedule': convert_rps_schedule,
            'instances_schedule': convert_instances_schedule,
            'stpd_file': convert_stpd_schedule,
            'autocases': TYPE_CASTERS['integer'],
            'headers': lambda key, value: {key: re.compile(r'\[(.*?)\]').findall(value)}
        },
        'Bfg': {
            'rps_schedule': convert_rps_schedule,
            'instances_schedule': convert_instances_schedule,
            'headers': lambda key, value: {key: re.compile(r'\[(.*?)\]').findall(value)}
        },
        'JMeter': {
            'exclude_markers': lambda key, value: {key: value.strip().split(' ')}
        },
        'Pandora': {
            'config_content': lambda key, value: {key: yaml.load(value)}  # works for json as well
        },
        'Autostop': {
            'autostop': lambda k, v: {k: re.findall(r'\w+\(.+?\)', v)}
        },
        'DataUploader': {
            'lock_targets': lambda k, v: {k: v.strip().split() if v != 'auto' else v}
        },
        'core': {
            'ignore_locks': lambda k, v: {'ignore_lock': to_bool(v)}
        }
    }
    CONVERTERS_FOR_UNKNOWN = {
        'DataUploader': lambda k, v: {'meta': {k: v}},
        'JMeter': lambda k, v: {'variables': {k: v}}
    }

    def __init__(self, plugin_name, key, value, schema=None):
        self.dummy_converter = lambda k, v: {k: v}
        self.plugin = plugin_name
        self._schema = schema

        if '.' in key:
            self.name, rest = key.split('.', 1)
            self.value = Option(plugin_name, rest, value, schema=self.schema[self.name]).converted
            self._converter = self.dummy_converter
        else:
            self.name = key
            self.value = value
            self._converter = None
        self._converted = None
        self._as_tuple = None

    @property
    def schema(self):
        if self._schema is None:
            module_paths = {
                'tank': 'yandextank.core'
            }

            def default_path(plugin):
                'yandextank.plugins.{}'.format(plugin)

            self._schema = load_plugin_schema(module_paths.get(self.plugin, default_path(self.plugin)))
        return self._schema

    @property
    def converted(self):
        """
        :rtype: {str: object}
        """
        if self._converted is None:
            self._converted = self.converter(self.name, self.value)
        return self._converted

    @property
    def as_tuple(self):
        """
        :rtype: (str, object)
        """
        if self._as_tuple is None:
            self._as_tuple = self.converted.items()[0]
        return self._as_tuple

    @property
    def converter(self):
        """
        :rtype: callable
        """
        if self._converter is None:
            try:
                self._converter = self.SPECIAL_CONVERTERS[self.plugin][self.name]
            except KeyError:
                try:
                    self._converter = self._get_scheme_converter()
                except UnknownOption:
                    self._converter = self.CONVERTERS_FOR_UNKNOWN.get(self.plugin, self.dummy_converter)
        return self._converter

    def _get_scheme_converter(self):
        if self.name == 'enabled':
            return self.TYPE_CASTERS['boolean']
        if self.schema.get(self.name) is None:
            logger.warning('Unknown option {}:{}'.format(self.plugin, self.name))
            raise UnknownOption('{}:{}'.format(self.plugin, self.name))

        _type = self.schema[self.name].get('type', None)
        if _type is None:
            logger.warning('Option {}:{}: no type specified in schema'.format(self.plugin, self.name))
            return self.dummy_converter

        return self.TYPE_CASTERS.get(_type, self.dummy_converter)

class Section(object):
    def __init__(self, name, plugin, options, enabled=None):
        self.name = old_section_name_mapper(name)
        self.new_name = rename(self.name)
        self.plugin = plugin
        self._schema = None
        self.options = [Option(plugin, *option, schema=self.schema) for option in without_deprecated(*check_options(plugin, options))]
        self.enabled = enabled
        self._merged_options = None

    @property
    def schema(self):
        if self._schema is None:
            self._schema = load_plugin_schema('yandextank.plugins.' + self.plugin)
        return self._schema

    def get_cfg_dict(self, with_meta=True):
        options_dict = self.merged_options
        if with_meta:
            if self.plugin:
                options_dict.update({'package': 'yandextank.plugins.{}'.format(self.plugin)})
            if self.enabled is not None:
                options_dict.update({'enabled': self.enabled})
        return options_dict

    @property
    def merged_options(self):
        if self._merged_options is None:
            self._merged_options = reduce(lambda acc, upd: recursive_dict_update(acc, upd),
                                          [opt.converted for opt in self.options],
                                          {})
        return self._merged_options

    @classmethod
    def from_multiple(cls, sections, parent_name=None, child_name=None, is_list=True):
        """
        :type parent_name: str
        :type sections: list of Section
        """
        if len(sections) == 1:
            return sections[0]
        if parent_name:
            master_section = filter(lambda section: section.name == parent_name, sections)[0]
            rest = filter(lambda section: section.name != parent_name, sections)
        else:
            master_section = sections[0]
            parent_name = master_section.name
            rest = sections[1:]
        child = {'multi': [section.get_cfg_dict(with_meta=False) for section in rest]} if is_list \
            else {child_name: cls._select_one(master_section, rest).get_cfg_dict(with_meta=False)}
        master_section.merged_options.update(child)
        return master_section

    def __repr__(self):
        return '{}/{}'.format(self.name, self.plugin)

    @classmethod
    def _select_one(cls, master_section, rest):
        MAP = {
            'bfg': lambda section: section.name == '{}_gun'.format(master_section.get_cfg_dict()['gun_type'])
        }
        return filter(MAP.get(master_section.name, lambda x: True), rest)[0]
        # return filter(lambda section: section.name == MAP.get(master_section.name, ), rest)[0]

def without_defaults(cfg_ini, section):
    """

    :rtype: (str, str)
    :type cfg_ini: ConfigParser
    """
    defaults = cfg_ini.defaults()
    options = cfg_ini.items(section) if cfg_ini.has_section(section) else []
    return [(key, value) for key, value in options if key not in defaults.keys()]

PLUGIN_PREFIX = 'plugin_'
CORE_SECTION_PATTERN = 'tank|core'
CORE_SECTION_OLD = 'tank'
CORE_SECTION_NEW = 'core'

def parse_sections(cfg_ini):
    """
    :type cfg_ini: ConfigParser
    """
    return [Section(section.lower(),
                    guess_plugin(section.lower()),
                    without_defaults(cfg_ini, section))
            for section in cfg_ini.sections()
            if not re.match(CORE_SECTION_PATTERN, section.lower()) and section.lower() not in DEPRECATED_SECTIONS]

class PluginInstance(object):
    def __init__(self, name, package_and_section):
        self.name = name
        self.enabled = len(package_and_section) > 0
        try:
            package_path, self.section_name = package_and_section.split()
            self.package = Package(package_path)
        except ValueError:
            self.package = Package(package_and_section)
            self.section_name = self._guess_section_name()
        self.plugin_name = self.package.plugin_name

    def __repr__(self):
        return self.name

    def _guess_section_name(self):
        package_map = {
            'Aggregator': 'aggregator',
            'Autostop': 'autostop',
            'BatteryHistorian': 'battery_historian',
            'Bfg': 'bfg',
            'Console': 'console',
            'DataUploader': 'meta',
            'JMeter': 'jmeter',
            'JsonReport': 'json_report',
            'Maven': 'maven',
            'Monitoring': 'monitoring',
            'Pandora': 'pandora',
            'Phantom': 'phantom',
            'RCAssert': 'rcassert',
            'ResourceCheck': 'rcheck',
            'ShellExec': 'shellexec',
            'ShootExec': 'shootexec',
            'SvgReport': 'svgreport',
            'Telegraf': 'telegraf',
            'TipsAndTricks': 'tips'
        }
        name_map = {
            'aggregate': 'aggregator',
            'overload': 'overload',
            'jsonreport': 'json_report'
        }
        return name_map.get(self.name, package_map.get(self.package.plugin_name, self.name))

def enable_sections(sections, core_opts):
    """

    :type sections: list of Section
    """
    DEPRECATED_PLUGINS = ['yandextank.plugins.Aggregator', 'Tank/Plugins/Aggregator.py']

    plugin_instances = [PluginInstance(key.split('_')[1], value) for key, value in core_opts if
                        key.startswith(PLUGIN_PREFIX) and value not in DEPRECATED_PLUGINS]
    enabled_instances = {instance.section_name: instance for instance in plugin_instances if instance.enabled}
    disabled_instances = {instance.section_name: instance for instance in plugin_instances if not instance.enabled}

    for section in sections:
        if section.name in enabled_instances.keys():
            section.enabled = True
            enabled_instances.pop(section.name)
        elif section.name in disabled_instances.keys():
            section.enabled = False
            disabled_instances.pop(section.name)
    # add leftovers
    for plugin_instance in [i for i in plugin_instances if
                            i.section_name in enabled_instances.keys() + disabled_instances.keys()]:
        sections.append(Section(plugin_instance.section_name, plugin_instance.plugin_name, [], plugin_instance.enabled))
    return sections

def partition(l, predicate):
    return reduce(lambda x, y: (x[0] + [y], x[1]) if predicate(y) else (x[0], x[1] + [y]), l, ([], []))

def combine_sections(sections):
    """
    :type sections: list of Section
    :rtype: list of Section
    """
    PLUGINS_TO_COMBINE = {
        'Phantom': ('phantom', 'multi', True),
        'Bfg': ('bfg', 'gun_config', False)
    }
    plugins = {}
    ready_sections = []
    for section in sections:
        if section.plugin in PLUGINS_TO_COMBINE.keys():
            try:
                plugins[section.plugin].append(section)
            except KeyError:
                plugins[section.plugin] = [section]
        else:
            ready_sections.append(section)

    for plugin_name, _sections in plugins.items():
        if isinstance(_sections, list):
            parent_name, child_name, is_list = PLUGINS_TO_COMBINE[plugin_name]
            ready_sections.append(Section.from_multiple(_sections, parent_name, child_name, is_list))
    return ready_sections

def core_options(cfg_ini):
    return cfg_ini.items(CORE_SECTION_OLD) if cfg_ini.has_section(CORE_SECTION_OLD) else []

def convert_ini(ini_file):
    cfg_ini = ConfigParser()
    try:
        if isinstance(ini_file, str):
            cfg_ini.read(ini_file)
        else:
            cfg_ini.readfp(ini_file)
    except ParsingError as e:
        raise ConversionError(e.message)

    ready_sections = enable_sections(combine_sections(parse_sections(cfg_ini)), core_options(cfg_ini))

    plugins_cfg_dict = {section.new_name: section.get_cfg_dict() for section in ready_sections}

    plugins_cfg_dict.update({
        'core': dict([Option('core', key, value, CORE_SCHEMA).as_tuple
                      for key, value in without_defaults(cfg_ini, CORE_SECTION_OLD)
                      if not key.startswith(PLUGIN_PREFIX)])
    })
    logger.info('Converted config:\n{}'.format(yaml.dump(plugins_cfg_dict)))
    return plugins_cfg_dict

def convert_single_option(key, value):
    """

    :type value: str
    :type key: str
    :rtype: {str: obj}
    """
    section_name, option_name = key.strip().split('.', 1)
    if not re.match(CORE_SECTION_PATTERN, section_name):
        section = Section(section_name,
                          guess_plugin(section_name),
                          [(option_name, value)])
        return {section.new_name: section.get_cfg_dict()}
    else:
        if option_name.startswith(PLUGIN_PREFIX):
            return {section.new_name: section.get_cfg_dict() for section in enable_sections([], [(option_name, value)])}
        else:
            return {'core': Option('core', option_name, value, CORE_SCHEMA).converted}