# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import calendar
import collections
from datetime import datetime, timedelta
from warnings import warn

import six
import regex as re
from dateutil.relativedelta import relativedelta

from dateparser.date_parser import date_parser
from dateparser.freshness_date_parser import freshness_date_parser
from dateparser.languages.loader import LocaleDataLoader
from dateparser.conf import apply_settings
from dateparser.timezone_parser import pop_tz_offset_from_string
from dateparser.utils import apply_timezone_from_settings

try:
    # Python 3
    from collections.abc import Set
except ImportError:
    # Python 2.7
    from collections import Set

APOSTROPHE_LOOK_ALIKE_CHARS = [
    u'\N{RIGHT SINGLE QUOTATION MARK}',     # u'\u2019'
    u'\N{MODIFIER LETTER APOSTROPHE}',      # u'\u02bc'
    u'\N{MODIFIER LETTER TURNED COMMA}',    # u'\u02bb'
    u'\N{ARMENIAN APOSTROPHE}',             # u'\u055a'
    u'\N{LATIN SMALL LETTER SALTILLO}',     # u'\ua78c'
    u'\N{PRIME}',                           # u'\u2032'
    u'\N{REVERSED PRIME}',                  # u'\u2035'
    u'\N{MODIFIER LETTER PRIME}',           # u'\u02b9'
    u'\N{FULLWIDTH APOSTROPHE}',            # u'\uff07'
]

RE_NBSP = re.compile(u'\xa0', flags=re.UNICODE)
RE_SPACES = re.compile(r'\s+')
RE_TRIM_SPACES = re.compile(r'^\s+(\S.*?)\s+$')

RE_SANITIZE_SKIP = re.compile(r'\t|\n|\r|\u00bb|,\s\u0432|\u200e|\xb7|\u200f|\u064e|\u064f', flags=re.M)
RE_SANITIZE_RUSSIAN = re.compile(r'([\W\d])\u0433\.', flags=re.I | re.U)
RE_SANITIZE_PERIOD = re.compile(r'(?<=\D+)\.', flags=re.U)
RE_SANITIZE_ON = re.compile(r'^.*?on:\s+(.*)')
RE_SANITIZE_APOSTROPHE = re.compile(u'|'.join(APOSTROPHE_LOOK_ALIKE_CHARS))

RE_SEARCH_TIMESTAMP = re.compile(r'^\d{10}(?![^\d.])')

def sanitize_spaces(date_string):
    date_string = RE_NBSP.sub(' ', date_string)
    date_string = RE_SPACES.sub(' ', date_string)
    date_string = RE_TRIM_SPACES.sub(r'\1', date_string)
    return date_string

def date_range(begin, end, **kwargs):
    dateutil_error_prone_args = ['year', 'month', 'week', 'day', 'hour',
                                 'minute', 'second']
    for arg in dateutil_error_prone_args:
        if arg in kwargs:
            raise ValueError("Invalid argument: %s" % arg)

    step = relativedelta(**kwargs) if kwargs else relativedelta(days=1)

    date = begin
    while date < end:
        yield date
        date += step

    # handles edge-case when iterating months and last interval is < 30 days
    if kwargs.get('months', 0) > 0 and (date.year, date.month) == (end.year, end.month):
        yield end

def get_intersecting_periods(low, high, period='day'):
    if period not in ['year', 'month', 'week', 'day', 'hour', 'minute', 'second', 'microsecond']:
        raise ValueError("Invalid period: {}".format(period))

    if high <= low:
        return

    step = relativedelta(**{period + 's': 1})

    current_period_start = low
    if isinstance(current_period_start, datetime):
        reset_arguments = {}
        for test_period in ['microsecond', 'second', 'minute', 'hour']:
            if test_period == period:
                break
            else:
                reset_arguments[test_period] = 0
        current_period_start = current_period_start.replace(**reset_arguments)

    if period == 'week':
        current_period_start \
            = current_period_start - timedelta(days=current_period_start.weekday())
    elif period == 'month':
        current_period_start = current_period_start.replace(day=1)
    elif period == 'year':
        current_period_start = current_period_start.replace(month=1, day=1)

    while current_period_start < high:
        yield current_period_start
        current_period_start += step

def sanitize_date(date_string):
    date_string = RE_SANITIZE_SKIP.sub(' ', date_string)
    date_string = RE_SANITIZE_RUSSIAN.sub(r'\1 ', date_string)  # remove u'г.' (Russian for year) but not in words
    date_string = sanitize_spaces(date_string)
    date_string = RE_SANITIZE_PERIOD.sub('', date_string)
    date_string = RE_SANITIZE_ON.sub(r'\1', date_string)

    date_string = RE_SANITIZE_APOSTROPHE.sub(u"'", date_string)

    return date_string

def get_date_from_timestamp(date_string, settings):
    if RE_SEARCH_TIMESTAMP.search(date_string):
        date_obj = datetime.fromtimestamp(int(date_string[:10]))
        date_obj = apply_timezone_from_settings(date_obj, settings)
        return date_obj

def get_last_day_of_month(year, month):
    return calendar.monthrange(year, month)[1]

def parse_with_formats(date_string, date_formats, settings):
    """ Parse with formats and return a dictionary with 'period' and 'obj_date'.

    :returns: :class:`datetime.datetime`, dict or None

    """
    period = 'day'
    for date_format in date_formats:
        try:
            date_obj = datetime.strptime(date_string, date_format)
        except ValueError:
            continue
        else:
            # If format does not include the day, use last day of the month
            # instead of first, because the first is usually out of range.
            if '%d' not in date_format:
                period = 'month'
                date_obj = date_obj.replace(
                    day=get_last_day_of_month(date_obj.year, date_obj.month))

            if not ('%y' in date_format or '%Y' in date_format):
                today = datetime.today()
                date_obj = date_obj.replace(year=today.year)

            date_obj = apply_timezone_from_settings(date_obj, settings)

            return {'date_obj': date_obj, 'period': period}
    else:
        return {'date_obj': None, 'period': period}

class _DateLocaleParser(object):
    DATE_FORMATS_ERROR_MESSAGE = "Date formats should be list, tuple or set of strings"

    def __init__(self, locale, date_string, date_formats, settings=None):
        self._settings = settings
        if isinstance(date_formats, six.string_types):
            warn(self.DATE_FORMATS_ERROR_MESSAGE, FutureWarning)
            date_formats = [date_formats]
        elif not (date_formats is None or isinstance(date_formats, (list, tuple, Set))):
            raise TypeError(self.DATE_FORMATS_ERROR_MESSAGE)

        self.locale = locale
        self.date_string = date_string
        self.date_formats = date_formats
        self._translated_date = None
        self._translated_date_with_formatting = None

    @classmethod
    def parse(cls, locale, date_string, date_formats=None, settings=None):
        instance = cls(locale, date_string, date_formats, settings)
        return instance._parse()

    def _parse(self):
        for parser in (
            self._try_timestamp,
            self._try_freshness_parser,
            self._try_given_formats,
            self._try_parser,
            self._try_hardcoded_formats,
        ):
            date_obj = parser()
            if self._is_valid_date_obj(date_obj):
                return date_obj
        else:
            return None

    def _try_timestamp(self):
        return {
            'date_obj': get_date_from_timestamp(self.date_string, self._settings),
            'period': 'day',
        }

    def _try_freshness_parser(self):
        return freshness_date_parser.get_date_data(self._get_translated_date(), self._settings)

    def _try_parser(self):
        _order = self._settings.DATE_ORDER
        try:
            if self._settings.PREFER_LOCALE_DATE_ORDER:
                if self._settings._default:
                    self._settings.DATE_ORDER = self.locale.info.get('date_order', _order)
            date_obj, period = date_parser.parse(
                self._get_translated_date(), settings=self._settings)
            self._settings.DATE_ORDER = _order
            return {
                'date_obj': date_obj,
                'period': period,
            }
        except ValueError:
            self._settings.DATE_ORDER = _order
            return None

    def _try_given_formats(self):
        if not self.date_formats:
            return

        return parse_with_formats(
            self._get_translated_date_with_formatting(),
            self.date_formats, settings=self._settings
        )

    def _try_hardcoded_formats(self):
        hardcoded_date_formats = [
            '%B %d, %Y, %I:%M:%S %p',
            '%b %d, %Y at %I:%M %p',
            '%d %B %Y %H:%M:%S',
            '%A, %B %d, %Y',
            '%Y-%m-%dT%H:%M:%S.%fZ'
        ]
        try:
            return parse_with_formats(
                self._get_translated_date_with_formatting(),
                hardcoded_date_formats,
                settings=self._settings
            )
        except TypeError:
            return None

    def _get_translated_date(self):
        if self._translated_date is None:
            self._translated_date = self.locale.translate(
                self.date_string, keep_formatting=False, settings=self._settings)
        return self._translated_date

    def _get_translated_date_with_formatting(self):
        if self._translated_date_with_formatting is None:
            self._translated_date_with_formatting = self.locale.translate(
                self.date_string, keep_formatting=True, settings=self._settings)
        return self._translated_date_with_formatting

    def _is_valid_date_obj(self, date_obj):
        if not isinstance(date_obj, dict):
            return False
        if len(date_obj) != 2:
            return False
        if 'date_obj' not in date_obj or 'period' not in date_obj:
            return False
        if not date_obj['date_obj']:
            return False
        if date_obj['period'] not in ('day', 'week', 'month', 'year'):
            return False

        return True

class DateDataParser(object):
    """
    Class which handles language detection, translation and subsequent generic parsing of
    string representing date and/or time.

    :param languages:
        A list of language codes, e.g. ['en', 'es', 'zh-Hant'].
        If locales are not given, languages and region are
        used to construct locales for translation.
    :type languages: list

    :param locales:
        A list of locale codes, e.g. ['fr-PF', 'qu-EC', 'af-NA'].
        The parser uses locales to translate date string.
    :type locales: list

    :param region:
        A region code, e.g. 'IN', '001', 'NE'.
        If locales are not given, languages and region are
        used to construct locales for translation.
    :type region: str|unicode

    :param try_previous_locales:
        If True, locales previously used to translate date are tried first.
    :type allow_redetect_language: bool

    :param use_given_order:
        If True, locales are tried for translation of date string
        in the order in which they are given.
    :type allow_redetect_language: bool

    :param settings:
        Configure customized behavior using settings defined in :mod:`dateparser.conf.Settings`.
    :type settings: dict

    :return: A parser instance

    :raises:
        ValueError - Unknown Language, TypeError - Languages argument must be a list
    """

    locale_loader = None

    @apply_settings
    def __init__(self, languages=None, locales=None, region=None, try_previous_locales=True,
                 use_given_order=False, settings=None):

        if not isinstance(languages, (list, tuple, Set)) and languages is not None:
            raise TypeError("languages argument must be a list (%r given)" % type(languages))

        if not isinstance(locales, (list, tuple, Set)) and locales is not None:
            raise TypeError("locales argument must be a list (%r given)" % type(locales))

        if not isinstance(region, six.string_types) and region is not None:
            raise TypeError("region argument must be str or unicode (%r given)" % type(region))

        if not isinstance(try_previous_locales, bool):
            raise TypeError("try_previous_locales argument must be a boolean (%r given)"
                            % type(try_previous_locales))

        if not isinstance(use_given_order, bool):
            raise TypeError("use_given_order argument must be a boolean (%r given)"
                            % type(use_given_order))

        if not locales and use_given_order:
            raise ValueError("locales must be given if use_given_order is True")

        self._settings = settings
        self.try_previous_locales = try_previous_locales
        self.use_given_order = use_given_order
        self.languages = languages
        self.locales = locales
        self.region = region
        self.previous_locales = []

    def get_date_data(self, date_string, date_formats=None):
        """
        Parse string representing date and/or time in recognizable localized formats.
        Supports parsing multiple languages and timezones.

        :param date_string:
            A string representing date and/or time in a recognizably valid format.
        :type date_string: str|unicode
        :param date_formats:
            A list of format strings using directives as given
            `here <https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_.
            The parser applies formats one by one, taking into account the detected languages.
        :type date_formats: list

        :return: a dict mapping keys to :mod:`datetime.datetime` object and *period*. For example:
            {'date_obj': datetime.datetime(2015, 6, 1, 0, 0), 'period': u'day'}

        :raises: ValueError - Unknown Language

        .. note:: *Period* values can be a 'day' (default), 'week', 'month', 'year'.

        *Period* represents the granularity of date parsed from the given string.

        In the example below, since no day information is present, the day is assumed to be current
        day ``16`` from *current date* (which is June 16, 2015, at the moment of writing this).
        Hence, the level of precision is ``month``:

            >>> DateDataParser().get_date_data(u'March 2015')
            {'date_obj': datetime.datetime(2015, 3, 16, 0, 0), 'period': u'month'}

        Similarly, for date strings with no day and month information present, level of precision
        is ``year`` and day ``16`` and month ``6`` are from *current_date*.

            >>> DateDataParser().get_date_data(u'2014')
            {'date_obj': datetime.datetime(2014, 6, 16, 0, 0), 'period': u'year'}

        Dates with time zone indications or UTC offsets are returned in UTC time unless
        specified using `Settings`_.

            >>> DateDataParser().get_date_data(u'23 March 2000, 1:21 PM CET')
            {'date_obj': datetime.datetime(2000, 3, 23, 14, 21), 'period': 'day'}

        """
        if not(isinstance(date_string, six.text_type) or isinstance(date_string, six.string_types)):
            raise TypeError('Input type must be str or unicode')

        if isinstance(date_string, bytes):
            date_string = date_string.decode('utf-8')

        res = parse_with_formats(date_string, date_formats or [], self._settings)
        if res['date_obj']:
            return res

        date_string = sanitize_date(date_string)

        for locale in self._get_applicable_locales(date_string):
            parsed_date = _DateLocaleParser.parse(
                locale, date_string, date_formats, settings=self._settings)
            if parsed_date:
                parsed_date['locale'] = locale.shortname
                if self.try_previous_locales:
                    self.previous_locales.insert(0, locale)
                return parsed_date
        else:
            return {'date_obj': None, 'period': 'day', 'locale': None}

    def get_date_tuple(self, *args, **kwargs):
        date_tuple = collections.namedtuple('DateData', 'date_obj period locale')
        date_data = self.get_date_data(*args, **kwargs)
        return date_tuple(**date_data)

    def _get_applicable_locales(self, date_string):
        pop_tz_cache = []

        def date_strings():
            """ A generator instead of a static list to avoid calling
            pop_tz_offset_from_string if the first locale matches on unmodified
            date_string.
            """
            yield date_string
            if not pop_tz_cache:
                stripped_date_string, _ = pop_tz_offset_from_string(
                    date_string, as_offset=False)
                if stripped_date_string == date_string:
                    stripped_date_string = None
                pop_tz_cache[:] = [stripped_date_string]
            stripped_date_string, = pop_tz_cache
            if stripped_date_string is not None:
                yield stripped_date_string

        if self.try_previous_locales:
            for locale in self.previous_locales:
                for s in date_strings():
                    if self._is_applicable_locale(locale, s):
                        yield locale

        for locale in self._get_locale_loader().get_locales(
                languages=self.languages, locales=self.locales, region=self.region,
                use_given_order=self.use_given_order):
            for s in date_strings():
                if self._is_applicable_locale(locale, s):
                    yield locale

    def _is_applicable_locale(self, locale, date_string):
        return locale.is_applicable(
            date_string,
            strip_timezone=False,  # it is stripped outside
            settings=self._settings)

    @classmethod
    def _get_locale_loader(cls):
        if not cls.locale_loader:
            cls.locale_loader = LocaleDataLoader()
        return cls.locale_loader


import logging

from netort.resource import manager as resource

from . import info
from . import instance_plan as ip
from . import load_plan as lp
from . import missile
from .mark import get_marker
from .module_exceptions import StepperConfigurationError, AmmoFileError

class ComponentFactory():
    def __init__(
            self,
            rps_schedule=None,
            http_ver='1.1',
            ammo_file=None,
            instances_schedule=None,
            instances=1000,
            loop_limit=-1,
            ammo_limit=-1,
            uris=None,
            headers=None,
            autocases=None,
            enum_ammo=False,
            ammo_type='phantom',
            chosen_cases=None,
            use_cache=True):
        self.log = logging.getLogger(__name__)
        self.ammo_file = ammo_file
        self.ammo_type = ammo_type
        self.rps_schedule = rps_schedule
        self.http_ver = http_ver
        self.instances_schedule = instances_schedule
        loop_limit = int(loop_limit)
        if loop_limit == -1:  # -1 means infinite
            loop_limit = None
        ammo_limit = int(ammo_limit)
        if ammo_limit == -1:  # -1 means infinite
            ammo_limit = None
        if loop_limit is None and ammo_limit is None and not rps_schedule:
            # we should have only one loop if we have instance_schedule
            loop_limit = 1
        info.status.loop_limit = loop_limit
        info.status.ammo_limit = ammo_limit
        info.status.publish("instances", instances)
        self.uris = uris
        if self.uris and loop_limit:
            info.status.ammo_limit = len(self.uris) * loop_limit
        self.headers = headers
        self.marker = get_marker(autocases, enum_ammo)
        self.chosen_cases = chosen_cases or []
        self.use_cache = use_cache

    def get_load_plan(self):
        """
        return load plan (timestamps generator)
        """
        if self.rps_schedule and self.instances_schedule:
            raise StepperConfigurationError(
                'Both rps and instances schedules specified. You must specify only one of them'
            )
        elif self.rps_schedule:
            info.status.publish('loadscheme', self.rps_schedule)
            return lp.create(self.rps_schedule)
        elif self.instances_schedule:
            info.status.publish('loadscheme', self.instances_schedule)
            return ip.create(self.instances_schedule)
        else:
            self.instances_schedule = []
            info.status.publish('loadscheme', self.instances_schedule)
            return ip.create(self.instances_schedule)

    def get_ammo_generator(self):
        """
        return ammo generator
        """
        af_readers = {
            'phantom': missile.AmmoFileReader,
            'slowlog': missile.SlowLogReader,
            'line': missile.LineReader,
            'uri': missile.UriReader,
            'uripost': missile.UriPostReader,
            'access': missile.AccessLogReader,
            'caseline': missile.CaseLineReader,
        }
        if self.uris and self.ammo_file:
            raise StepperConfigurationError(
                'Both uris and ammo file specified. You must specify only one of them'
            )
        elif self.uris:
            ammo_gen = missile.UriStyleGenerator(
                self.uris, self.headers, http_ver=self.http_ver)
        elif self.ammo_file:
            if self.ammo_type in af_readers:
                if self.ammo_type == 'phantom':
                    opener = resource.get_opener(self.ammo_file)
                    with opener(self.use_cache) as ammo:
                        try:
                            if not ammo.next()[0].isdigit():
                                self.ammo_type = 'uri'
                                self.log.info(
                                    "Setting ammo_type 'uri' because ammo is not started with digit and you did not specify ammo format"
                                )
                            else:
                                self.log.info(
                                    "Default ammo type ('phantom') used, use 'phantom.ammo_type' option to override it"
                                )
                        except StopIteration:
                            self.log.exception(
                                "Couldn't read first line of ammo file")
                            raise AmmoFileError(
                                "Couldn't read first line of ammo file")
            else:
                raise NotImplementedError(
                    'No such ammo type implemented: "%s"' % self.ammo_type)
            ammo_gen = af_readers[self.ammo_type](
                self.ammo_file, headers=self.headers, http_ver=self.http_ver, use_cache=self.use_cache)
        else:
            raise StepperConfigurationError(
                'Ammo not found. Specify uris or ammo file')
        self.log.info("Using %s ammo reader" % type(ammo_gen).__name__)
        return ammo_gen

    def get_marker(self):
        return self.marker

    def get_filter(self):
        if len(self.chosen_cases):

            def is_chosen_case(ammo_tuple):
                return ammo_tuple[1] in self.chosen_cases

            return is_chosen_case
        else:
            return lambda ammo_tuple: True


# -*- coding: UTF-8 -*-
import logging
from StringIO import StringIO

import numpy as np
import pandas as pd
import queue as q

from yandextank.aggregator import TimeChopper
from yandextank.aggregator import aggregator as agg

logger = logging.getLogger(__name__)

KNOWN_EXC = {
    "java.net.NoRouteToHostException": 113,
    "java.net.ConnectException": 110,
    "java.net.BindException": 99,
    "java.net.PortUnreachableException": 101,
    "java.net.ProtocolException": 71,
    "java.net.SocketException": 32,
    "java.net.SocketTimeoutException": 110,
    "java.net.UnknownHostException": 14,
    "java.net.URISyntaxException": 22,
    "java.io.FileNotFoundException": 2,
    "java.io.IOException": 5,
    "java.io.EOFException": 104,
    "org.apache.http.conn.ConnectTimeoutException": 110,
    "org.apache.commons.net.MalformedServerReplyException": 71,
    "org.apache.http.NoHttpResponseException": 32,
    "java.io.InterruptedIOException": 32,
    "javax.net.ssl.SSLHandshakeException": 5,
}

def _exc_to_net(param1, success):
    """ translate http code to net code. if accertion failed, set net code to 314 """
    if len(param1) <= 3:
        # FIXME: we're unable to use better logic here, because we should support non-http codes
        # but, we should look for core.util.HTTP or some other common logic
        # here
        if success:
            return 0
        else:
            return 314

    exc = param1.split(' ')[-1]
    if exc in KNOWN_EXC.keys():
        return KNOWN_EXC[exc]
    else:
        logger.warning(
            "Unknown Java exception, consider adding it to dictionary: %s",
            param1)
        return 41

def _exc_to_http(param1):
    """ translate exception str to http code"""
    if len(param1) <= 3:
        try:
            int(param1)
        except BaseException:
            logger.error(
                "JMeter wrote some strange data into codes column: %s", param1)
        else:
            return int(param1)

    exc = param1.split(' ')[-1]
    if exc in KNOWN_EXC.keys():
        return 0
    else:
        logger.warning("Unknown Java exception. %s", param1)
        return 0

exc_to_net = np.vectorize(_exc_to_net)
exc_to_http = np.vectorize(_exc_to_http)

# phout_columns = [
#     'send_ts', 'tag', 'interval_real', 'connect_time', 'send_time', 'latency',
#     'receive_time', 'interval_event', 'size_out', 'size_in', 'net_code',
#     'proto_code'
# ]

jtl_columns = [
    'send_ts', 'interval_real', 'tag', 'retcode', 'success', 'size_in',
    'grpThreads', 'allThreads', 'latency', 'connect_time'
]
jtl_types = {
    'send_ts': np.int64,
    'interval_real': np.int64,
    'tag': np.str,
    'retcode': np.str,
    'success': np.bool,
    'size_in': np.int64,
    'grpThreads': np.int64,
    'allThreads': np.int64,
    'latency': np.int64,
    'connect_time': np.float64,
}

def fix_latency(row):
    if row['latency'] < row['connect_time']:
        if row['interval_real'] < row['connect_time']:
            latency = 0
        else:
            latency = row['interval_real'] - row['connect_time']
    else:
        latency = row['latency'] - row['connect_time']
    return latency

# timeStamp,elapsed,label,responseCode,success,bytes,grpThreads,allThreads,Latency
def string_to_df(data):
    chunk = pd.read_csv(StringIO(data),
                        sep='\t',
                        names=jtl_columns, dtype=jtl_types,
                        keep_default_na=False)
    chunk["receive_ts"] = (chunk["send_ts"] + chunk['interval_real']) / 1000.0
    chunk['receive_sec'] = chunk["receive_ts"].astype(np.int64)
    chunk['interval_real'] = chunk["interval_real"] * 1000  # convert to µs
    chunk.set_index(['receive_sec'], inplace=True)
    chunk_length = len(chunk)
    chunk['connect_time'] = (chunk['connect_time'].fillna(0) * 1000).astype(np.int64)
    chunk['latency'] = chunk['latency'] * 1000
    chunk['latency'] = chunk.apply(fix_latency, axis=1)
    chunk['send_time'] = np.zeros(chunk_length)
    chunk['receive_time'] = chunk['interval_real'] - \
        chunk['latency'] - chunk['connect_time']
    chunk['interval_event'] = np.zeros(chunk_length)
    chunk['size_out'] = np.zeros(chunk_length).astype(int)
    chunk['net_code'] = exc_to_net(chunk['retcode'], chunk['success'])
    chunk['proto_code'] = exc_to_http(chunk['retcode'])
    return chunk

class JMeterStatAggregator(object):
    def __init__(self, source):
        self.worker = agg.Worker({"allThreads": ["max"]}, False)
        self.source = source

    def __iter__(self):
        for ts, chunk, rps in self.source:
            stats = self.worker.aggregate(chunk)
            yield [{
                'ts': ts,
                'metrics': {
                    'instances': stats['allThreads']['max'],
                    'reqps': 0
                }
            }]

    def close(self):
        pass

class JMeterReader(object):
    def __init__(self, filename):
        self.buffer = ""
        self.stat_buffer = ""
        self.jtl_file = filename
        self.jmeter_finished = False
        self.agg_finished = False
        self.closed = False
        self.stat_queue = q.Queue()
        self.stats_reader = JMeterStatAggregator(
            TimeChopper(self._read_stat_queue(), 3))

    def _read_stat_queue(self):
        while not self.closed:
            # for _ in range(self.stat_queue.qsize()):
            try:
                si = self.stat_queue.get_nowait()
                if si is not None:
                    yield si
            except q.Empty:
                pass

    def _read_jtl_chunk(self, jtl):
        data = jtl.read(1024 * 1024 * 10)
        if data:
            parts = data.rsplit('\n', 1)
            if len(parts) > 1:
                ready_chunk = self.buffer + parts[0] + '\n'
                self.buffer = parts[1]
                df = string_to_df(ready_chunk)
                self.stat_queue.put(df)
                return df
            else:
                self.buffer += parts[0]
        else:
            if self.jmeter_finished:
                self.agg_finished = True
            jtl.readline()
        return None

    def __iter__(self):
        with open(self.jtl_file, 'r') as jtl:
            while not self.closed:
                yield self._read_jtl_chunk(jtl)
            yield self._read_jtl_chunk(jtl)

    def close(self):
        self.closed = True


""" Utility classes for phantom module """
# TODO: use separate answ log per benchmark
import copy
import logging
import multiprocessing
import string

from pkg_resources import resource_string
from yandextank.common.util import AddressWizard

from ...stepper import StepperWrapper
from ...stepper.util import parse_duration

logger = logging.getLogger(__name__)

class PhantomConfig:
    """ config file generator """
    OPTION_PHOUT = "phout_file"

    def __init__(self, core, cfg, stat_log):
        self.core = core
        self.cfg = cfg
        self.streams = []

        # common
        self.timeout = 11000
        self.answ_log = None
        self.answ_log_level = None
        self._phout_file = None
        self.stat_log = stat_log
        self.phantom_log = None
        self.phantom_start_time = None
        self.phantom_modules_path = None
        self.threads = None
        self.additional_libs = None
        self.enum_ammo = False
        self._config_file = None

    @staticmethod
    def get_available_options():
        opts = [
            "threads",
            "phantom_modules_path",
            "additional_libs",
            "writelog",
            "enum_ammo",
            "timeout",
        ]
        opts += StreamConfig.get_available_options()
        return opts

    @property
    def phout_file(self):
        if not self._phout_file:
            self._phout_file = self.cfg['phout_file'] or self.core.mkstemp(".log", "phout_")
        return self._phout_file

    def read_config(self):
        """        Read phantom tool specific options        """
        self.threads = self.cfg["threads"] or str(int(multiprocessing.cpu_count() / 2) + 1)
        self.phantom_modules_path = self.cfg["phantom_modules_path"]
        self.additional_libs = ' '.join(self.cfg["additional_libs"])
        self.answ_log_level = self.cfg["writelog"]
        if self.answ_log_level.lower() in ['0', 'false']:
            self.answ_log_level = 'none'
        elif self.answ_log_level.lower() in ['1', 'true']:
            self.answ_log_level = 'all'
        self.timeout = parse_duration(self.cfg["timeout"])
        if self.timeout > 120000:
            logger.warning(
                "You've set timeout over 2 minutes."
                " Are you a functional tester?")
        self.answ_log = self.core.mkstemp(".log", "answ_")
        self.core.add_artifact_file(self.answ_log)
        self.core.add_artifact_file(self.phout_file)
        self.core.add_artifact_file(self.stat_log)
        self.phantom_log = self.core.mkstemp(".log", "phantom_")
        self.core.add_artifact_file(self.phantom_log)

        main_stream = StreamConfig(
            self.core,
            len(self.streams), self.phout_file, self.answ_log,
            self.answ_log_level, self.timeout, self.cfg, True)
        self.streams.append(main_stream)

        for section in self.multi():
            self.streams.append(
                StreamConfig(
                    self.core,
                    len(self.streams), self.phout_file, self.answ_log,
                    self.answ_log_level, self.timeout, section))

        for stream in self.streams:
            stream.read_config()

        if any(stream.ssl for stream in self.streams):
            self.additional_libs += ' ssl io_benchmark_method_stream_transport_ssl'

    def multi(self):
        return (dict(self.cfg, **section) for section in self.cfg['multi'])

    @property
    def config_file(self):
        if self._config_file is None:
            self._config_file = self.compose_config()
        return self._config_file

    def compose_config(self):
        """        Generate phantom tool run config        """
        streams_config = ''
        stat_benchmarks = ''
        for stream in self.streams:
            streams_config += stream.compose_config()
            if not stream.is_main:
                stat_benchmarks += " " + "benchmark_io%s" % stream.sequence_no

        kwargs = {}
        kwargs['threads'] = self.threads
        kwargs['phantom_log'] = self.phantom_log
        kwargs['stat_log'] = self.stat_log
        kwargs['benchmarks_block'] = streams_config
        kwargs['stat_benchmarks'] = stat_benchmarks
        kwargs['additional_libs'] = self.additional_libs
        kwargs['phantom_modules_path'] = self.phantom_modules_path
        filename = self.core.mkstemp(".conf", "phantom_")
        self.core.add_artifact_file(filename)
        logger.debug("Generating phantom config: %s", filename)
        template_str = resource_string(__name__, "config/phantom.conf.tpl")
        tpl = string.Template(template_str)
        config = tpl.substitute(kwargs)

        with open(filename, 'w') as conffile:
            conffile.write(config)
        return filename

    def set_timeout(self, timeout):
        """ pass timeout to all streams """
        for stream in self.streams:
            stream.timeout = timeout

    def get_info(self):
        """ get merged info about phantom conf """
        result = copy.copy(self.streams[0])
        result.stat_log = self.stat_log
        result.steps = []
        result.ammo_file = ''
        result.rps_schedule = None
        result.ammo_count = 0
        result.duration = 0

        result.instances = 0
        result.loadscheme = []
        result.loop_count = 0

        for stream in self.streams:
            sec_no = 0
            logger.debug("Steps: %s", stream.stepper_wrapper.steps)
            for item in stream.stepper_wrapper.steps:
                for x in range(0, item[1]):
                    if len(result.steps) > sec_no:
                        result.steps[sec_no][0] += item[0]
                    else:
                        result.steps.append([item[0], 1])
                    sec_no += 1

            if result.rps_schedule:
                result.rps_schedule = []
            else:
                result.rps_schedule = stream.stepper_wrapper.loadscheme
            if result.loadscheme:
                result.loadscheme = ''
            else:
                # FIXME: add formatted load scheme for server:
                # <step_size,step_type,first_rps,last_rps,original_step_params>
                # as a string
                result.loadscheme = ''

            if result.loop_count:
                result.loop_count = u'0'
            else:
                result.loop_count = stream.stepper_wrapper.loop_count

            result.ammo_file += '{} '.format(stream.stepper_wrapper.ammo_file)
            result.ammo_count += stream.stepper_wrapper.ammo_count
            result.duration = max(
                result.duration, stream.stepper_wrapper.duration)
            result.instances += stream.instances

        if not result.ammo_count:
            raise ValueError("Total ammo count cannot be zero")
        return result

class StreamConfig:
    """ each test stream's config """

    OPTION_INSTANCES_LIMIT = 'instances'

    def __init__(self, core, sequence, phout, answ, answ_level, timeout, cfg, is_main=False):
        self.core = core
        self.cfg = cfg
        self.address_wizard = AddressWizard()

        self.sequence_no = sequence
        self.stepper_wrapper = StepperWrapper(core, cfg)
        self.phout_file = phout
        self.answ_log = answ
        self.answ_log_level = answ_level
        self.timeout = timeout
        self.is_main = is_main

        # per benchmark
        self.instances = self.get_option('instances')
        self.ipv6 = None
        self.ssl = None
        self.address = None
        self.port = None
        self.tank_type = None
        self.stpd = None
        self.gatling = None
        self.phantom_http_line = None
        self.phantom_http_field_num = None
        self.phantom_http_field = None
        self.phantom_http_entity = None
        self.resolved_ip = None
        self.method_prefix = None
        self.source_log_prefix = None
        self.method_options = None
        self.client_cipher_suites = None
        self.client_certificate = None
        self.client_key = None

    def get_option(self, option, default=None):
        """ get option wrapper """
        return self.cfg[option]

    @staticmethod
    def get_available_options():
        opts = [
            "ssl", "tank_type", 'gatling_ip', "method_prefix",
            "source_log_prefix"
        ]
        opts += [
            "phantom_http_line", "phantom_http_field_num", "phantom_http_field",
            "phantom_http_entity"
        ]
        opts += ['address', "port", StreamConfig.OPTION_INSTANCES_LIMIT]
        opts += StepperWrapper.get_available_options()
        opts += ["connection_test"]
        return opts

    def read_config(self):
        """ reads config """
        # multi-options
        self.ssl = self.get_option("ssl")
        self.tank_type = self.get_option("tank_type")
        # TODO: refactor. Maybe we should decide how to interact with
        # StepperWrapper here.
        # self.instances = self.get_option('instances')
        self.gatling = ' '.join(self.get_option('gatling_ip').split("\n"))
        self.method_prefix = self.get_option("method_prefix")
        self.method_options = self.get_option("method_options")
        self.source_log_prefix = self.get_option("source_log_prefix")

        self.phantom_http_line = self.get_option("phantom_http_line")
        self.phantom_http_field_num = self.get_option("phantom_http_field_num")
        self.phantom_http_field = self.get_option("phantom_http_field")
        self.phantom_http_entity = self.get_option("phantom_http_entity")

        self.address = self.get_option('address')
        do_test_connect = self.get_option("connection_test")
        explicit_port = self.get_option('port', '')
        self.ipv6, self.resolved_ip, self.port, self.address = self.address_wizard.resolve(
            self.address, do_test_connect, explicit_port)

        logger.info(
            "Resolved %s into %s:%s", self.address, self.resolved_ip, self.port)

        self.client_cipher_suites = self.get_option("client_cipher_suites", "")
        self.client_certificate = self.get_option("client_certificate", "")
        self.client_key = self.get_option("client_key", "")
        self.stepper_wrapper.read_config()

    def compose_config(self):
        """ compose benchmark block """
        # step file
        self.stepper_wrapper.prepare_stepper()
        self.stpd = self.stepper_wrapper.stpd
        if self.stepper_wrapper.instances:
            self.instances = self.stepper_wrapper.instances

        if not self.stpd:
            raise RuntimeError("Cannot proceed with no STPD file")

        kwargs = {}
        kwargs['sequence_no'] = self.sequence_no
        if self.ssl:
            _auth_section = ''
            _ciphers = ''
            ssl_template = "transport_t ssl_transport = transport_ssl_t {\n" \
                           "                timeout = 1s\n" \
                           "                %s\n" \
                           "                %s}\n" \
                           "                transport = ssl_transport"

            if self.client_certificate or self.client_key:
                _auth_section = 'auth_t def_auth = auth_t { key = "%s" cert = "%s"} auth = def_auth' \
                                % (self.client_key, self.client_certificate)
            if self.client_cipher_suites:
                _ciphers = 'ciphers = "%s"' % self.client_cipher_suites
            kwargs['ssl_transport'] = ssl_template % (_auth_section, _ciphers)
        else:
            kwargs['ssl_transport'] = ""
        kwargs['method_stream'] = self.method_prefix + \
            "_ipv6_t" if self.ipv6 else self.method_prefix + "_ipv4_t"
        kwargs['phout'] = self.phout_file
        kwargs['answ_log'] = self.answ_log
        kwargs['answ_log_level'] = self.answ_log_level
        kwargs['comment_answ'] = "# " if self.answ_log_level == 'none' else ''
        kwargs['stpd'] = self.stpd
        kwargs['source_log_prefix'] = self.source_log_prefix
        kwargs['method_options'] = self.method_options
        if self.tank_type:
            kwargs[
                'proto'] = "proto=http_proto%s" % self.sequence_no if self.tank_type == 'http' else "proto=none_proto"
            kwargs['comment_proto'] = ""
        else:
            kwargs['proto'] = ""
            kwargs['comment_proto'] = "#"

        if self.gatling:
            kwargs['bind'] = 'bind={ ' + self.gatling + ' }'
        else:
            kwargs['bind'] = ''
        kwargs['ip'] = self.resolved_ip
        kwargs['port'] = self.port
        kwargs['timeout'] = self.timeout
        kwargs['instances'] = self.instances
        tune = ''
        if self.phantom_http_entity:
            tune += "entity = " + self.phantom_http_entity + "\n"
        if self.phantom_http_field:
            tune += "field = " + self.phantom_http_field + "\n"
        if self.phantom_http_field_num:
            tune += "field_num = {}\n".format(self.phantom_http_field_num)
        if self.phantom_http_line:
            tune += "line = " + self.phantom_http_line + "\n"
        if tune:
            kwargs['reply_limits'] = 'reply_limits = {\n' + tune + "}"
        else:
            kwargs['reply_limits'] = ''

        if self.is_main:
            fname = 'phantom_benchmark_main.tpl'
        else:
            fname = 'phantom_benchmark_additional.tpl'
        template_str = resource_string(
            __name__, "config/" + fname)
        tpl = string.Template(template_str)
        config = tpl.substitute(kwargs)

        return config

# ========================================================================


import datetime
import logging
import subprocess
import time
from threading import Event

import yaml

from netort.resource import manager as resource_manager

from .reader import PandoraStatsReader
from ..Console import Plugin as ConsolePlugin
from ..Console import screen as ConsoleScreen
from ..Phantom import PhantomReader, string_to_df
from ...common.interfaces import AbstractInfoWidget, GeneratorPlugin
from ...common.util import tail_lines, FileMultiReader

logger = logging.getLogger(__name__)

class Plugin(GeneratorPlugin):
    """    Pandora load generator plugin    """

    OPTION_CONFIG = "config"
    SECTION = "pandora"
    DEFAULT_REPORT_FILE = "phout.log"
    DEFAULT_EXPVAR_PORT = 1234

    def __init__(self, core, cfg, name):
        super(Plugin, self).__init__(core, cfg, name)
        self.output_finished = Event()
        self.enum_ammo = False
        self.process_start_time = None
        self.pandora_cmd = None
        self.pandora_config_file = None
        self.config_contents = None
        self.custom_config = False
        self.expvar = None
        self.expvar_port = None
        self.sample_log = None
        self.__address = None
        self.__schedule = None
        self.ammofile = None
        self.process_stderr_file = None

    @staticmethod
    def get_key():
        return __file__

    def get_available_options(self):
        opts = [
            "pandora_cmd", "buffered_seconds",
            "config_content", "config_file"
        ]
        return opts

    def configure(self):
        self.pandora_cmd = self.get_option("pandora_cmd")
        self.buffered_seconds = self.get_option("buffered_seconds")
        self.affinity = self.get_option("affinity", "")

        # get config_contents and patch it: expand resources via resource manager
        # config_content option has more priority over config_file
        if self.get_option("config_content"):
            logger.info('Found config_content option configuration')
            self.config_contents = self.__patch_raw_config_and_dump(self.get_option("config_content"))
        elif self.get_option("config_file"):
            logger.info('Found config_file option configuration')
            with open(self.get_option("config_file"), 'rb') as config:
                external_file_config_contents = yaml.load(config.read())
            self.config_contents = self.__patch_raw_config_and_dump(external_file_config_contents)
        else:
            raise RuntimeError("Neither pandora.config_content, nor pandora.config_file specified")
        logger.debug('Config after parsing for patching: %s', self.config_contents)

        # find report filename and add to artifacts
        self.sample_log = self.__find_report_filename()
        with open(self.sample_log, 'w'):
            pass
        self.core.add_artifact_file(self.sample_log)

    def __patch_raw_config_and_dump(self, cfg_dict):
        if not cfg_dict:
            raise RuntimeError('Empty pandora config')
        # patch
        config_content = self.patch_config(cfg_dict)
        # dump
        self.pandora_config_file = self.core.mkstemp(".yaml", "pandora_config_")
        self.core.add_artifact_file(self.pandora_config_file)
        with open(self.pandora_config_file, 'w') as config_file:
            yaml.dump(config_content, config_file)
        return config_content

    def patch_config(self, config):
        """
        download remote resources, replace links with local filenames
        add result file section
        :param dict config: pandora config
        """
        # get expvar parameters
        if config.get("monitoring"):
            if config["monitoring"].get("expvar"):
                self.expvar = config["monitoring"]["expvar"].get("enabled")
                if config["monitoring"]["expvar"].get("port"):
                    self.expvar_port = config["monitoring"]["expvar"].get("port")
                else:
                    self.expvar_port = self.DEFAULT_EXPVAR_PORT
        # or set if expvar not exists
        else:
            config["monitoring"] = {
                "expvar": {
                    "enabled": True,
                }
            }
            self.expvar = True
            self.expvar_port = self.DEFAULT_EXPVAR_PORT

        # FIXME this is broken for custom ammo providers due to interface incompatibility
        # FIXME refactor pandora plx
        for pool in config['pools']:
            if pool.get('ammo', {}).get('file', ''):
                self.ammofile = pool['ammo']['file']
                pool['ammo']['file'] = resource_manager.resource_filename(
                    self.ammofile
                )
            if not pool.get('result') or 'phout' not in pool.get('result', {}).get('type', ''):
                logger.warning('Seems like pandora result file not specified... adding defaults')
                pool['result'] = dict(
                    destination=self.DEFAULT_REPORT_FILE,
                    type='phout',
                )
        return config

    @property
    def address(self):
        if not self.__address:
            for pool in self.config_contents['pools']:
                if pool.get('gun', {}).get('target'):
                    self.__address = pool.get('gun', {}).get('target')
                    break
            else:
                self.__address = 'unknown'
        return self.__address

    @property
    def schedule(self):
        if not self.__schedule:
            for pool in self.config_contents['pools']:
                if pool.get('rps'):
                    self.__schedule = pool.get('rps')
                    break
            else:
                self.__schedule = 'unknown'
        return self.__schedule

    def get_info(self):
        return self.Info(
            address=self.address,
            ammo_file=self.ammofile,
            duration=0,
            instances=0,
            loop_count=0,
            port=self.address.split(':')[-1],
            rps_schedule=self.schedule
        )

    def __find_report_filename(self):
        for pool in self.config_contents['pools']:
            if pool.get('result', {}).get('destination', None):
                report_filename = pool.get('result').get('destination')
                logger.info('Found report file in pandora config: %s', report_filename)
                return report_filename
        return self.DEFAULT_REPORT_FILE

    def get_reader(self, parser=string_to_df):
        if self.reader is None:
            self.reader = FileMultiReader(self.sample_log, self.output_finished)
        return PhantomReader(self.reader.get_file(), parser=parser)

    def get_stats_reader(self):
        if self.stats_reader is None:
            self.stats_reader = PandoraStatsReader(self.expvar, self.expvar_port)
        return self.stats_reader

    def prepare_test(self):
        try:
            console = self.core.get_plugin_of_type(ConsolePlugin)
        except KeyError as ex:
            logger.debug("Console not found: %s", ex)
            console = None

        if console:
            widget = PandoraInfoWidget(self)
            console.add_info_widget(widget)
            self.core.job.aggregator.add_result_listener(widget)

    def start_test(self):
        args = [self.pandora_cmd, self.pandora_config_file]
        if self.affinity:
            self.core.__setup_affinity(self.affinity, args=args)
        logger.info("Starting: %s", args)
        self.process_start_time = time.time()
        self.process_stderr_file = self.core.mkstemp(".log", "pandora_")
        self.core.add_artifact_file(self.process_stderr_file)
        self.process_stderr = open(self.process_stderr_file, 'w')
        try:
            self.process = subprocess.Popen(
                args,
                stderr=self.process_stderr,
                stdout=self.process_stderr,
                close_fds=True)
        except OSError:
            logger.debug(
                "Unable to start Pandora binary. Args: %s", args, exc_info=True)
            raise RuntimeError(
                "Unable to start Pandora binary and/or file does not exist: %s" % args)

    def is_test_finished(self):
        retcode = self.process.poll()
        if retcode is not None and retcode == 0:
            logger.info("Pandora subprocess done its work successfully and finished w/ retcode 0")
            self.output_finished.set()
            return retcode
        elif retcode is not None and retcode != 0:
            lines_amount = 20
            logger.info("Pandora finished with non-zero retcode. Last %s logs of Pandora log:", lines_amount)
            self.output_finished.set()
            last_log_contents = tail_lines(self.process_stderr_file, lines_amount)
            for logline in last_log_contents:
                logger.info(logline.strip('\n'))
            return abs(retcode)
        else:
            return -1

    def end_test(self, retcode):
        if self.process and self.process.poll() is None:
            logger.warning(
                "Terminating worker process with PID %s", self.process.pid)
            self.process.terminate()
            if self.process_stderr:
                self.process_stderr.close()
        else:
            logger.debug("Seems subprocess finished OK")
        self.output_finished.set()
        return retcode

class PandoraInfoWidget(AbstractInfoWidget):
    ''' Right panel widget '''

    def __init__(self, pandora):
        AbstractInfoWidget.__init__(self)
        self.krutilka = ConsoleScreen.krutilka()
        self.owner = pandora
        self.reqps = 0
        self.active = 0

    def get_index(self):
        return 0

    def on_aggregated_data(self, data, stats):
        self.reqps = stats["metrics"]["reqps"]
        self.active = stats["metrics"]["instances"]

    def render(self, screen):
        text = " Pandora Test %s" % self.krutilka.next()
        space = screen.right_panel_width - len(text) - 1
        left_spaces = space / 2
        right_spaces = space / 2

        dur_seconds = int(time.time()) - int(self.owner.process_start_time)
        duration = str(datetime.timedelta(seconds=dur_seconds))

        template = screen.markup.BG_BROWN + '~' * left_spaces + \
            text + ' ' + '~' * right_spaces + screen.markup.RESET + "\n"
        template += "Command Line: %s\n"
        template += "    Duration: %s\n"
        template += "  Requests/s: %s\n"
        template += " Active reqs: %s\n"
        template += "      Target: %s\n"
        template += "    Schedule: \n%s\n"
        data = (
            self.owner.pandora_cmd,
            duration,
            self.reqps,
            self.active,
            self.owner.address,
            yaml.dump(self.owner.schedule)
        )

        return template % data


import imp
import os
import re
import sys
import uuid

import logging
import pkg_resources
import yaml
from cerberus.validator import Validator

from yandextank.common.util import recursive_dict_update
logger = logging.getLogger(__name__)

class ValidationError(Exception):
    MSG_TEMPLATE = """Validation error:\n{}"""

    def __init__(self, errors):
        self.errors = errors
        self.message = self.MSG_TEMPLATE.format(yaml.dump(self.errors))

    def __str__(self):
        return self.message

def load_yaml_schema(path):
    # DEFAULT_FILENAME = 'schema.yaml'
    with open(path, 'r') as f:
        return yaml.load(f)

def load_py_schema(path):
    schema_module = imp.load_source('schema', path)
    return schema_module.SCHEMA

def load_plugin_schema(package):
    try:
        return load_yaml_schema(
            pkg_resources.resource_filename(
                package, 'config/schema.yaml'))
    except IOError:
        try:
            return load_py_schema(
                pkg_resources.resource_filename(
                    package, 'config/schema.py'))
        except ImportError:
            logger.error(
                "Could not find schema for %s (should be located in config/ directory of a plugin)",
                package)
            raise IOError('No schema found for plugin %s' % package)
    except ImportError:
        if 'aggregator' in package.lower():
            logger.exception('Plugin Aggregator is now deprecated, please remove this section from your config.')
        raise ValidationError({'package': ['No module named {}'.format(package)]})

def load_schema(directory, filename=None):
    try:
        return load_yaml_schema(directory)
    except IOError:
        try:
            return load_py_schema(directory)
        except ImportError:
            raise IOError(
                'Neither .yaml nor .py schema found in %s' %
                directory)

class PatchedValidator(Validator):

    def _validate_description(self, description, field, value):
        """ {'type': 'string'} """
        pass

    def _validate_values_description(self, values_description, field, value):
        """ {'type': 'dict'} """
        pass

    def _validate_tutorial_link(self, tutorial_link, field, value):
        """ {'type': 'string'} """
        pass

    def _validate_examples(self, examples, field, value):
        """ {'type': 'dict'} """
        pass

    @staticmethod
    def is_number(value):
        try:
            float(value)
            return True
        except ValueError:
            return False

    def validate_duration(self, field, duration):
        '''
        2h
        2h5m
        5m
        180
        1h4m3
        :param duration:
        :return:
        '''
        DURATION_RE = r'^(\d+d)?(\d+h)?(\d+m)?(\d+s?)?$'
        if not re.match(DURATION_RE, duration):
            self._error(field, 'Load duration examples: 2h30m; 5m15; 180')

    def _validator_load_scheme(self, field, value):
        '''
        step(10,200,5,180)
        step(5,50,2.5,5m)
        line(22,154,2h5m)
        step(5,50,2.5,5m) line(22,154,2h5m)
        const(10,1h4m3s)
        :param field:
        :param value:
        :return:
        '''
        # stpd file can be any value
        if self.document['load_type'] in 'stpd_file':
            return

        PRIMARY_RE = r'(step|line|const)\((.+?)\)'
        N_OF_ARGS = {
            'step': 4,
            'line': 3,
            'const': 2,
        }
        matches = re.findall(PRIMARY_RE, value)
        if len(matches) == 0:
            self._error(field, 'Should match one of the following patterns: step(...) / line(...) / const(...)')
        else:
            for match in matches:
                curve, params_str = match
                params = [v.strip() for v in params_str.split(',')]
                # check number of arguments
                if not len(params) == N_OF_ARGS[curve]:
                    self._error(field, '{} load scheme: expected {} arguments, found {}'.format(curve,
                                                                                N_OF_ARGS[curve],
                                                                                len(params)))
                # check arguments' types
                for param in params[:-1]:
                    if not self.is_number(param):
                        self._error(field, 'Argument {} in load scheme should be a number'.format(param))
                self.validate_duration(field, params[-1])

class TankConfig(object):
    DYNAMIC_OPTIONS = {
        'uuid': lambda: str(uuid.uuid4()),
        'pid': lambda: os.getpid(),
        'cmdline': lambda: ' '.join(sys.argv)
    }

    def __init__(
            self,
            configs,
            with_dynamic_options=True,
            core_section='core',
            error_output=None):
        """

        :param configs: list of configs dicts
        :param with_dynamic_options: insert uuid, pid, and other DYNAMIC_OPTIONS
        :param core_section: name of core section in config
        :param error_output: file to output error messages
        """
        self._errors = None
        if not isinstance(configs, list):
            configs = [configs]
        self.raw_config_dict = self.__load_multiple(
            [config for config in configs if config is not None])
        if self.raw_config_dict.get(core_section) is None:
            self.raw_config_dict[core_section] = {}
        self.with_dynamic_options = with_dynamic_options
        self.CORE_SECTION = core_section
        self._validated = None
        self._plugins = None
        self.ERROR_OUTPUT = error_output
        self.BASE_SCHEMA = load_yaml_schema(pkg_resources.resource_filename('yandextank.core', 'config/schema.yaml'))
        self.PLUGINS_SCHEMA = load_yaml_schema(pkg_resources.resource_filename('yandextank.core', 'config/plugins_schema.yaml'))

    def get_configinitial(self):
        return self.raw_config_dict

    def validate(self):
        if not self._validated:
            try:
                self._validated = ValidatedConfig(self.__validate(), self.BASE_SCHEMA)
                self._errors = {}
            except ValidationError as e:
                self._validated = None
                self._errors = e.errors
        return self._validated, self._errors, self.raw_config_dict

    @property
    def validated(self):
        if not self._validated:
            try:
                self._validated = self.__validate()
            except ValidationError as e:
                self._errors = e.errors
                if self.ERROR_OUTPUT:
                    with open(self.ERROR_OUTPUT, 'w') as f:
                        yaml.dump(e.errors, f)
                raise
        return self._validated

    def save_raw(self, filename):
        with open(filename, 'w') as f:
            yaml.dump(self.raw_config_dict, f)

    def __load_multiple(self, configs):
        configs_count = len(configs)
        if configs_count == 0:
            return {}
        elif configs_count == 1:
            return configs[0]
        elif configs_count == 2:
            return recursive_dict_update(configs[0], configs[1])
        else:
            return self.__load_multiple(
                [recursive_dict_update(configs[0], configs[1])] + configs[2:])

    def __parse_enabled_plugins(self):
        """
        :returns: [(plugin_name, plugin_package, plugin_config), ...]
        :rtype: list of tuple
        """
        return [
            (
                plugin_name,
                plugin['package'],
                plugin) for plugin_name,
            plugin in self.raw_config_dict.items() if (
                plugin_name not in self.BASE_SCHEMA.keys()) and isinstance(
                plugin,
                dict) and plugin.get('enabled')]

    def __validate(self):
        core_validated = self.__validate_core()
        # plugins:
        errors = {}
        results = {}
        for plugin_name, package, config in self.__parse_enabled_plugins():
            try:
                results[plugin_name] = \
                    self.__validate_plugin(config,
                                           load_plugin_schema(package))
            except ValidationError as e:
                errors[plugin_name] = e.errors
        if len(errors) > 0:
            raise ValidationError((dict(errors)))

        for plugin_name, plugin_conf in results.items():
            core_validated[plugin_name] = plugin_conf
        return core_validated

    def __validate_core(self):
        v = PatchedValidator(allow_unknown=self.PLUGINS_SCHEMA)
        result = v.validate(self.raw_config_dict, self.BASE_SCHEMA)
        if not result:
            errors = v.errors
            for key, value in v.errors.items():
                if 'must be of dict type' in value:
                    errors[key] = ['unknown field']
            raise ValidationError(errors)
        normalized = v.normalized(self.raw_config_dict)
        return self.__set_core_dynamic_options(
            normalized) if self.with_dynamic_options else normalized

    def __validate_plugin(self, config, schema):
        schema.update(self.PLUGINS_SCHEMA['schema'])
        v = PatchedValidator(schema, allow_unknown=False)
        # .validate() makes .errors as side effect if there's any
        if not v.validate(config):
            raise ValidationError(v.errors)
        # .normalized() returns config with defaults
        return v.normalized(config)

    def __set_core_dynamic_options(self, config):
        for option, setter in self.DYNAMIC_OPTIONS.items():
            try:
                config[self.CORE_SECTION][option] = setter()
            except KeyError:
                config[self.CORE_SECTION] = {option: setter()}
        return config

    def __str__(self):
        return yaml.dump(self.raw_config_dict)

class ValidatedConfig(object):
    def __init__(self, validated, base_schema):
        """

        :type validated: dict
        """
        self.validated = validated
        self.base_schema = base_schema
        self._plugins = None

    @property
    def plugins(self):
        """
            :returns: [(plugin_name, plugin_package, plugin_config), ...]
            :rtype: list of tuple
        """
        if not self._plugins:
            self._plugins = [
                (plugin_name,
                 plugin_cfg['package'],
                 plugin_cfg) for plugin_name, plugin_cfg in self.validated.items() if (
                    plugin_name not in self.base_schema.keys()) and plugin_cfg['enabled']]
        return self._plugins

    def get_option(self, section, option, default=None):
        try:
            return self.validated[section][option]
        except KeyError:
            if default is not None:
                return default
            raise

    def __nonzero__(self):
        return len(self.validated) > 0

    def __bool__(self):
        return self.__nonzero__()

    def dump(self, path):
        with open(path, 'w') as f:
            yaml.dump(self.validated, f)

    def __str__(self):
        return yaml.dump(self.validated)


import collections
import os
import pwd
import socket
import traceback

import http.client
import logging
import errno
import itertools
import re
import select
import psutil
import argparse

from paramiko import SSHClient, AutoAddPolicy
from retrying import retry

logger = logging.getLogger(__name__)

class SecuredShell(object):
    def __init__(self, host, port, username, timeout=10):
        self.host = host
        self.port = port
        self.username = username
        self.timeout = timeout

    def connect(self):
        logger.debug(
            "Opening SSH connection to {host}:{port}".format(
                host=self.host, port=self.port))
        client = SSHClient()
        client.load_system_host_keys()
        client.set_missing_host_key_policy(AutoAddPolicy())

        try:
            client.connect(
                self.host,
                port=self.port,
                username=self.username,
                timeout=self.timeout, )
        except ValueError as e:
            logger.error(e)
            logger.warning(
                """
Patching Crypto.Cipher.AES.new and making another attempt.

See here for the details:
http://uucode.com/blog/2015/02/20/workaround-for-ctr-mode-needs-counter-parameter-not-iv/
            """)
            client.close()
            import Crypto.Cipher.AES
            orig_new = Crypto.Cipher.AES.new

            def fixed_AES_new(key, *ls):
                if Crypto.Cipher.AES.MODE_CTR == ls[0]:
                    ls = list(ls)
                    ls[1] = ''
                return orig_new(key, *ls)

            Crypto.Cipher.AES.new = fixed_AES_new
            client.connect(
                self.host,
                port=self.port,
                username=self.username,
                timeout=self.timeout)
        return client

    def execute(self, cmd):
        logger.info("Execute on %s: %s", self.host, cmd)
        with self.connect() as client:
            _, stdout, stderr = client.exec_command(cmd)
            output = stdout.read()
            errors = stderr.read()
            err_code = stdout.channel.recv_exit_status()
        return output, errors, err_code

    def rm(self, path):
        return self.execute("rm -f %s" % path)

    def rm_r(self, path):
        return self.execute("rm -rf %s" % path)

    def mkdir(self, path):
        return self.execute("mkdir -p %s" % path)

    def send_file(self, local_path, remote_path):
        logger.info(
            "Sending [{local}] to {host}:[{remote}]".format(
                local=local_path, host=self.host, remote=remote_path))

        with self.connect() as client, client.open_sftp() as sftp:
            result = sftp.put(local_path, remote_path, self.get_progress_logger(local_path))
        return result

    @staticmethod
    def get_progress_logger(name):

        def print_progress(done, total):
            logger.info("Transferring {}: {}%".format(name, done * 100 / total))
        return print_progress

    def get_file(self, remote_path, local_path):
        logger.info(
            "Receiving from {host}:[{remote}] to [{local}]".format(
                local=local_path, host=self.host, remote=remote_path))
        with self.connect() as client, client.open_sftp() as sftp:
            result = sftp.get(remote_path, local_path, self.get_progress_logger(remote_path))
        return result

    def async_session(self, cmd):
        return AsyncSession(self, cmd)

def check_ssh_connection():
    logging.basicConfig(
        level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s')
    logging.getLogger("paramiko.transport").setLevel(logging.DEBUG)

    parser = argparse.ArgumentParser(
        description='Test SSH connection for monitoring.')
    parser.add_argument(
        '-e', '--endpoint', default='example.org', help='which host to try')

    parser.add_argument(
        '-u', '--username', default=pwd.getpwuid(os.getuid())[0], help='SSH username')

    parser.add_argument('-p', '--port', default=22, type=int, help='SSH port')
    args = parser.parse_args()
    logging.info(
        "Checking SSH to %s@%s:%d", args.username, args.endpoint, args.port)
    ssh = SecuredShell(args.endpoint, args.port, args.username, 10)
    data = ssh.execute("ls -l")
    logging.info('Output data of ssh.execute("ls -l"): %s', data[0])
    logging.info('Output errors of ssh.execute("ls -l"): %s', data[1])
    logging.info('Output code of ssh.execute("ls -l"): %s', data[2])

    logging.info('Trying to create paramiko ssh connection client')
    client = ssh.connect()
    logging.info('Created paramiko ssh connection client: %s', client)
    logging.info('Trying to open sftp')
    sftp = client.open_sftp()
    logging.info('Opened sftp: %s', sftp)
    logging.info('Trying to send test file to /tmp')
    res = sftp.put('/usr/lib/yandex/yandex-tank/bin/tank.log', '/opt')
    logging.info('Result of sending test file: %s', res)

class AsyncSession(object):
    def __init__(self, ssh, cmd):
        self.client = ssh.connect()
        self.session = self.client.get_transport().open_session()
        self.session.get_pty()
        self.session.exec_command(cmd)

    def send(self, data):
        self.session.send(data)

    def close(self):
        self.session.close()
        self.client.close()

    def finished(self):
        return self.session.exit_status_ready()

    def read_maybe(self):
        if self.session.recv_ready():
            return self.session.recv(4096)
        else:
            return None

# HTTP codes
HTTP = http.client.responses

# Extended list of HTTP status codes(WEBdav etc.)
# HTTP://en.wikipedia.org/wiki/List_of_HTTP_status_codes
WEBDAV = {
    102: 'Processing',
    103: 'Checkpoint',
    122: 'Request-URI too long',
    207: 'Multi-Status',
    226: 'IM Used',
    308: 'Resume Incomplete',
    418: 'I\'m a teapot',
    422: 'Unprocessable Entity',
    423: 'Locked',
    424: 'Failed Dependency',
    425: 'Unordered Collection',
    426: 'Upgrade Required',
    444: 'No Response',
    449: 'Retry With',
    450: 'Blocked by Windows Parental Controls',
    499: 'Client Closed Request',
    506: 'Variant Also Negotiates',
    507: 'Insufficient Storage',
    509: 'Bandwidth Limit Exceeded',
    510: 'Not Extended',
    598: 'network read timeout error',
    599: 'network connect timeout error',
    999: 'Common Failure',
}
HTTP.update(WEBDAV)

# NET codes
NET = {
    0: "Success",
    1: "Operation not permitted",
    2: "No such file or directory",
    3: "No such process",
    4: "Interrupted system call",
    5: "Input/output error",
    6: "No such device or address",
    7: "Argument list too long",
    8: "Exec format error",
    9: "Bad file descriptor",
    10: "No child processes",
    11: "Resource temporarily unavailable",
    12: "Cannot allocate memory",
    13: "Permission denied",
    14: "Bad address",
    15: "Block device required",
    16: "Device or resource busy",
    17: "File exists",
    18: "Invalid cross-device link",
    19: "No such device",
    20: "Not a directory",
    21: "Is a directory",
    22: "Invalid argument",
    23: "Too many open files in system",
    24: "Too many open files",
    25: "Inappropriate ioctl for device",
    26: "Text file busy",
    27: "File too large",
    28: "No space left on device",
    29: "Illegal seek",
    30: "Read-only file system",
    31: "Too many links",
    32: "Broken pipe",
    33: "Numerical argument out of domain",
    34: "Numerical result out of range",
    35: "Resource deadlock avoided",
    36: "File name too long",
    37: "No locks available",
    38: "Function not implemented",
    39: "Directory not empty",
    40: "Too many levels of symbolic links",
    41: "Unknown error 41",
    42: "No message of desired type",
    43: "Identifier removed",
    44: "Channel number out of range",
    45: "Level 2 not synchronized",
    46: "Level 3 halted",
    47: "Level 3 reset",
    48: "Link number out of range",
    49: "Protocol driver not attached",
    50: "No CSI structure available",
    51: "Level 2 halted",
    52: "Invalid exchange",
    53: "Invalid request descriptor",
    54: "Exchange full",
    55: "No anode",
    56: "Invalid request code",
    57: "Invalid slot",
    58: "Unknown error 58",
    59: "Bad font file format",
    60: "Device not a stream",
    61: "No data available",
    62: "Timer expired",
    63: "Out of streams resources",
    64: "Machine is not on the network",
    65: "Package not installed",
    66: "Object is remote",
    67: "Link has been severed",
    68: "Advertise error",
    69: "Srmount error",
    70: "Communication error on send",
    71: "Protocol error",
    72: "Multihop attempted",
    73: "RFS specific error",
    74: "Bad message",
    75: "Value too large for defined data type",
    76: "Name not unique on network",
    77: "File descriptor in bad state",
    78: "Remote address changed",
    79: "Can not access a needed shared library",
    80: "Accessing a corrupted shared library",
    81: ".lib section in a.out corrupted",
    82: "Attempting to link in too many shared libraries",
    83: "Cannot exec a shared library directly",
    84: "Invalid or incomplete multibyte or wide character",
    85: "Interrupted system call should be restarted",
    86: "Streams pipe error",
    87: "Too many users",
    88: "Socket operation on non-socket",
    89: "Destination address required",
    90: "Message too long",
    91: "Protocol wrong type for socket",
    92: "Protocol not available",
    93: "Protocol not supported",
    94: "Socket type not supported",
    95: "Operation not supported",
    96: "Protocol family not supported",
    97: "Address family not supported by protocol",
    98: "Address already in use",
    99: "Cannot assign requested address",
    100: "Network is down",
    101: "Network is unreachable",
    102: "Network dropped connection on reset",
    103: "Software caused connection abort",
    104: "Connection reset by peer",
    105: "No buffer space available",
    106: "Transport endpoint is already connected",
    107: "Transport endpoint is not connected",
    108: "Cannot send after transport endpoint shutdown",
    109: "Too many references: cannot splice",
    110: "Connection timed out",
    111: "Connection refused",
    112: "Host is down",
    113: "No route to host",
    114: "Operation already in progress",
    115: "Operation now in progress",
    116: "Stale NFS file handle",
    117: "Structure needs cleaning",
    118: "Not a XENIX named type file",
    119: "No XENIX semaphores available",
    120: "Is a named type file",
    121: "Remote I/O error",
    122: "Disk quota exceeded",
    123: "No medium found",
    124: "Wrong medium type",
    125: "Operation canceled",
    126: "Required key not available",
    127: "Key has expired",
    128: "Key has been revoked",
    129: "Key was rejected by service",
    130: "Owner died",
    131: "State not recoverable",
    999: 'Common Failure',
}

def log_stdout_stderr(log, stdout, stderr, comment=""):
    """
    This function polls stdout and stderr streams and writes their contents
    to log
    """
    readable = select.select([stdout], [], [], 0)[0]
    if stderr:
        exceptional = select.select([stderr], [], [], 0)[0]
    else:
        exceptional = []

    log.debug("Selected: %s, %s", readable, exceptional)

    for handle in readable:
        line = handle.read()
        readable.remove(handle)
        if line:
            log.debug("%s stdout: %s", comment, line.strip())

    for handle in exceptional:
        line = handle.read()
        exceptional.remove(handle)
        if line:
            log.warn("%s stderr: %s", comment, line.strip())

def expand_to_milliseconds(str_time):
    """
    converts 1d2s into milliseconds
    """
    return expand_time(str_time, 'ms', 1000)

def expand_to_seconds(str_time):
    """
    converts 1d2s into seconds
    """
    return expand_time(str_time, 's', 1)

def expand_time(str_time, default_unit='s', multiplier=1):
    """
    helper for above functions
    """
    parser = re.compile(r'(\d+)([a-zA-Z]*)')
    parts = parser.findall(str_time)
    result = 0.0
    for value, unit in parts:
        value = int(value)
        unit = unit.lower()
        if unit == '':
            unit = default_unit

        if unit == 'ms':
            result += value * 0.001
            continue
        elif unit == 's':
            result += value
            continue
        elif unit == 'm':
            result += value * 60
            continue
        elif unit == 'h':
            result += value * 60 * 60
            continue
        elif unit == 'd':
            result += value * 60 * 60 * 24
            continue
        elif unit == 'w':
            result += value * 60 * 60 * 24 * 7
            continue
        else:
            raise ValueError(
                "String contains unsupported unit %s: %s" % (unit, str_time))
    return int(result * multiplier)

def pid_exists(pid):
    """Check whether pid exists in the current process table."""
    if pid < 0:
        return False
    try:
        os.kill(pid, 0)
    except OSError as exc:
        logging.debug("No process[%s]: %s", exc.errno, exc)
        return exc.errno == errno.EPERM
    else:
        p = psutil.Process(pid)
        return p.status != psutil.STATUS_ZOMBIE

def splitstring(string):
    """
    >>> string = 'apple orange "banana tree" green'
    >>> splitstring(string)
    ['apple', 'orange', 'green', '"banana tree"']
    """
    patt = re.compile(r'"[\w ]+"')
    if patt.search(string):
        quoted_item = patt.search(string).group()
        newstring = patt.sub('', string)
        return newstring.split() + [quoted_item]
    else:
        return string.split()

def pairs(lst):
    """
    Iterate over pairs in the list
    """
    return itertools.izip(lst[::2], lst[1::2])

def update_status(status, multi_key, value):
    if len(multi_key) > 1:
        update_status(status.setdefault(multi_key[0], {}), multi_key[1:], value)
    else:
        status[multi_key[0]] = value

class AddressWizard:
    def __init__(self):
        self.lookup_fn = socket.getaddrinfo
        self.socket_class = socket.socket

    def resolve(self, address_str, do_test=False, explicit_port=False):
        """

        :param address_str:
        :return: tuple of boolean, string, int - isIPv6, resolved_ip, port (may be null), extracted_address
        """

        if not address_str:
            raise RuntimeError("Mandatory option was not specified: address")

        logger.debug("Trying to resolve address string: %s", address_str)

        port = None

        braceport_re = re.compile(r"""
            ^
            \[           # opening brace
            \s?          # space sym?
            (\S+)        # address - string
            \s?          # space sym?
            \]           # closing brace
            :            # port separator
            \s?          # space sym?
            (\d+)        # port
            $
        """, re.X)
        braceonly_re = re.compile(r"""
            ^
            \[           # opening brace
            \s?          # space sym?
            (\S+)        # address - string
            \s?          # space sym?
            \]           # closing brace
            $
        """, re.X)

        if braceport_re.match(address_str):
            logger.debug("Braces and port present")
            match = braceport_re.match(address_str)
            logger.debug("Match: %s %s ", match.group(1), match.group(2))
            address_str, port = match.group(1), match.group(2)
        elif braceonly_re.match(address_str):
            logger.debug("Braces only present")
            match = braceonly_re.match(address_str)
            logger.debug("Match: %s", match.group(1))
            address_str = match.group(1)
        else:
            logger.debug("Parsing port")
            parts = address_str.split(":")
            if len(parts) <= 2:  # otherwise it is v6 address
                address_str = parts[0]
                if len(parts) == 2:
                    port = int(parts[1])
        if port is not None:
            port = int(port)
        address_str = address_str.strip()
        try:
            resolved = self.lookup_fn(address_str, port)
            logger.debug("Lookup result: %s", resolved)
        except Exception:
            logger.debug("Exception trying to resolve hostname %s :", address_str, exc_info=True)
            raise

        for (family, socktype, proto, canonname, sockaddr) in resolved:
            is_v6 = family == socket.AF_INET6
            parsed_ip, port = sockaddr[0], sockaddr[1]

            if explicit_port:
                logger.warn(
                    "Using phantom.port option is deprecated. Use phantom.address=[address]:port instead"
                )
                port = int(explicit_port)
            elif not port:
                port = 80

            if do_test:
                try:
                    logger.info("Testing connection to resolved address %s and port %s", parsed_ip, port)
                    self.__test(family, (parsed_ip, port))
                except RuntimeError:
                    logger.info("Failed TCP connection test using [%s]:%s", parsed_ip, port)
                    logger.debug("Failed TCP connection test using [%s]:%s", parsed_ip, port, exc_info=True)
                    continue
            return is_v6, parsed_ip, int(port), address_str

        msg = "All connection attempts failed for %s, use {phantom.connection_test: false} to disable it"
        raise RuntimeError(msg % address_str)

    def __test(self, af, sa):
        test_sock = self.socket_class(af)
        try:
            test_sock.settimeout(5)
            test_sock.connect(sa)
        except Exception as exc:
            logger.debug(
                "Exception on connect attempt [%s]:%s : %s", sa[0], sa[1],
                traceback.format_exc(exc))
            msg = "TCP Connection test failed for [%s]:%s, use phantom.connection_test=0 to disable it"
            raise RuntimeError(msg % (sa[0], sa[1]))
        finally:
            test_sock.close()

def recursive_dict_update(d1, d2):
    for k, v in d2.items():
        if isinstance(v, collections.Mapping):
            r = recursive_dict_update(d1.get(k, {}), v)
            d1[k] = r
        else:
            d1[k] = d2[k]
    return d1

class FileScanner(object):
    """
    Basic class for stats reader for continiuos reading file line by line

    Default line separator is a newline symbol. You can specify other separator
    via constructor argument
    """

    _BUFSIZE = 4096

    def __init__(self, path, sep="\n"):
        self.__path = path
        self.__sep = sep
        self.__closed = False
        self.__buffer = ""

    def _read_lines(self, chunk):
        self.__buffer += chunk
        portions = self.__buffer.split(self.__sep)
        for portion in portions[:-1]:
            yield portion
        self.__buffer = portions[-1]

    def _read_data(self, lines):
        raise NotImplementedError()

    def __iter__(self):
        with open(self.__path) as stats_file:
            while not self.__closed:
                chunk = stats_file.read(self._BUFSIZE)
                yield self._read_data(self._read_lines(chunk))

    def close(self):
        self.__closed = True

def tail_lines(filepath, lines_num, bufsize=8192):
    fsize = os.stat(filepath).st_size
    iter_ = 0
    with open(filepath) as f:
        if bufsize > fsize:
            bufsize = fsize - 1
        data = []
        try:
            while True:
                iter_ += 1
                f.seek(fsize - bufsize * iter_)
                data.extend(f.readlines())
                if len(data) >= lines_num or f.tell() == 0:
                    return data[-lines_num:]
        except (IOError, OSError):
            return data

class FileLockedError(RuntimeError):
    pass

    @classmethod
    def retry(cls, exception):
        return isinstance(exception, cls)

class FileMultiReader(object):
    def __init__(self, filename, provider_stop_event, cache_size=1024 * 1024 * 50):
        self.buffer = ""
        self.filename = filename
        self.cache_size = cache_size
        self._cursor_map = {}
        self._is_locked = False
        self._opened_file = open(self.filename)
        self.stop = provider_stop_event

    def close(self, force=False):
        self.wait_lock()
        self._opened_file.close()
        self.unlock()

    def get_file(self, cache_size=None):
        cache_size = self.cache_size if not cache_size else cache_size
        fileobj = FileLike(self, cache_size)
        return fileobj

    def read_with_lock(self, pos, _len=None):
        """
        Reads {_len} characters if _len is not None else reads line
        :param pos: start reading position
        :param _len: number of characters to read
        :rtype: (string, int)
        """
        self.wait_lock()
        try:
            self._opened_file.seek(pos)
            result = self._opened_file.read(_len) if _len is not None else self._opened_file.readline()
            stop_pos = self._opened_file.tell()
        finally:
            self.unlock()
        if not result and self.stop.is_set():
            result = None
        return result, stop_pos

    @retry(wait_random_min=5, wait_random_max=20, stop_max_delay=10000,
           retry_on_exception=FileLockedError.retry, wrap_exception=True)
    def wait_lock(self):
        if self._is_locked:
            raise FileLockedError('Generator output file {} is locked'.format(self.filename))
        else:
            self._is_locked = True
            return True

    def unlock(self):
        self._is_locked = False

class FileLike(object):
    def __init__(self, multireader, cache_size):
        """
        :type multireader: FileMultiReader
        """
        self.multireader = multireader
        self.cache_size = cache_size
        self._cursor = 0

    def read(self, _len=None):
        _len = self.cache_size if not _len else _len
        result, self._cursor = self.multireader.read_with_lock(self._cursor, _len)
        return result

    def readline(self):
        result, self._cursor = self.multireader.read_with_lock(self._cursor)
        return result


'''
Module contains top-level generators.
'''
import hashlib
import json
import logging
import os

from builtins import zip

from netort.resource import manager as resource

from . import format as fmt
from . import info
from .config import ComponentFactory

class AmmoFactory(object):
    '''
    A generator that produces ammo.
    '''

    def __init__(self, factory):
        '''
        Factory parameter is a configured ComponentFactory that
        is able to produce load plan and ammo generator.
        '''
        self.factory = factory
        self.load_plan = factory.get_load_plan()
        self.ammo_generator = factory.get_ammo_generator()
        self.filter = factory.get_filter()
        self.marker = factory.get_marker()

    def __iter__(self):
        '''
        Returns a generator of (timestamp, marker, missile) tuples
        where missile is in a string representation. Load Plan (timestamps
        generator) and ammo generator are taken from the previously
        configured ComponentFactory, passed as a parameter to the
        __init__ method of this class.
        '''
        ammo_stream = (
            ammo
            for ammo in ((missile, marker or self.marker(missile))
                         for missile, marker in self.ammo_generator)
            if self.filter(ammo))

        return ((timestamp, marker or self.marker(missile), missile)
                for timestamp, (missile, marker
                                ) in zip(self.load_plan, ammo_stream))

class Stepper(object):
    def __init__(self, core, **kwargs):
        info.status = info.StepperStatus()
        info.status.core = core
        self.af = AmmoFactory(ComponentFactory(**kwargs))
        self.ammo = fmt.Stpd(self.af)

    def write(self, f):
        for missile in self.ammo:
            f.write(missile)
            try:
                info.status.inc_ammo_count()
            except StopIteration:
                break

class LoadProfile(object):

    def __init__(self, load_type, schedule):
        self.load_type = load_type
        self.schedule = self.__make_steps(schedule)

    def is_rps(self):
        return self.load_type == 'rps'

    def is_instances(self):
        return self.load_type == 'instances'

    @staticmethod
    def __make_steps(schedule):
        steps = []
        for step in " ".join(schedule.split("\n")).split(')'):
            if step.strip():
                steps.append(step.strip() + ')')
        return steps

class StepperWrapper(object):
    # TODO: review and rewrite this class
    '''
    Wrapper for cached stepper functionality
    '''
    OPTION_LOAD = 'load_profile'
    OPTION_LOAD_TYPE = 'load_type'
    OPTION_SCHEDULE = 'schedule'
    OPTION_STEPS = 'steps'
    OPTION_TEST_DURATION = 'test_duration'
    OPTION_AMMO_COUNT = 'ammo_count'
    OPTION_LOOP = 'loop'
    OPTION_LOOP_COUNT = 'loop_count'
    OPTION_AMMOFILE = "ammofile"
    OPTION_LOADSCHEME = 'loadscheme'
    OPTION_INSTANCES_LIMIT = 'instances'

    def __init__(self, core, cfg):
        self.log = logging.getLogger(__name__)
        self.core = core
        self.cfg = cfg

        self.cache_dir = '.'

        # per-shoot params
        self.instances = 1000
        self.http_ver = '1.0'
        self.ammo_file = None
        self.loop_limit = -1
        self.ammo_limit = -1
        self.uris = []
        self.headers = []
        self.autocases = 0
        self.enum_ammo = False
        self.force_stepping = None
        self.chosen_cases = []

        # out params
        self.stpd = None
        self.steps = []
        self.ammo_count = 1
        self.duration = 0
        self.loop_count = 0
        self.loadscheme = ""
        self.file_cache = 8192

    def get_option(self, option, param2=None):
        ''' get_option wrapper'''
        result = self.cfg[option]
        self.log.debug(
            "Option %s = %s", option, result)
        return result

    @staticmethod
    def get_available_options():
        opts = [
            StepperWrapper.OPTION_AMMOFILE, StepperWrapper.OPTION_LOOP,
            StepperWrapper.OPTION_SCHEDULE, StepperWrapper.OPTION_INSTANCES_LIMIT
        ]
        opts += [
            "instances_schedule", "uris", "headers", "header_http", "autocases",
            "enum_ammo", "ammo_type", "ammo_limit"
        ]
        opts += [
            "use_caching", "cache_dir", "force_stepping", "file_cache",
            "chosen_cases"
        ]
        return opts

    def read_config(self):
        ''' stepper part of reading options '''
        self.log.info("Configuring StepperWrapper...")
        self.ammo_file = self.get_option(self.OPTION_AMMOFILE)
        self.ammo_type = self.get_option('ammo_type')
        if self.ammo_file:
            self.ammo_file = os.path.expanduser(self.ammo_file)
        self.loop_limit = self.get_option(self.OPTION_LOOP)
        self.ammo_limit = self.get_option("ammo_limit")

        self.load_profile = LoadProfile(**self.get_option('load_profile'))

        self.instances = int(
            self.get_option(self.OPTION_INSTANCES_LIMIT, '1000'))
        self.uris = self.get_option("uris", [])
        while '' in self.uris:
            self.uris.remove('')
        self.headers = self.get_option("headers")
        self.http_ver = self.get_option("header_http")
        self.autocases = self.get_option("autocases")
        self.enum_ammo = self.get_option("enum_ammo")
        self.use_caching = self.get_option("use_caching")

        self.file_cache = self.get_option('file_cache')
        cache_dir = self.get_option("cache_dir") or self.core.artifacts_base_dir
        self.cache_dir = os.path.expanduser(cache_dir)
        self.force_stepping = self.get_option("force_stepping")
        if self.get_option(self.OPTION_LOAD)[self.OPTION_LOAD_TYPE] == 'stpd_file':
            self.stpd = self.get_option(self.OPTION_LOAD)[self.OPTION_SCHEDULE]

        self.chosen_cases = self.get_option("chosen_cases").split()
        if self.chosen_cases:
            self.log.info("chosen_cases LIMITS: %s", self.chosen_cases)

    def prepare_stepper(self):
        ''' Generate test data if necessary '''

        def publish_info(stepper_info):
            info.status.publish('loadscheme', stepper_info.loadscheme)
            info.status.publish('loop_count', stepper_info.loop_count)
            info.status.publish('steps', stepper_info.steps)
            info.status.publish('duration', stepper_info.duration)
            info.status.ammo_count = stepper_info.ammo_count
            info.status.publish('instances', stepper_info.instances)
            self.core.publish('stepper', 'loadscheme', stepper_info.loadscheme)
            self.core.publish('stepper', 'loop_count', stepper_info.loop_count)
            self.core.publish('stepper', 'steps', stepper_info.steps)
            self.core.publish('stepper', 'duration', stepper_info.duration)
            self.core.publish('stepper', 'ammo_count', stepper_info.ammo_count)
            self.core.publish('stepper', 'instances', stepper_info.instances)
            return stepper_info

        if not self.stpd:
            self.stpd = self.__get_stpd_filename()
            if self.use_caching and not self.force_stepping and os.path.exists(
                    self.stpd) and os.path.exists(self.__si_filename()):
                self.log.info("Using cached stpd-file: %s", self.stpd)
                stepper_info = self.__read_cached_options()
                if self.instances and self.load_profile.is_rps():
                    self.log.info(
                        "rps_schedule is set. Overriding cached instances param from config: %s",
                        self.instances)
                    stepper_info = stepper_info._replace(
                        instances=self.instances)
                publish_info(stepper_info)
            else:
                if (
                        self.force_stepping and os.path.exists(self.__si_filename())):
                    os.remove(self.__si_filename())
                self.__make_stpd_file()
                stepper_info = info.status.get_info()
                self.__write_cached_options(stepper_info)
        else:
            self.log.info("Using specified stpd-file: %s", self.stpd)
            stepper_info = publish_info(self.__read_cached_options())
        self.ammo_count = stepper_info.ammo_count
        self.duration = stepper_info.duration
        self.loop_count = stepper_info.loop_count
        self.loadscheme = stepper_info.loadscheme
        self.steps = stepper_info.steps
        if stepper_info.instances:
            self.instances = stepper_info.instances

    def __si_filename(self):
        '''Return name for stepper_info json file'''
        return "%s_si.json" % self.stpd

    def __get_stpd_filename(self):
        ''' Choose the name for stepped data file '''
        if self.use_caching:
            sep = "|"
            hasher = hashlib.md5()
            hashed_str = "cache version 6" + sep + \
                ';'.join(self.load_profile.schedule) + sep + str(self.loop_limit)
            hashed_str += sep + str(self.ammo_limit) + sep + ';'.join(
                self.load_profile.schedule) + sep + str(self.autocases)
            hashed_str += sep + ";".join(self.uris) + sep + ";".join(
                self.headers) + sep + self.http_ver + sep + ";".join(
                    self.chosen_cases)
            hashed_str += sep + str(self.enum_ammo) + sep + str(self.ammo_type)
            if self.load_profile.is_instances():
                hashed_str += sep + str(self.instances)
            if self.ammo_file:
                opener = resource.get_opener(self.ammo_file)
                hashed_str += sep + opener.hash
            else:
                if not self.uris:
                    raise RuntimeError("Neither ammofile nor uris specified")
                hashed_str += sep + \
                    ';'.join(self.uris) + sep + ';'.join(self.headers)
            self.log.debug("stpd-hash source: %s", hashed_str)
            hasher.update(hashed_str.encode('utf8'))
            if not os.path.exists(self.cache_dir):
                os.makedirs(self.cache_dir)
            stpd = self.cache_dir + '/' + \
                os.path.basename(self.ammo_file) + \
                "_" + hasher.hexdigest() + ".stpd"
        else:
            stpd = os.path.realpath("ammo.stpd")
        self.log.debug("Generated cache file name: %s", stpd)
        return stpd

    def __read_cached_options(self):
        '''
        Read stepper info from json
        '''
        self.log.debug("Reading cached stepper info: %s", self.__si_filename())
        with open(self.__si_filename(), 'r') as si_file:
            si = info.StepperInfo(**json.load(si_file))
        return si

    def __write_cached_options(self, si):
        '''
        Write stepper info to json
        '''
        self.log.debug("Saving stepper info: %s", self.__si_filename())
        with open(self.__si_filename(), 'w') as si_file:
            json.dump(si._asdict(), si_file, indent=4)

    def __make_stpd_file(self):
        ''' stpd generation using Stepper class '''
        self.log.info("Making stpd-file: %s", self.stpd)
        stepper = Stepper(
            self.core,
            rps_schedule=self.load_profile.schedule if self.load_profile.is_rps() else None,
            http_ver=self.http_ver,
            ammo_file=self.ammo_file,
            instances_schedule=self.load_profile.schedule if self.load_profile.is_instances() else None,
            instances=self.instances,
            loop_limit=self.loop_limit,
            ammo_limit=self.ammo_limit,
            uris=self.uris,
            headers=[header.strip('[]') for header in self.headers],
            autocases=self.autocases,
            enum_ammo=self.enum_ammo,
            ammo_type=self.ammo_type,
            chosen_cases=self.chosen_cases,
            use_cache=self.use_caching)
        with open(self.stpd, 'w', self.file_cache) as os:
            stepper.write(os)