""" Contains Phantom Plugin, Console widgets, result reader classes """
# FIXME: 3 there is no graceful way to interrupt the process of phout import
# TODO: phout import
import logging
import subprocess
import time
from threading import Event

from .reader import PhantomReader, PhantomStatsReader, string_to_df
from .utils import PhantomConfig
from .widget import PhantomInfoWidget, PhantomProgressBarWidget
from ..Autostop import Plugin as AutostopPlugin
from ..Console import Plugin as ConsolePlugin
from ...common.interfaces import AbstractCriterion, GeneratorPlugin
from ...common.util import expand_to_seconds, FileMultiReader

from netort.process import execute

logger = logging.getLogger(__name__)

class Plugin(GeneratorPlugin):
    """     Plugin for running phantom tool    """

    OPTION_CONFIG = "config"

    def __init__(self, core, cfg, name):
        super(Plugin, self).__init__(core, cfg, name)
        self.phout_finished = Event()
        self.predefined_phout = None
        self.did_phout_import_try = False
        self.eta_file = None
        self.processed_ammo_count = 0
        self.cached_info = None
        self.exclude_markers = []
        self._stat_log = None
        self._phantom = None
        self.config = None
        self.enum_ammo = None
        self.phout_import_mode = None

    @staticmethod
    def get_key():
        return __file__

    def get_available_options(self):
        opts = [
            "phantom_path", "buffered_seconds", "exclude_markers", "affinity"
        ]
        opts += [PhantomConfig.OPTION_PHOUT, self.OPTION_CONFIG]
        opts += PhantomConfig.get_available_options()
        return opts

    def configure(self):
        # plugin part
        self.config = self.get_option(self.OPTION_CONFIG, '')
        self.affinity = self.get_option('affinity', '')
        self.enum_ammo = self.get_option("enum_ammo", False)
        self.buffered_seconds = int(
            self.get_option("buffered_seconds", self.buffered_seconds))

        try:
            autostop = self.core.get_plugin_of_type(AutostopPlugin)
            autostop.add_criterion_class(UsedInstancesCriterion)
        except KeyError:
            logger.debug(
                "No autostop plugin found, not adding instances criterion")

        self.predefined_phout = self.get_option(PhantomConfig.OPTION_PHOUT, '')
        if not self.get_option(self.OPTION_CONFIG, '') and self.predefined_phout:
            self.phout_import_mode = True
        self.phantom_config = self.phantom.config_file

    @property
    def phantom(self):
        """
        :rtype: PhantomConfig
        """
        if not self._phantom:
            self._phantom = PhantomConfig(self.core, self.cfg, self.stat_log)
            self._phantom.read_config()
        return self._phantom

    @property
    def stat_log(self):
        if not self._stat_log:
            self._stat_log = self.core.mkstemp(".log", "phantom_stat_")
        return self._stat_log

    def get_reader(self, parser=string_to_df):
        if self.reader is None:
            self.reader = FileMultiReader(self.phantom.phout_file, self.phout_finished)
        return PhantomReader(self.reader.get_file(), parser=parser)

    def get_stats_reader(self):
        if self.stats_reader is None:
            self.stats_reader = PhantomStatsReader(self.stat_log, self.phantom.get_info())
        return self.stats_reader

    def prepare_test(self):
        try:
            retcode, stdout, stderr = execute(
                [self.get_option("phantom_path"), 'check', self.phantom.config_file], catch_out=True
            )
        except OSError:
            logger.debug("Phantom I/O engine is not installed!", exc_info=True)
            raise OSError("Phantom I/O engine not found. \nMore information: {doc_url}".format(
                doc_url='http://yandextank.readthedocs.io/en/latest/install.html')
            )
        else:
            if retcode or stderr:
                raise RuntimeError("Config check failed. Subprocess returned code %s. Stderr: %s" % (retcode, stderr))

        logger.debug(
            "Linking sample reader to aggregator."
            " Reading samples from %s", self.phantom.phout_file)

        logger.debug(
            "Linking stats reader to aggregator."
            " Reading stats from %s", self.phantom.stat_log)

        self.core.job.aggregator.add_result_listener(self)

        self.core.job.phantom_info = self.phantom.get_info()

        try:
            console = self.core.get_plugin_of_type(ConsolePlugin)
        except KeyError:
            logger.debug("Console not found: %s", exc_info=True)
        else:
            widget1 = PhantomProgressBarWidget(self)
            console.add_info_widget(widget1)
            self.core.job.aggregator.add_result_listener(widget1)

            widget2 = PhantomInfoWidget(self)
            console.add_info_widget(widget2)
            self.core.job.aggregator.add_result_listener(widget2)

    def start_test(self):
        args = [self.get_option("phantom_path"), 'run', self.phantom.config_file]
        if self.affinity:
            logger.info('Enabled cpu affinity %s for phantom', self.affinity)
            args = self.core.__setup_affinity(self.affinity, args=args)
        logger.debug("Starting %s with arguments: %s", self.get_option("phantom_path"), args)
        self.start_time = time.time()
        phantom_stderr_file = self.core.mkstemp(
            ".log", "phantom_stdout_stderr_")
        self.core.add_artifact_file(phantom_stderr_file)
        self.process_stderr = open(phantom_stderr_file, 'w')
        self.process = subprocess.Popen(
            args,
            stderr=self.process_stderr,
            stdout=self.process_stderr,
            close_fds=True)

    def is_test_finished(self):
        retcode = self.process.poll()
        if retcode is not None:
            logger.info("Phantom done its work with exit code: %s", retcode)
            self.phout_finished.set()
            return abs(retcode)
        else:
            info = self.get_info()
            if info:
                eta = int(info.duration) - (int(time.time()) - int(self.start_time))
                self.publish('eta', eta)
            return -1

    def end_test(self, retcode):
        if self.process and self.process.poll() is None:
            logger.info("Terminating phantom process with PID %s", self.process.pid)
            self.process.terminate()
            if self.process:
                self.process.communicate()
        else:
            logger.debug("Seems phantom finished OK")
        self.phout_finished.set()
        if self.process_stderr:
            self.process_stderr.close()
        return retcode

    def post_process(self, retcode):
        if not retcode:
            info = self.get_info()
            if info and info.ammo_count != self.processed_ammo_count:
                logger.warning(
                    "Planned ammo count %s differs from processed %s",
                    info.ammo_count, self.processed_ammo_count)
        return retcode

    def on_aggregated_data(self, data, stat):
        self.processed_ammo_count += data["overall"]["interval_real"]["len"]
        logger.debug("Processed ammo count: %s/", self.processed_ammo_count)

    def get_info(self):
        """ returns info object """
        if not self.cached_info:
            if not self.phantom:
                return None
            self.cached_info = self.phantom.get_info()
        return self.cached_info

class UsedInstancesCriterion(AbstractCriterion):
    """
    Autostop criterion, based on active instances count
    """
    RC_INST = 24

    @staticmethod
    def get_type_string():
        return 'instances'

    def __init__(self, autostop, param_str):
        AbstractCriterion.__init__(self)
        self.seconds_count = 0
        self.autostop = autostop
        self.threads_limit = 1

        level_str = param_str.split(',')[0].strip()
        if level_str[-1:] == '%':
            self.level = float(level_str[:-1]) / 100
            self.is_relative = True
        else:
            self.level = int(level_str)
            self.is_relative = False
        self.seconds_limit = expand_to_seconds(param_str.split(',')[1])

        try:
            phantom = autostop.core.get_plugin_of_type(Plugin)
            info = phantom.get_info()
            if info:
                self.threads_limit = info.instances
            if not self.threads_limit:
                raise ValueError(
                    "Cannot create 'instances' criterion"
                    " with zero instances limit")
        except KeyError:
            logger.warning("No phantom module, 'instances' autostop disabled")

    def notify(self, data, stat):
        threads = stat["metrics"]["instances"]
        if self.is_relative:
            threads = float(threads) / self.threads_limit
        if threads > self.level:
            if not self.seconds_count:
                self.cause_second = (data, stat)

            logger.debug(self.explain())

            self.seconds_count += 1
            self.autostop.add_counting(self)
            if self.seconds_count >= self.seconds_limit:
                return True
        else:
            self.seconds_count = 0

        return False

    def get_rc(self):
        return self.RC_INST

    def get_level_str(self):
        """
        String value for instances level
        """
        if self.is_relative:
            level_str = str(100 * self.level) + "%"
        else:
            level_str = self.level
        return level_str

    def explain(self):
        items = (
            self.get_level_str(), self.seconds_count,
            self.cause_second[0].get('ts'))
        return (
            "Testing threads (instances) utilization"
            " higher than %s for %ss, since %s" % items)

    def widget_explain(self):
        items = (self.get_level_str(), self.seconds_count, self.seconds_limit)
        return "Instances >%s for %s/%ss" % items, float(
            self.seconds_count) / self.seconds_limit


"""Monitoring collector """
import hashlib
import logging
import sys
import time

import copy

from ...common.interfaces import MonitoringDataListener

from ..Telegraf.client import SSHClient, LocalhostClient
from ..Telegraf.config import ConfigManager

logger = logging.getLogger(__name__)

class MonitoringCollector(object):
    """Aggregate data from several collectors

    Workflow:
    plugin creates collector
    collector reads monitoring config and creates agents
    collector creates configs for agents (telegraf, shutdown&startup, custom scripts)
    collector installs agents on targets, send configs on targets
    agent starts startups on target, then starts telegraf
    agent reads output of telegraf, consolidates output, caches 5 seconds and then sends output to collector
    collector polls agents for data, decodes known metrics and counts diffs for diff-like metrics
    collector sends data to listeners

    """

    def __init__(self, disguise_hostnames, kill_old):
        self.kill_old = kill_old
        self.disguise_hostnames = disguise_hostnames
        self.config = None
        self.default_target = None
        self.agents = []
        self.agent_sessions = []
        self.listeners = []
        self.first_data_received = False
        self.__collected_data = []
        self.artifact_files = []
        self.load_start_time = None
        self.config_manager = ConfigManager()
        self.old_style_configs = False
        self.clients = {'localhost': LocalhostClient, 'ssh': SSHClient}

    def add_listener(self, obj):
        self.listeners.append(obj)

    def prepare(self):
        """Prepare for monitoring - install agents etc"""

        # Parse config
        agent_configs = []
        if self.config:
            agent_configs = self.config_manager.getconfig(
                self.config, self.default_target)

        # Creating agent for hosts
        for config in agent_configs:
            if config['host'] in ['localhost', '127.0.0.1', '::1']:
                client = self.clients['localhost'](
                    config, self.old_style_configs, kill_old=self.kill_old)
            else:
                client = self.clients['ssh'](
                    config, self.old_style_configs, timeout=5, kill_old=self.kill_old)
            logger.debug('Installing monitoring agent. Host: %s', client.host)
            agent_config, startup_config, customs_script = client.install()
            if agent_config:
                self.agents.append(client)
                self.artifact_files.append(agent_config)
            if startup_config:
                self.artifact_files.append(startup_config)
            if customs_script:
                self.artifact_files.append(customs_script)

    def start(self):
        """ Start agents

        execute popen of agent.py on target and start output reader thread.
        """
        [agent.start() for agent in self.agents]
        [agent.reader_thread.start() for agent in self.agents]

    def poll(self):
        """ Poll agents for data
        """
        start_time = time.time()
        for agent in self.agents:
            for collect in agent.reader:
                # don't crush if trash or traceback came from agent to stdout
                if not collect:
                    return 0
                for chunk in collect:
                    ts, prepared_results = chunk
                    if self.load_start_time and int(
                            ts) >= self.load_start_time:
                        ready_to_send = {
                            "timestamp": int(ts),
                            "data": {
                                self.hash_hostname(agent.host): {
                                    "comment": agent.config.comment,
                                    "metrics": prepared_results
                                }
                            }
                        }
                        self.__collected_data.append(ready_to_send)

        logger.debug(
            'Polling/decoding agents data took: %.2fms',
            (time.time() - start_time) * 1000)

        collected_data_length = len(self.__collected_data)

        if not self.first_data_received and self.__collected_data:
            self.first_data_received = True
            logger.info("Monitoring received first data.")
        else:
            self.send_collected_data()
        return collected_data_length

    def stop(self):
        """Shutdown agents"""
        logger.debug("Uninstalling monitoring agents")
        for agent in self.agents:
            log_filename, data_filename = agent.uninstall()
            self.artifact_files.append(log_filename)
            self.artifact_files.append(data_filename)
        for agent in self.agents:
            try:
                logger.debug(
                    'Waiting for agent %s reader thread to finish.', agent)
                agent.reader_thread.join(10)
            except BaseException:
                logger.error('Monitoring reader thread stuck!', exc_info=True)

    def send_collected_data(self):
        """sends pending data set to listeners"""
        data = self.__collected_data
        self.__collected_data = []
        for listener in self.listeners:
            # deep copy to ensure each listener gets it's own copy
            listener.monitoring_data(copy.deepcopy(data))

    def not_empty(self):
        return len(self.__collected_data) > 0

    def send_rest_data(self):
        while self.not_empty():
            logger.info("Sending monitoring data rests...")
            self.send_collected_data()

    def hash_hostname(self, host):
        if self.disguise_hostnames and host:
            return hashlib.md5(host).hexdigest()
        else:
            return host

class StdOutPrintMon(MonitoringDataListener):
    """Simple listener, writing data to stdout"""

    def __init__(self):
        MonitoringDataListener.__init__(self)

    def monitoring_data(self, data_list):
        [sys.stdout.write(data) for data in data_list]


"""Module provides target monitoring
metrics collector - influxdata's `telegraf` - https://github.com/influxdata/telegraf/
backward compatibility with yandextank's Monitoring module configuration and tools.
"""

import datetime
import fnmatch
import json
import logging
import os
import time
import sys

from copy import deepcopy

from netort.resource import manager as resource

from yandextank.plugins.DataUploader.client import LPRequisites
from ...common.interfaces import MonitoringDataListener, AbstractInfoWidget, MonitoringPlugin
from ...common.util import expand_to_seconds
from ..Autostop import Plugin as AutostopPlugin, AbstractCriterion
from ..Console import Plugin as ConsolePlugin
from ..Telegraf.collector import MonitoringCollector

if sys.version_info[0] < 3:
    from ConfigParser import NoOptionError
else:
    from configparser import NoOptionError

logger = logging.getLogger(__name__)

class Plugin(MonitoringPlugin):
    """  resource mon plugin  """

    SECTION = 'telegraf'  # may be redefined to 'monitoring' sometimes.

    def __init__(self, core, cfg, name):
        super(Plugin, self).__init__(core, cfg, name)
        self.jobno = None
        self.default_target = None
        self.default_config = "{path}/config/monitoring_default_config.xml".format(
            path=os.path.dirname(__file__))
        self.process = None
        self.monitoring = MonitoringCollector(
            disguise_hostnames=self.get_option('disguise_hostnames'),
            kill_old=self.get_option('kill_old'))
        self.die_on_fail = True
        self.data_file = None
        self.mon_saver = None
        self._config = None

    @staticmethod
    def get_key():
        return __file__

    def start_test(self):
        if self.monitoring:
            self.monitoring.load_start_time = time.time()
            logger.debug(
                "load_start_time = %s", self.monitoring.load_start_time)

    def get_available_options(self):
        return [
            "config",
            "default_target",
            "ssh_timeout",
            "disguise_hostnames"
        ]

    def __detect_configuration(self):
        """
        we need to be flexible in order to determine which plugin's configuration
        specified and make appropriate configs to metrics collector

        :return: SECTION name or None for defaults
        """
        try:
            is_telegraf = self.core.get_option('telegraf', "config")
        except KeyError:
            is_telegraf = None
        try:
            is_monitoring = self.core.get_option('monitoring', "config")
        except KeyError:
            is_monitoring = None

        if is_telegraf and is_monitoring:
            raise ValueError(
                'Both telegraf and monitoring configs specified. '
                'Clean up your config and delete one of them')
        if is_telegraf and not is_monitoring:
            return 'telegraf'
        if not is_telegraf and is_monitoring:
            return 'monitoring'
        if not is_telegraf and not is_monitoring:
            # defaults target logic
            try:
                is_telegraf_dt = self.core.get_option('telegraf')
            except NoOptionError:
                is_telegraf_dt = None
            try:
                is_monitoring_dt = self.core.get_option('monitoring')
            except BaseException:
                is_monitoring_dt = None
            if is_telegraf_dt and is_monitoring_dt:
                raise ValueError(
                    'Both telegraf and monitoring default targets specified. '
                    'Clean up your config and delete one of them')
            if is_telegraf_dt and not is_monitoring_dt:
                return
            if not is_telegraf_dt and is_monitoring_dt:
                self.core.set_option(
                    "telegraf", "default_target", is_monitoring_dt)
            if not is_telegraf_dt and not is_monitoring_dt:
                return

    @property
    def config(self):
        """

        :rtype: str
        """
        if self._config is None:
            value = self.get_option('config')

            if value.lower() == "none":
                self.monitoring = None
                self.die_on_fail = False
                self._config = value
            # handle http/https url or file path
            else:
                if value.startswith("<"):
                    config_contents = value
                elif value.lower() == "auto":
                    self.die_on_fail = False
                    with open(resource.resource_filename(self.default_config), 'rb') as def_config:
                        config_contents = def_config.read()
                else:
                    with open(resource.resource_filename(value), 'rb') as config:
                        config_contents = config.read()
                self._config = self._save_config_contents(config_contents)
        return self._config

    def _save_config_contents(self, contents):
        xmlfile = self.core.mkstemp(".xml", "monitoring_")
        self.core.add_artifact_file(xmlfile)
        with open(xmlfile, "wb") as f:  # output file should be in binary mode to support py3
            f.write(contents)
        return xmlfile

    def configure(self):
        self.detected_conf = self.__detect_configuration()
        if self.detected_conf:
            logger.info(
                'Detected monitoring configuration: %s', self.detected_conf)
            self.SECTION = self.detected_conf
        self.default_target = self.get_option("default_target", "localhost")
        if self.config.lower() == "none":
            self.monitoring = None
            self.die_on_fail = False
            return

        with open(self.config) as f:
            self.core.add_artifact_to_send(LPRequisites.MONITORING, unicode(f.read()))

        # FIXME [legacy] backward compatibility with Monitoring module
        # configuration below.
        self.monitoring.ssh_timeout = expand_to_seconds(
            self.get_option("ssh_timeout", "5s"))
        try:
            autostop = self.core.get_plugin_of_type(AutostopPlugin)
            autostop.add_criterion_class(MetricHigherCriterion)
            autostop.add_criterion_class(MetricLowerCriterion)
        except KeyError:
            logger.debug(
                "No autostop plugin found, not adding instances criterion")

    def prepare_test(self):
        if not self.config or self.config.lower() == 'none':
            return

        if "Phantom" in self.core.job.generator_plugin.__module__:
            phantom = self.core.job.generator_plugin

            info = phantom.get_info()
            if info:
                self.default_target = info.address
                logger.debug(
                    "Changed monitoring target to %s", self.default_target)

        self.monitoring.config = self.config
        if self.default_target:
            self.monitoring.default_target = self.default_target

        try:
            console = self.core.get_plugin_of_type(ConsolePlugin)
        except Exception as ex:
            logger.debug("Console not found: %s", ex)
            console = None
        if console:
            widget = MonitoringWidget(self)
            console.add_info_widget(widget)
            self.monitoring.add_listener(widget)

        try:
            self.monitoring.prepare()
            self.monitoring.start()
            count = 0
            while not self.monitoring.first_data_received and count < 15 * 5:
                time.sleep(0.2)
                self.monitoring.poll()
                count += 1
        except BaseException:
            logger.error("Could not start monitoring", exc_info=True)
            if self.die_on_fail:
                raise
            else:
                self.monitoring = None

    def add_listener(self, plugin):
        return self.monitoring.add_listener(plugin)

    def is_test_finished(self):
        if self.monitoring:
            data_len = self.monitoring.poll()
            logger.debug("Monitoring got %s lines", data_len)
        return -1

    def end_test(self, retcode):
        logger.info("Finishing monitoring")
        if self.monitoring:
            self.monitoring.stop()
            for log in self.monitoring.artifact_files:
                self.core.add_artifact_file(log)

            self.monitoring.send_rest_data()
        if self.mon_saver:
            self.mon_saver.close()
        return retcode

    def post_process(self, retcode):
        return retcode

class SaveMonToFile(MonitoringDataListener):
    """
    Default listener - saves data to file
    """

    def __init__(self, out_file):
        MonitoringDataListener.__init__(self)
        if out_file:
            self.store = open(out_file, 'w')

    def monitoring_data(self, data):
        self.store.write(json.dumps(data))
        self.store.write('\n')
        self.store.flush()

    def close(self):
        """ close open files """
        logger.debug("Closing monitoring file")
        if self.store:
            self.store.close()

class MonitoringWidget(AbstractInfoWidget, MonitoringDataListener):
    """
    Screen widget
    """

    def __init__(self, owner):
        AbstractInfoWidget.__init__(self)
        self.owner = owner
        self.data = {}
        self.sign = {}
        self.time = {}
        self.max_metric_len = 0

    def get_index(self):
        return 50

    def __handle_data_items(self, host, data):
        """ store metric in data tree and calc offset signs

        sign < 0 is CYAN, means metric value is lower then previous,
        sign > 1 is YELLOW, means metric value is higher then prevoius,
        sign == 0 is WHITE, means initial or equal metric value
        """
        for metric, value in data.iteritems():
            if value == '':
                self.sign[host][metric] = -1
                self.data[host][metric] = value
            else:
                if not self.data[host].get(metric, None):
                    self.sign[host][metric] = 1
                elif float(value) > float(self.data[host][metric]):
                    self.sign[host][metric] = 1
                elif float(value) < float(self.data[host][metric]):
                    self.sign[host][metric] = -1
                else:
                    self.sign[host][metric] = 0
                self.data[host][metric] = "%.2f" % float(value)

    def monitoring_data(self, block):
        # block sample :
        # [{'timestamp': 1480536634,
        #   'data': {
        #     'some.hostname.tld': {
        #       'comment': '',
        #       'metrics': {
        #         'custom:diskio_reads': 0,
        #         'Net_send': 9922,
        #         'CPU_steal': 0,
        #         'Net_recv': 8489
        #       }
        #     }
        #   },
        #   ...
        # }]
        for chunk in block:
            host = chunk['data'].keys()[0]
            self.time[host] = chunk['timestamp']
            # if initial call, we create dicts w/ data and `signs`
            # `signs` used later to paint metrics w/ different colors
            if not self.data.get(host, None):
                self.data[host] = {}
                self.sign[host] = {}
                for key, value in chunk['data'][host]['metrics'].iteritems():
                    self.sign[host][key] = 0
                    self.data[host][key] = value
            else:
                self.__handle_data_items(host, chunk['data'][host]['metrics'])

    def render(self, screen):
        if not self.owner.monitoring:
            return "Monitoring is " + screen.markup.RED + "offline" + screen.markup.RESET
        else:
            res = "Monitoring is " + screen.markup.GREEN + \
                  "online" + screen.markup.RESET + ":\n"
            for hostname, metrics in self.data.items():
                tm_stamp = datetime.datetime.fromtimestamp(
                    float(self.time[hostname])).strftime('%H:%M:%S')
                res += ("   " + screen.markup.CYAN + "%s" + screen.markup.RESET + " at %s:\n") % (hostname, tm_stamp)
                for metric, value in sorted(metrics.iteritems()):
                    if self.sign[hostname][metric] > 0:
                        value = screen.markup.YELLOW + value + screen.markup.RESET
                    elif self.sign[hostname][metric] < 0:
                        value = screen.markup.CYAN + value + screen.markup.RESET
                    res += "      %s%s: %s\n" % (
                        ' ' * (self.max_metric_len - len(metric)),
                        metric.replace('custom:', '').replace('_', ' '), value)

            return res.strip()

class AbstractMetricCriterion(AbstractCriterion, MonitoringDataListener):
    """ Parent class for metric criterion """

    def __init__(self, autostop, param_str):
        AbstractCriterion.__init__(self)
        try:
            self.mon = autostop.core.get_plugin_of_type(Plugin)
            if self.mon.monitoring:
                self.mon.monitoring.add_listener(self)
        except KeyError:
            logger.warning("No monitoring module, mon autostop disabled")
        self.triggered = False
        self.autostop = autostop

        self.host = param_str.split(',')[0].strip()
        self.metric = param_str.split(',')[1].strip()
        self.value_limit = float(param_str.split(',')[2])
        self.seconds_limit = expand_to_seconds(param_str.split(',')[3])
        self.last_second = None
        self.seconds_count = 0

    def monitoring_data(self, _block):
        if self.triggered:
            return

        block = deepcopy(_block)
        for chunk in block:
            host = chunk['data'].keys()[0]
            data = chunk['data'][host]['metrics']

            if not fnmatch.fnmatch(host, self.host):
                continue

            # some magic, converting custom metric names into names that was in
            # config
            for metric_name in data.keys():
                if metric_name.startswith('custom:'):
                    config_metric_name = metric_name.replace('custom:', '')
                    data[config_metric_name] = data.pop(metric_name)

            if self.metric not in data.keys() or not data[self.metric]:
                data[self.metric] = 0
            logger.debug(
                "Compare %s %s/%s=%s to %s",
                self.get_type_string(), host, self.metric, data[self.metric],
                self.value_limit)
            if self.comparison_fn(float(data[self.metric]), self.value_limit):
                if not self.seconds_count:
                    self.cause_second = self.last_second

                logger.debug(self.explain())

                self.seconds_count += 1
                if self.seconds_count >= self.seconds_limit:
                    logger.debug("Triggering autostop")
                    self.triggered = True
                    return
            else:
                self.seconds_count = 0

    def notify(self, data, stat):
        if self.seconds_count:
            self.autostop.add_counting(self)

        self.last_second = (data, stat)
        return self.triggered

    def comparison_fn(self, arg1, arg2):
        """ comparison function """
        raise NotImplementedError()

class MetricHigherCriterion(AbstractMetricCriterion):
    """ trigger if metric is higher than limit """

    def __init__(self, autostop, param_str):
        AbstractMetricCriterion.__init__(self, autostop, param_str)

    def get_rc(self):
        return 31

    @staticmethod
    def get_type_string():
        return 'metric_higher'

    def explain(self):
        items = (self.host, self.metric, self.value_limit, self.seconds_count)
        return "%s/%s metric value is higher than %s for %s seconds" % items

    def widget_explain(self):
        items = (
            self.host, self.metric, self.value_limit, self.seconds_count,
            self.seconds_limit)
        return "%s/%s > %s for %s/%ss" % items, float(
            self.seconds_count) / self.seconds_limit

    def comparison_fn(self, arg1, arg2):
        return arg1 > arg2

class MetricLowerCriterion(AbstractMetricCriterion):
    """ trigger if metric is lower than limit """

    def __init__(self, autostop, param_str):
        AbstractMetricCriterion.__init__(self, autostop, param_str)

    def get_rc(self):
        return 32

    @staticmethod
    def get_type_string():
        return 'metric_lower'

    def explain(self):
        items = (self.host, self.metric, self.value_limit, self.seconds_count)
        return "%s/%s metric value is lower than %s for %s seconds" % items

    def widget_explain(self):
        items = (
            self.host, self.metric, self.value_limit, self.seconds_count,
            self.seconds_limit)
        return "%s/%s < %s for %s/%ss" % items, float(
            self.seconds_count) / self.seconds_limit

    def comparison_fn(self, arg1, arg2):
        return arg1 < arg2


"""
Monitoring stdout pipe reader. Read chunks from stdout and produce data frames
"""
import logging
import queue
import json

from ..Telegraf.decoder import decoder

logger = logging.getLogger(__name__)

class MonitoringReader(object):
    def __init__(self, source):
        self.buffer = []
        self.source = source
        self.finished = False
        self.prev_check = None

    def __iter__(self):
        while not self.finished:
            try:
                yield self._decode_agents_data(self.source.get_nowait())
            except queue.Empty:
                return

    def _decode_agents_data(self, block):
        """
        decode agents jsons, count diffs
        """
        collect = []
        if block:
            for chunk in block.split('\n'):
                try:
                    if chunk:
                        prepared_results = {}
                        jsn = json.loads(chunk)
                        for ts, values in jsn.iteritems():
                            for key, value in values.iteritems():
                                # key sample: diskio-sda1_io_time
                                # key_group sample: diskio
                                # key_name sample: io_time
                                try:
                                    key_group, key_name = key.split('_')[0].split('-')[0], '_'.join(key.split('_')[1:])
                                except:  # noqa: E722
                                    key_group, key_name = key.split('_')[0], '_'.join(key.split('_')[1:])
                                if key_group in decoder.diff_metrics.keys():
                                    if key_name in decoder.diff_metrics[key_group]:
                                        decoded_key = decoder.find_common_names(
                                            key)
                                        if self.prev_check:
                                            try:
                                                value = jsn[ts][key] - \
                                                    self.prev_check[key]
                                            except KeyError:
                                                logger.debug(
                                                    'There is no diff value for metric %s.\n'
                                                    'Timestamp: %s. Is it initial data?', key, ts, exc_info=True)
                                                value = 0
                                            prepared_results[decoded_key] = value
                                    else:
                                        decoded_key = decoder.find_common_names(
                                            key)
                                        prepared_results[decoded_key] = value
                                else:
                                    decoded_key = decoder.find_common_names(
                                        key)
                                    prepared_results[decoded_key] = value
                            self.prev_check = jsn[ts]
                            collect.append((ts, prepared_results))
                except ValueError:
                    logger.error(
                        'Telegraf agent send trash to output: %s', chunk)
                    logger.debug(
                        'Telegraf agent data block w/ trash: %s',
                        exc_info=True)
                    return []
                except BaseException:
                    logger.error(
                        'Exception trying to parse agent data: %s',
                        chunk,
                        exc_info=True)
                    return []
            if collect:
                return collect


import asyncio
import json
import re
import websockets
from .common import get_base_url, get_credentials
from .entity import Account, Entity
from . import polygon

class StreamConn(object):
    def __init__(self, key_id=None, secret_key=None, base_url=None):
        self._key_id, self._secret_key = get_credentials(key_id, secret_key)
        base_url = re.sub(r'^http', 'ws', base_url or get_base_url())
        self._endpoint = base_url + '/stream'
        self._handlers = {}
        self._base_url = base_url
        self._ws = None
        self.polygon = None

    async def _connect(self):
        ws = await websockets.connect(self._endpoint)
        await ws.send(json.dumps({
            'action': 'authenticate',
            'data': {
                'key_id': self._key_id,
                'secret_key': self._secret_key,
            }
        }))
        r = await ws.recv()
        if isinstance(r, bytes):
            r = r.decode('utf-8')
        msg = json.loads(r)

        if 'data' not in msg or msg['data']['status'] != 'authorized':
            raise ValueError(
                ("Invalid Alpaca API credentials, Failed to authenticate: {}"
                    .format(msg))
            )

        self._ws = ws
        await self._dispatch('authorized', msg)

        asyncio.ensure_future(self._consume_msg())
        return ws

    async def _consume_msg(self):
        ws = self._ws
        try:
            while True:
                r = await ws.recv()
                if isinstance(r, bytes):
                    r = r.decode('utf-8')
                msg = json.loads(r)
                stream = msg.get('stream')
                if stream is not None:
                    await self._dispatch(stream, msg)
        finally:
            await ws.close()
            self._ws = None

    async def _ensure_nats(self):
        if self.polygon is not None:
            return
        key_id = self._key_id
        if 'staging' in self._base_url:
            key_id += '-staging'
        self.polygon = polygon.Stream(key_id)
        self.polygon.register(r'.*', self._dispatch_nats)
        await self.polygon.connect()

    async def _ensure_ws(self):
        if self._ws is not None:
            return
        self._ws = await self._connect()

    async def subscribe(self, channels):
        '''Start subscribing channels.
        If the necessary connection isn't open yet, it opens now.
        '''
        ws_channels = []
        nats_channels = []
        for c in channels:
            if c.startswith(('Q.', 'T.', 'A.', 'AM.',)):
                nats_channels.append(c)
            else:
                ws_channels.append(c)

        if len(ws_channels) > 0:
            await self._ensure_ws()
            await self._ws.send(json.dumps({
                'action': 'listen',
                'data': {
                    'streams': ws_channels,
                }
            }))

        if len(nats_channels) > 0:
            await self._ensure_nats()
            await self.polygon.subscribe(nats_channels)

    def run(self, initial_channels=[]):
        '''Run forever and block until exception is rasised.
        initial_channels is the channels to start with.
        '''
        loop = asyncio.get_event_loop()
        try:
            loop.run_until_complete(self.subscribe(initial_channels))
            loop.run_forever()
        finally:
            loop.run_until_complete(self.close())

    async def close(self):
        '''Close any of open connections'''
        if self._ws is not None:
            await self._ws.close()
        if self.polygon is not None:
            await self.polygon.close()

    def _cast(self, channel, msg):
        if channel == 'account_updates':
            return Account(msg)
        return Entity(msg)

    async def _dispatch_nats(self, conn, subject, data):
        for pat, handler in self._handlers.items():
            if pat.match(subject):
                await handler(self, subject, data)

    async def _dispatch(self, channel, msg):
        for pat, handler in self._handlers.items():
            if pat.match(channel):
                ent = self._cast(channel, msg['data'])
                await handler(self, channel, ent)

    def on(self, channel_pat):
        def decorator(func):
            self.register(channel_pat, func)
            return func

        return decorator

    def register(self, channel_pat, func):
        if not asyncio.iscoroutinefunction(func):
            raise ValueError('handler must be a coroutine function')
        if isinstance(channel_pat, str):
            channel_pat = re.compile(channel_pat)
        self._handlers[channel_pat] = func

    def deregister(self, channel_pat):
        if isinstance(channel_pat, str):
            channel_pat = re.compile(channel_pat)
        del self._handlers[channel_pat]


import pandas as pd
import pprint
import re

ISO8601YMD = re.compile(r'\d{4}-\d{2}-\d{2}T')
NY = 'America/New_York'

class Entity(object):
    '''This helper class provides property access (the "dot notation")
    to the json object, backed by the original object stored in the _raw
    field.
    '''

    def __init__(self, raw):
        self._raw = raw

    def __getattr__(self, key):
        if key in self._raw:
            val = self._raw[key]
            if (isinstance(val, str) and
                    (key.endswith('_at') or
                     key.endswith('_timestamp') or
                     key.endswith('_time')) and
                    ISO8601YMD.match(val)):
                return pd.Timestamp(val)
            else:
                return val
        return super().__getattribute__(key)

    def __repr__(self):
        return '{name}({raw})'.format(
            name=self.__class__.__name__,
            raw=pprint.pformat(self._raw, indent=4),
        )

class Account(Entity):
    pass

class Asset(Entity):
    pass

class Order(Entity):
    pass

class Position(Entity):
    pass

class Bar(Entity):
    def __getattr__(self, key):
        if key == 't':
            val = self._raw[key[0]]
            return pd.Timestamp(val, unit='s', tz=NY)
        return super().__getattr__(key)

class Bars(list):
    def __init__(self, raw):
        super().__init__([Bar(o) for o in raw])
        self._raw = raw

    @property
    def df(self):
        if not hasattr(self, '_df'):
            df = pd.DataFrame(
                self._raw, columns=('t', 'o', 'h', 'l', 'c', 'v'),
            )
            alias = {
                't': 'time',
                'o': 'open',
                'h': 'high',
                'l': 'low',
                'c': 'close',
                'v': 'volume',
            }
            df.columns = [alias[c] for c in df.columns]
            df.set_index('time', inplace=True)
            if not df.empty:
                df.index = pd.to_datetime(
                    (df.index * 1e9).astype('int64'), utc=True,
                ).tz_convert(NY)
            else:
                df.index = pd.to_datetime(
                    df.index, utc=True
                )
            self._df = df
        return self._df

class BarSet(dict):
    def __init__(self, raw):
        for symbol in raw:
            self[symbol] = Bars(raw[symbol])
        self._raw = raw

    @property
    def df(self):
        '''## Experimental '''
        if not hasattr(self, '_df'):
            dfs = []
            for symbol, bars in self.items():
                df = bars.df.copy()
                df.columns = pd.MultiIndex.from_product(
                    [[symbol, ], df.columns])
                dfs.append(df)
            if len(dfs) == 0:
                self._df = pd.DataFrame()
            else:
                self._df = pd.concat(dfs, axis=1)
        return self._df

class Clock(Entity):
    def __getattr__(self, key):
        if key in self._raw:
            val = self._raw[key]
            if key in ('timestamp', 'next_open', 'next_close'):
                return pd.Timestamp(val)
            else:
                return val
        return super().__getattr__(key)

class Calendar(Entity):
    def __getattr__(self, key):
        if key in self._raw:
            val = self._raw[key]
            if key in ('date',):
                return pd.Timestamp(val)
            elif key in ('open', 'close'):
                return pd.Timestamp(val).time()
            else:
                return val
        return super().__getattr__(key)


import logging
import os
import requests
from requests.exceptions import HTTPError
import time
from .common import (
    get_base_url,
    get_data_url,
    get_credentials,
    get_api_version,
)
from .entity import (
    Account, Asset, Order, Position,
    BarSet, Clock, Calendar,
)
from . import polygon

logger = logging.getLogger(__name__)

class RetryException(Exception):
    pass

class APIError(Exception):
    '''Represent API related error.
    error.status_code will have http status code.
    '''

    def __init__(self, error, http_error=None):
        super().__init__(error['message'])
        self._error = error
        self._http_error = http_error

    @property
    def code(self):
        return self._error['code']

    @property
    def status_code(self):
        http_error = self._http_error
        if http_error is not None and hasattr(http_error, 'response'):
            return http_error.response.status_code

    @property
    def request(self):
        if self._http_error is not None:
            return self._http_error.request

    @property
    def response(self):
        if self._http_error is not None:
            return self._http_error.response

class REST(object):
    def __init__(
        self,
        key_id=None,
        secret_key=None,
        base_url=None,
        api_version=None
    ):
        self._key_id, self._secret_key = get_credentials(key_id, secret_key)
        self._base_url = base_url or get_base_url()
        self._api_version = get_api_version(api_version)
        self._session = requests.Session()
        self._retry = int(os.environ.get('APCA_RETRY_MAX', 3))
        self._retry_wait = int(os.environ.get('APCA_RETRY_WAIT', 3))
        self._retry_codes = [int(o)for o in os.environ.get(
            'APCA_RETRY_CODES', '429,504').split(',')]
        self.polygon = polygon.REST(
            self._key_id, 'staging' in self._base_url)

    def _request(self, method, path, data=None, base_url=None):
        base_url = base_url or self._base_url
        url = base_url + '/' + self._api_version + path
        headers = {
            'APCA-API-KEY-ID': self._key_id,
            'APCA-API-SECRET-KEY': self._secret_key,
        }
        opts = {
            'headers': headers,
            # Since we allow users to set endpoint URL via env var,
            # human error to put non-SSL endpoint could exploit
            # uncanny issues in non-GET request redirecting http->https.
            # It's better to fail early if the URL isn't right.
            'allow_redirects': False,
        }
        if method.upper() == 'GET':
            opts['params'] = data
        else:
            opts['json'] = data

        retry = self._retry
        if retry < 0:
            retry = 0
        while retry >= 0:
            try:
                return self._one_request(method, url, opts, retry)
            except RetryException:
                retry_wait = self._retry_wait
                logger.warning(
                    'sleep {} seconds and retrying {} '
                    '{} more time(s)...'.format(
                        retry_wait, url, retry))
                time.sleep(retry_wait)
                retry -= 1
                continue

    def _one_request(self, method, url, opts, retry):
        '''
        Perform one request, possibly raising RetryException in the case
        the response is 429. Otherwise, if error text contain "code" string,
        then it decodes to json object and returns APIError.
        Returns the body json in the 200 status.
        '''
        retry_codes = self._retry_codes
        resp = self._session.request(method, url, **opts)
        try:
            resp.raise_for_status()
        except HTTPError as http_error:
            # retry if we hit Rate Limit
            if resp.status_code in retry_codes and retry > 0:
                raise RetryException()
            if 'code' in resp.text:
                error = resp.json()
                if 'code' in error:
                    raise APIError(error, http_error)
            else:
                raise
        if resp.text != '':
            return resp.json()
        return None

    def get(self, path, data=None):
        return self._request('GET', path, data)

    def post(self, path, data=None):
        return self._request('POST', path, data)

    def delete(self, path, data=None):
        return self._request('DELETE', path, data)

    def data_get(self, path, data=None):
        base_url = get_data_url()
        return self._request('GET', path, data, base_url=base_url)

    def get_account(self):
        '''Get the account'''
        resp = self.get('/account')
        return Account(resp)

    def list_orders(self, status=None, limit=None, after=None, until=None,
                    direction=None, params=None):
        '''
        Get a list of orders
        https://docs.alpaca.markets/web-api/orders/#get-a-list-of-orders
        '''
        if params is None:
            params = dict()
        if limit is not None:
            params['limit'] = limit
        if after is not None:
            params['after'] = after
        if until is not None:
            params['until'] = until
        if direction is not None:
            params['direction'] = direction
        if status is not None:
            params['status'] = status
        resp = self.get('/orders', params)
        return [Order(o) for o in resp]

    def submit_order(self, symbol, qty, side, type, time_in_force,
                     limit_price=None, stop_price=None, client_order_id=None):
        '''Request a new order'''
        params = {
            'symbol': symbol,
            'qty': qty,
            'side': side,
            'type': type,
            'time_in_force': time_in_force,
        }
        if limit_price is not None:
            params['limit_price'] = limit_price
        if stop_price is not None:
            params['stop_price'] = stop_price
        if client_order_id is not None:
            params['client_order_id'] = client_order_id
        resp = self.post('/orders', params)
        return Order(resp)

    def get_order_by_client_order_id(self, client_order_id):
        '''Get an order by client order id'''
        resp = self.get('/orders:by_client_order_id', {
            'client_order_id': client_order_id,
        })
        return Order(resp)

    def get_order(self, order_id):
        '''Get an order'''
        resp = self.get('/orders/{}'.format(order_id))
        return Order(resp)

    def cancel_order(self, order_id):
        '''Cancel an order'''
        self.delete('/orders/{}'.format(order_id))

    def list_positions(self):
        '''Get a list of open positions'''
        resp = self.get('/positions')
        return [Position(o) for o in resp]

    def get_position(self, symbol):
        '''Get an open position'''
        resp = self.get('/positions/{}'.format(symbol))
        return Position(resp)

    def list_assets(self, status=None, asset_class=None):
        '''Get a list of assets'''
        params = {
            'status': status,
            'assert_class': asset_class,
        }
        resp = self.get('/assets', params)
        return [Asset(o) for o in resp]

    def get_asset(self, symbol):
        '''Get an asset'''
        resp = self.get('/assets/{}'.format(symbol))
        return Asset(resp)

    def get_barset(self,
                   symbols,
                   timeframe,
                   limit=None,
                   start=None,
                   end=None,
                   after=None,
                   until=None):
        '''Get BarSet(dict[str]->list[Bar])
        The parameter symbols can be either a comma-split string
        or a list of string. Each symbol becomes the key of
        the returned value.
        '''
        if not isinstance(symbols, str):
            symbols = ','.join(symbols)
        params = {
            'symbols': symbols,
        }
        if limit is not None:
            params['limit'] = limit
        if start is not None:
            params['start'] = start
        if end is not None:
            params['end'] = end
        if after is not None:
            params['after'] = after
        if until is not None:
            params['until'] = until
        resp = self.data_get('/bars/{}'.format(timeframe), params)
        return BarSet(resp)

    def get_clock(self):
        resp = self.get('/clock')
        return Clock(resp)

    def get_calendar(self, start=None, end=None):
        params = {}
        if start is not None:
            params['start'] = start
        if end is not None:
            params['end'] = end
        resp = self.get('/calendar', data=params)
        return [Calendar(o) for o in resp]


from collections import namedtuple
from functools import wraps
import inspect

from dagster import check

from dagster.core.errors import DagsterInvalidDefinitionError, DagsterInvariantViolationError

from . import InputDefinition, OutputDefinition, Result, SolidDefinition

if hasattr(inspect, 'signature'):
    funcsigs = inspect
else:
    import funcsigs

# Error messages are long
# pylint: disable=C0301

class MultipleResults(namedtuple('_MultipleResults', 'results')):
    '''A shortcut to output multiple results.

    When using the :py:func:`@solid <dagster.solid>` API, you may return an instance of
    ``MultipleResults`` from a decorated transform function instead of yielding multiple results.

    Attributes:
      results (list[Result]): list of :py:class:`Result`

    Examples:

        .. code-block:: python

            @solid(outputs=[
                OutputDefinition(name='foo'),
                OutputDefinition(name='bar'),
            ])
            def my_solid():
                return MultipleResults(
                    Result('Barb', 'foo'),
                    Result('Glarb', 'bar'),
                )

            @solid(outputs=[
                OutputDefinition(name='foo'),
                OutputDefinition(name='bar'),
            ])
            def my_solid_from_dict():
                return MultipleResults.from_dict({
                    'foo': 'Barb',
                    'bar': 'Glarb',
                })

    '''

    def __new__(cls, *results):
        return super(MultipleResults, cls).__new__(
            cls,
            # XXX(freiksenet): should check.list_param accept tuples from rest?
            check.opt_list_param(list(results), 'results', Result),
        )

    @staticmethod
    def from_dict(result_dict):
        '''Create a new ``MultipleResults`` object from a dictionary.

        Keys of the dictionary are unpacked into result names.

        Args:
            result_dict (dict) - The dictionary to unpack.

        Returns:
            (:py:class:`MultipleResults <dagster.MultipleResults>`) A new ``MultipleResults`` object

        '''
        check.dict_param(result_dict, 'result_dict', key_type=str)
        results = []
        for name, value in result_dict.items():
            results.append(Result(value, name))
        return MultipleResults(*results)

class _LambdaSolid(object):
    def __init__(self, name=None, inputs=None, output=None, description=None):
        self.name = check.opt_str_param(name, 'name')
        self.input_defs = check.opt_list_param(inputs, 'inputs', InputDefinition)
        self.output_def = check.inst_param(output, 'output', OutputDefinition)
        self.description = check.opt_str_param(description, 'description')

    def __call__(self, fn):
        check.callable_param(fn, 'fn')

        if not self.name:
            self.name = fn.__name__

        _validate_transform_fn(self.name, fn, self.input_defs)
        transform_fn = _create_lambda_solid_transform_wrapper(fn, self.input_defs, self.output_def)
        return SolidDefinition(
            name=self.name,
            inputs=self.input_defs,
            outputs=[self.output_def],
            transform_fn=transform_fn,
            description=self.description,
        )

class _Solid(object):
    def __init__(self, name=None, inputs=None, outputs=None, description=None, config_field=None):
        self.name = check.opt_str_param(name, 'name')
        self.input_defs = check.opt_list_param(inputs, 'inputs', InputDefinition)
        outputs = outputs or ([OutputDefinition()] if outputs is None else [])
        self.outputs = check.list_param(outputs, 'outputs', OutputDefinition)
        self.description = check.opt_str_param(description, 'description')
        # config_field will be checked within SolidDefinition
        self.config_field = config_field

    def __call__(self, fn):
        check.callable_param(fn, 'fn')

        if not self.name:
            self.name = fn.__name__

        _validate_transform_fn(self.name, fn, self.input_defs, [('context',)])
        transform_fn = _create_solid_transform_wrapper(fn, self.input_defs, self.outputs)
        return SolidDefinition(
            name=self.name,
            inputs=self.input_defs,
            outputs=self.outputs,
            transform_fn=transform_fn,
            config_field=self.config_field,
            description=self.description,
        )

def lambda_solid(name=None, inputs=None, output=None, description=None):
    '''(decorator) Create a simple solid.

    This shortcut allows the creation of simple solids that do not require
    configuration and whose implementations do not require a context.

    Lambda solids take inputs and produce a single output. The body of the function
    should return a single value.

    Args:
        name (str): Name of solid.
        inputs (list[InputDefinition]): List of inputs.
        output (OutputDefinition): The output of the solid. Defaults to ``OutputDefinition()``.
        description (str): Solid description.

    Examples:

        .. code-block:: python

            @lambda_solid
            def hello_world():
                return 'hello'

            @lambda_solid(inputs=[InputDefinition(name='foo')])
            def hello_world(foo):
                return foo

    '''
    output = output or OutputDefinition()

    if callable(name):
        check.invariant(inputs is None)
        check.invariant(description is None)
        return _LambdaSolid(output=output)(name)

    return _LambdaSolid(name=name, inputs=inputs, output=output, description=description)

def solid(name=None, inputs=None, outputs=None, config_field=None, description=None):
    '''(decorator) Create a solid with specified parameters.

    This shortcut simplifies the core solid API by exploding arguments into kwargs of the
    transform function and omitting additional parameters when they are not needed.
    Parameters are otherwise as in the core API, :py:class:`SolidDefinition`.

    The decorated function will be used as the solid's transform function. Unlike in the core API,
    the transform function does not have to yield :py:class:`Result` object directly. Several
    simpler alternatives are available:

    1. Return a value. This is returned as a :py:class:`Result` for a single output solid.
    2. Return a :py:class:`Result`. Works like yielding result.
    3. Return an instance of :py:class:`MultipleResults`. Works like yielding several results for
       multiple outputs. Useful for solids that have multiple outputs.
    4. Yield :py:class:`Result`. Same as default transform behaviour.

    Args:
        name (str): Name of solid.
        inputs (list[InputDefinition]): List of inputs.
        outputs (list[OutputDefinition]): List of outputs.
        config_field (Field):
            The configuration for this solid.
        description (str): Description of this solid.

    Examples:

        .. code-block:: python

            @solid
            def hello_world(_context):
                print('hello')

            @solid()
            def hello_world(_context):
                print('hello')

            @solid(outputs=[OutputDefinition()])
            def hello_world(_context):
                return {'foo': 'bar'}

            @solid(outputs=[OutputDefinition()])
            def hello_world(_context):
                return Result(value={'foo': 'bar'})

            @solid(outputs=[OutputDefinition()])
            def hello_world(_context):
                yield Result(value={'foo': 'bar'})

            @solid(outputs=[
                OutputDefinition(name="left"),
                OutputDefinition(name="right"),
            ])
            def hello_world(_context):
                return MultipleResults.from_dict({
                    'left': {'foo': 'left'},
                    'right': {'foo': 'right'},
                })

            @solid(
                inputs=[InputDefinition(name="foo")],
                outputs=[OutputDefinition()]
            )
            def hello_world(_context, foo):
                return foo

            @solid(
                inputs=[InputDefinition(name="foo")],
                outputs=[OutputDefinition()],
            )
            def hello_world(context, foo):
                context.log.info('log something')
                return foo

            @solid(
                inputs=[InputDefinition(name="foo")],
                outputs=[OutputDefinition()],
                config_field=Field(types.Dict({'str_value' : Field(types.String)})),
            )
            def hello_world(context, foo):
                # context.solid_config is a dictionary with 'str_value' key
                return foo + context.solid_config['str_value']

    '''
    # This case is for when decorator is used bare, without arguments. e.g. @solid versus @solid()
    if callable(name):
        check.invariant(inputs is None)
        check.invariant(outputs is None)
        check.invariant(description is None)
        check.invariant(config_field is None)
        return _Solid()(name)

    return _Solid(
        name=name,
        inputs=inputs,
        outputs=outputs,
        config_field=config_field,
        description=description,
    )

def _create_lambda_solid_transform_wrapper(fn, input_defs, output_def):
    check.callable_param(fn, 'fn')
    check.list_param(input_defs, 'input_defs', of_type=InputDefinition)
    check.inst_param(output_def, 'output_def', OutputDefinition)

    input_names = [
        input_def.name for input_def in input_defs if not input_def.runtime_type.is_nothing
    ]

    @wraps(fn)
    def transform(_context, inputs):
        kwargs = {}
        for input_name in input_names:
            kwargs[input_name] = inputs[input_name]

        result = fn(**kwargs)
        yield Result(value=result, output_name=output_def.name)

    return transform

def _create_solid_transform_wrapper(fn, input_defs, output_defs):
    check.callable_param(fn, 'fn')
    check.list_param(input_defs, 'input_defs', of_type=InputDefinition)
    check.list_param(output_defs, 'output_defs', of_type=OutputDefinition)

    input_names = [
        input_def.name for input_def in input_defs if not input_def.runtime_type.is_nothing
    ]

    @wraps(fn)
    def transform(context, inputs):
        kwargs = {}
        for input_name in input_names:
            kwargs[input_name] = inputs[input_name]

        result = fn(context, **kwargs)

        if inspect.isgenerator(result):
            for item in result:
                yield item
        else:
            if isinstance(result, Result):
                yield result
            elif isinstance(result, MultipleResults):
                for item in result.results:
                    yield item
            elif len(output_defs) == 1:
                yield Result(value=result, output_name=output_defs[0].name)
            elif result is not None:
                if not output_defs:
                    raise DagsterInvariantViolationError(
                        (
                            'Solid unexpectedly returned output {result} of type {type_}. Solid is '
                            'explicitly defined to return no results.'
                        ).format(result=result, type_=type(result))
                    )
                else:
                    raise DagsterInvariantViolationError(
                        (
                            'Solid unexpectedly returned output {result} of type {type_}. Should '
                            'be a MultipleResults object, or a generator, containing or yielding '
                            '{n_results} results: {{{expected_results}}}.'
                        ).format(
                            result=result,
                            type_=type(result),
                            n_results=len(output_defs),
                            expected_results=', '.join(
                                [
                                    '\'{result_name}\': {runtime_type}'.format(
                                        result_name=output_def.name,
                                        runtime_type=output_def.runtime_type,
                                    )
                                    for output_def in output_defs
                                ]
                            ),
                        )
                    )

    return transform

class FunctionValidationError(Exception):
    TYPES = {'vararg': 1, 'missing_name': 2, 'missing_positional': 3, 'extra': 4}

    def __init__(self, error_type, param=None, missing_names=None, **kwargs):
        super(FunctionValidationError, self).__init__(**kwargs)
        self.error_type = error_type
        self.param = param
        self.missing_names = missing_names

def _validate_transform_fn(solid_name, transform_fn, inputs, expected_positionals=None):
    check.str_param(solid_name, 'solid_name')
    check.callable_param(transform_fn, 'transform_fn')
    check.list_param(inputs, 'inputs', of_type=InputDefinition)
    expected_positionals = check.opt_list_param(
        expected_positionals, 'expected_positionals', of_type=(str, tuple)
    )

    names = set(inp.name for inp in inputs if not inp.runtime_type.is_nothing)
    # Currently being super strict about naming. Might be a good idea to relax. Starting strict.
    try:
        _validate_decorated_fn(transform_fn, names, expected_positionals)
    except FunctionValidationError as e:
        if e.error_type == FunctionValidationError.TYPES['vararg']:
            raise DagsterInvalidDefinitionError(
                "solid '{solid_name}' transform function has positional vararg parameter "
                "'{e.param}'. Transform functions should only have keyword arguments that match "
                "input names and a first positional parameter named 'context'.".format(
                    solid_name=solid_name, e=e
                )
            )
        elif e.error_type == FunctionValidationError.TYPES['missing_name']:
            raise DagsterInvalidDefinitionError(
                "solid '{solid_name}' transform function has parameter '{e.param}' that is not "
                "one of the solid inputs. Transform functions should only have keyword arguments "
                "that match input names and a first positional parameter named 'context'.".format(
                    solid_name=solid_name, e=e
                )
            )
        elif e.error_type == FunctionValidationError.TYPES['missing_positional']:
            raise DagsterInvalidDefinitionError(
                "solid '{solid_name}' transform function do not have required positional "
                "parameter '{e.param}'. Transform functions should only have keyword arguments "
                "that match input names and a first positional parameter named 'context'.".format(
                    solid_name=solid_name, e=e
                )
            )
        elif e.error_type == FunctionValidationError.TYPES['extra']:
            undeclared_inputs_printed = ", '".join(e.missing_names)
            raise DagsterInvalidDefinitionError(
                "solid '{solid_name}' transform function do not have parameter(s) "
                "'{undeclared_inputs_printed}', which are in solid's inputs. Transform functions "
                "should only have keyword arguments that match input names and a first positional "
                "parameter named 'context'.".format(
                    solid_name=solid_name, undeclared_inputs_printed=undeclared_inputs_printed
                )
            )
        else:
            raise e

def _validate_decorated_fn(fn, names, expected_positionals):
    used_inputs = set()
    has_kwargs = False

    signature = funcsigs.signature(fn)
    params = list(signature.parameters.values())

    expected_positional_params = params[0 : len(expected_positionals)]
    other_params = params[len(expected_positionals) :]

    for expected_names, actual in zip(expected_positionals, expected_positional_params):
        possible_names = []
        for expected in expected_names:
            possible_names.extend(
                [
                    '_',
                    expected,
                    '_{expected}'.format(expected=expected),
                    '{expected}_'.format(expected=expected),
                ]
            )
        if (
            actual.kind
            not in [funcsigs.Parameter.POSITIONAL_OR_KEYWORD, funcsigs.Parameter.POSITIONAL_ONLY]
        ) or (actual.name not in possible_names):
            raise FunctionValidationError(
                FunctionValidationError.TYPES['missing_positional'], param=expected_names
            )

    for param in other_params:
        if param.kind == funcsigs.Parameter.VAR_KEYWORD:
            has_kwargs = True
        elif param.kind == funcsigs.Parameter.VAR_POSITIONAL:
            raise FunctionValidationError(error_type=FunctionValidationError.TYPES['vararg'])

        else:
            if param.name not in names:
                raise FunctionValidationError(
                    FunctionValidationError.TYPES['missing_name'], param=param.name
                )
            else:
                used_inputs.add(param.name)

    undeclared_inputs = names - used_inputs
    if not has_kwargs and undeclared_inputs:
        raise FunctionValidationError(
            FunctionValidationError.TYPES['extra'], missing_names=undeclared_inputs
        )


from dagster import check
from dagster.core.definitions import Solid, PipelineDefinition

from .objects import (
    ExecutionValueSubplan,
    ExecutionStep,
    StepInput,
    StepOutput,
    StepOutputHandle,
    StepOutputValue,
    StepKind,
)

JOIN_OUTPUT = 'join_output'

def __join_lambda(_context, inputs):
    yield StepOutputValue(output_name=JOIN_OUTPUT, value=list(inputs.values())[0])

def create_join_step(pipeline_def, solid, step_key, prev_steps, prev_output_name):
    check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
    check.inst_param(solid, 'solid', Solid)
    check.str_param(step_key, 'step_key')
    check.list_param(prev_steps, 'prev_steps', of_type=ExecutionStep)
    check.invariant(len(prev_steps) > 0)
    check.str_param(prev_output_name, 'output_name')

    step_inputs = []
    seen_runtime_type = None
    seen_optionality = None
    for prev_step in prev_steps:
        prev_step_output = prev_step.step_output_named(prev_output_name)

        if seen_runtime_type is None:
            seen_runtime_type = prev_step_output.runtime_type
        else:
            check.invariant(seen_runtime_type == prev_step_output.runtime_type)

        if seen_optionality is None:
            seen_optionality = prev_step_output.optional
        else:
            check.invariant(seen_optionality == prev_step_output.optional)

        output_handle = StepOutputHandle.from_step(prev_step, prev_output_name)

        step_inputs.append(StepInput(prev_step.key, prev_step_output.runtime_type, output_handle))

    return ExecutionStep(
        pipeline_name=pipeline_def.name,
        key=step_key,
        step_inputs=step_inputs,
        step_outputs=[StepOutput(JOIN_OUTPUT, seen_runtime_type, optional=seen_optionality)],
        compute_fn=__join_lambda,
        kind=StepKind.JOIN,
        solid=solid,
    )

def create_joining_subplan(
    pipeline_def, solid, join_step_key, parallel_steps, parallel_step_output
):
    '''
    This captures a common pattern of fanning out a single value to N steps,
    where each step has similar structure. The strict requirement here is that each step
    must provide an output named the parameters parallel_step_output.

    This takes those steps and then uses a join node to coalesce them so that downstream
    steps can depend on a single output.

    Currently the join step just does a passthrough with no computation. It remains
    to be seen if there should be any work or verification done in this step, especially
    in multi-process environments that require persistence between steps.
    '''
    check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
    check.inst_param(solid, 'solid', Solid)
    check.str_param(join_step_key, 'join_step_key')
    check.list_param(parallel_steps, 'parallel_steps', of_type=ExecutionStep)
    check.str_param(parallel_step_output, 'parallel_step_output')

    for parallel_step in parallel_steps:
        check.invariant(parallel_step.has_step_output(parallel_step_output))

    join_step = create_join_step(
        pipeline_def, solid, join_step_key, parallel_steps, parallel_step_output
    )

    output_name = join_step.step_outputs[0].name
    return ExecutionValueSubplan(
        parallel_steps + [join_step], StepOutputHandle.from_step(join_step, output_name)
    )


"""Pipeline definitions for the airline_demo."""

import gzip
import os
import shutil

from dagster import (
    lambda_solid,
    DependencyDefinition,
    InputDefinition,
    OutputDefinition,
    List,
    PipelineDefinition,
    SolidInstance,
    String,
)
from dagster.utils import safe_isfile, mkdir_p

from dagster_spark import SparkSolidDefinition
from dagster_snowflake import (
    SnowflakeSolidDefinition,
    SnowflakeLoadSolidDefinition,
)
from dagster_aws import download_from_s3

@lambda_solid(
    inputs=[InputDefinition('gzip_file', String)],
    output=OutputDefinition(List(String)),
)
def gunzipper(gzip_file):
    '''gunzips /path/to/foo.gz to /path/to/raw/2019/01/01/data.json
    '''
    # TODO: take date as an input

    path_prefix = os.path.dirname(gzip_file)
    output_folder = os.path.join(path_prefix, 'raw/2019/01/01')
    outfile = os.path.join(output_folder, 'data.json')

    if not safe_isfile(outfile):
        mkdir_p(output_folder)

        with gzip.open(gzip_file, 'rb') as f_in, open(outfile, 'wb') as f_out:
            shutil.copyfileobj(f_in, f_out)

    return [path_prefix]

def define_event_ingest_pipeline():
    event_ingest = SparkSolidDefinition(
        name='event_ingest',
        main_class='io.dagster.events.EventPipeline',
        description='Ingest events from JSON to Parquet',
    )

    # TODO: express dependency of this solid on event_ingest
    snowflake_load = SnowflakeLoadSolidDefinition(
        'snowflake_load',
        # TODO: need to pull this out to a config
        src='file:///tmp/dagster/events/data/output/2019/01/01/*.parquet',
        table='events',
    )

    return PipelineDefinition(
        name='event_ingest_pipeline',
        solids=[download_from_s3, gunzipper, event_ingest, snowflake_load],
        dependencies={
            SolidInstance('gunzipper'): {
                'gzip_file': DependencyDefinition('download_from_s3')
            },
            SolidInstance('event_ingest'): {
                'spark_inputs': DependencyDefinition('gunzipper')
            },
            SolidInstance('snowflake_load'): {
                SnowflakeSolidDefinition.INPUT_READY: DependencyDefinition(
                    'event_ingest', 'paths'
                )
            },
        },
    )


import inspect

from future.utils import raise_with_traceback

from six import string_types

class CheckError(Exception):
    pass

class ParameterCheckError(CheckError):
    pass

class ElementCheckError(CheckError):
    pass

class NotImplementedCheckError(CheckError):
    pass

def _param_type_mismatch_exception(obj, ttype, param_name):
    if isinstance(ttype, tuple):
        type_names = sorted([t.__name__ for t in ttype])
        return ParameterCheckError(
            'Param "{name}" is not one of {type_names}. Got {obj} which is type {obj_type}.'.format(
                name=param_name, obj=repr(obj), type_names=type_names, obj_type=type(obj)
            )
        )
    else:
        return ParameterCheckError(
            'Param "{name}" is not a {type}. Got {obj} which is type {obj_type}.'.format(
                name=param_name, obj=repr(obj), type=ttype.__name__, obj_type=type(obj)
            )
        )

def _not_type_param_subclass_mismatch_exception(obj, param_name):
    return ParameterCheckError(
        'Param "{name}" was supposed to be a type. Got {obj} instead of type {obj_type}'.format(
            name=param_name, obj=repr(obj), obj_type=type(obj)
        )
    )

def _param_subclass_mismatch_exception(obj, superclass, param_name):
    return ParameterCheckError(
        'Param "{name}" is a type but not a subclass of {superclass}. Got {obj} instead'.format(
            name=param_name, superclass=superclass, obj=obj
        )
    )

def _type_mismatch_error(obj, ttype, desc=None):
    if desc:
        return CheckError(
            'Object {obj} is not a {type}. Got {obj} with type {obj_type}. Desc: {desc}'.format(
                obj=repr(obj), type=ttype.__name__, obj_type=type(obj), desc=desc
            )
        )
    else:
        return CheckError(
            'Object {obj} is not a {type}. Got {obj} with type {obj_type}.'.format(
                obj=repr(obj), type=ttype.__name__, obj_type=type(obj)
            )
        )

def _not_callable_exception(obj, param_name):
    return ParameterCheckError(
        'Param "{name}" is not callable. Got {obj} with type {obj_type}.'.format(
            name=param_name, obj=repr(obj), obj_type=type(obj)
        )
    )

def _param_invariant_exception(param_name, desc):
    return ParameterCheckError(
        'Invariant violation for parameter {param_name}. Description: {desc}'.format(
            param_name=param_name, desc=desc
        )
    )

def failed(desc):
    if not _is_str(desc):
        raise_with_traceback(CheckError('desc argument must be a string'))

    raise_with_traceback(CheckError('Failure condition: {desc}'.format(desc=desc)))

def not_implemented(desc):
    if not _is_str(desc):
        raise_with_traceback(CheckError('desc argument must be a string'))

    raise_with_traceback(NotImplementedCheckError('Not implemented: {desc}'.format(desc=desc)))

def inst(obj, ttype, desc=None):
    if not isinstance(obj, ttype):
        raise_with_traceback(_type_mismatch_error(obj, ttype, desc))
    return obj

def is_callable(obj, desc=None):
    if not callable(obj):
        if desc:
            raise_with_traceback(
                CheckError(
                    'Must be callable. Got {obj}. Description: {desc}'.format(
                        obj=repr(obj), desc=desc
                    )
                )
            )
        else:
            raise_with_traceback(
                CheckError(
                    'Must be callable. Got {obj}. Description: {desc}'.format(obj=obj, desc=desc)
                )
            )

    return obj

def not_none_param(obj, param_name):
    if obj is None:
        raise_with_traceback(
            _param_invariant_exception(
                param_name, 'Param {param_name} cannot be none'.format(param_name=param_name)
            )
        )
    return obj

def invariant(condition, desc=None):
    if not condition:
        if desc:
            raise_with_traceback(
                CheckError('Invariant failed. Description: {desc}'.format(desc=desc))
            )
        else:
            raise_with_traceback(CheckError('Invariant failed.'))

    return True

def param_invariant(condition, param_name, desc=None):
    if not condition:
        raise_with_traceback(_param_invariant_exception(param_name, desc))

def inst_param(obj, param_name, ttype):
    if not isinstance(obj, ttype):
        raise_with_traceback(_param_type_mismatch_exception(obj, ttype, param_name))
    return obj

def opt_inst_param(obj, param_name, ttype, default=None):
    if obj is not None and not isinstance(obj, ttype):
        raise_with_traceback(_param_type_mismatch_exception(obj, ttype, param_name))
    return default if obj is None else obj

def callable_param(obj, param_name):
    if not callable(obj):
        raise_with_traceback(_not_callable_exception(obj, param_name))
    return obj

def opt_callable_param(obj, param_name, default=None):
    if obj is not None and not callable(obj):
        raise_with_traceback(_not_callable_exception(obj, param_name))
    return default if obj is None else obj

def int_param(obj, param_name):
    if not isinstance(obj, int):
        raise_with_traceback(_param_type_mismatch_exception(obj, int, param_name))
    return obj

def int_value_param(obj, value, param_name):
    if not isinstance(obj, int):
        raise_with_traceback(_param_type_mismatch_exception(obj, int, param_name))
    if obj != value:
        raise_with_traceback(
            _param_invariant_exception(param_name, "Should be equal to {value}".format(value=value))
        )
    return obj

def opt_int_param(obj, param_name):
    if obj is not None and not isinstance(obj, int):
        raise_with_traceback(_param_type_mismatch_exception(obj, int, param_name))
    return obj

def float_param(obj, param_name):
    if not isinstance(obj, float):
        raise_with_traceback(_param_type_mismatch_exception(obj, float, param_name))
    return obj

def opt_float_param(obj, param_name):
    if obj is not None and not isinstance(obj, float):
        raise_with_traceback(_param_type_mismatch_exception(obj, float, param_name))
    return obj

def _is_str(obj):
    return isinstance(obj, string_types)

def str_param(obj, param_name):
    if not _is_str(obj):
        raise_with_traceback(_param_type_mismatch_exception(obj, str, param_name))
    return obj

def opt_str_param(obj, param_name, default=None):
    if obj is not None and not isinstance(obj, string_types):
        raise_with_traceback(_param_type_mismatch_exception(obj, str, param_name))
    return default if obj is None else obj

def opt_nonempty_str_param(obj, param_name, default=None):
    if obj is not None and not isinstance(obj, string_types):
        raise_with_traceback(_param_type_mismatch_exception(obj, str, param_name))
    return default if obj is None or obj == '' else obj

def bool_param(obj, param_name):
    if not isinstance(obj, bool):
        raise_with_traceback(_param_type_mismatch_exception(obj, bool, param_name))
    return obj

def opt_bool_param(obj, param_name, default=None):
    if obj is not None and not isinstance(obj, bool):
        raise_with_traceback(_param_type_mismatch_exception(obj, bool, param_name))
    return default if obj is None else obj

def is_list(obj_list, of_type=None, desc=None):
    if not isinstance(obj_list, list):
        raise_with_traceback(_type_mismatch_error(obj_list, list, desc))

    if not of_type:
        return obj_list

    return _check_list_items(obj_list, of_type)

def list_param(obj_list, param_name, of_type=None):
    if not isinstance(obj_list, list):
        raise_with_traceback(_param_type_mismatch_exception(obj_list, list, param_name))

    if not of_type:
        return obj_list

    return _check_list_items(obj_list, of_type)

def tuple_param(obj, param_name):
    if not isinstance(obj, tuple):
        raise_with_traceback(_param_type_mismatch_exception(obj, tuple, param_name))
    return obj

def opt_tuple_param(obj, param_name, default=None):
    if obj is not None and not isinstance(obj, tuple):
        raise_with_traceback(_param_type_mismatch_exception(obj, tuple, param_name))
    return default if obj is None else obj

def _check_list_items(obj_list, of_type):
    if of_type is str:
        of_type = string_types

    for obj in obj_list:

        if not isinstance(obj, of_type):
            raise_with_traceback(
                CheckError(
                    'Member of list mismatches type. Expected {of_type}. Got {obj_repr} of type '
                    '{obj_type}.'.format(of_type=of_type, obj_repr=repr(obj), obj_type=type(obj))
                )
            )
    return obj_list

def opt_list_param(obj_list, param_name, of_type=None):
    '''Ensures argument obj_list is a list or None; in the latter case, instantiates an empty list
    and returns it.

    If the of_type argument is provided, also ensures that list items conform to the type specified
    by of_type.
    '''
    if obj_list is not None and not isinstance(obj_list, list):
        raise_with_traceback(_param_type_mismatch_exception(obj_list, list, param_name))
    if not obj_list:
        return []
    if not of_type:
        return obj_list

    return _check_list_items(obj_list, of_type)

def opt_nullable_list_param(obj_list, param_name, of_type=None):
    '''Ensures argument obj_list is a list or None. Returns None if input is None.

    If the of_type argument is provided, also ensures that list items conform to the type specified
    by of_type.
    '''
    if obj_list is not None and not isinstance(obj_list, list):
        raise_with_traceback(_param_type_mismatch_exception(obj_list, list, param_name))
    if not obj_list:
        return None if obj_list is None else []
    if not of_type:
        return obj_list

    return _check_list_items(obj_list, of_type)

def _check_key_value_types(obj, key_type, value_type, key_check=isinstance, value_check=isinstance):
    '''Ensures argument obj is a dictionary, and enforces that the keys/values conform to the types
    specified by key_type, value_type.
    '''
    if not isinstance(obj, dict):
        raise_with_traceback(_type_mismatch_error(obj, dict))

    if key_type is str:
        key_type = string_types

    if value_type is str:
        value_type = string_types

    for key, value in obj.items():
        if key_type and not key_check(key, key_type):
            raise_with_traceback(
                CheckError(
                    'Key in dictionary mismatches type. Expected {key_type}. Got {obj_repr}'.format(
                        key_type=repr(key_type), obj_repr=repr(key)
                    )
                )
            )
        if value_type and not value_check(value, value_type):
            raise_with_traceback(
                CheckError(
                    'Value in dictionary mismatches expected type for key {key}. Expected value '
                    'of type {vtype}. Got value {value} of type {obj_type}.'.format(
                        vtype=repr(value_type), obj_type=type(value), key=key, value=value
                    )
                )
            )
    return obj

def dict_param(obj, param_name, key_type=None, value_type=None):
    '''Ensures argument obj is a native Python dictionary, raises an exception if not, and otherwise
    returns obj.
    '''
    if not isinstance(obj, dict):
        raise_with_traceback(_param_type_mismatch_exception(obj, dict, param_name))

    if not (key_type or value_type):
        return obj

    return _check_key_value_types(obj, key_type, value_type)

def opt_dict_param(obj, param_name, key_type=None, value_type=None, value_class=None):
    '''Ensures argument obj is either a dictionary or None; if the latter, instantiates an empty
    dictionary.
    '''
    if obj is not None and not isinstance(obj, dict):
        raise_with_traceback(_param_type_mismatch_exception(obj, dict, param_name))

    if not obj:
        return {}

    if value_class:
        return _check_key_value_types(obj, key_type, value_type=value_class, value_check=issubclass)
    return _check_key_value_types(obj, key_type, value_type)

def opt_nullable_dict_param(obj, param_name, key_type=None, value_type=None, value_class=None):
    '''Ensures argument obj is either a dictionary or None;
    '''
    if obj is not None and not isinstance(obj, dict):
        raise_with_traceback(_param_type_mismatch_exception(obj, dict, param_name))

    if not obj:
        return None if obj is None else {}

    if value_class:
        return _check_key_value_types(obj, key_type, value_type=value_class, value_check=issubclass)
    return _check_key_value_types(obj, key_type, value_type)

def _check_two_dim_key_value_types(obj, key_type, _param_name, value_type):
    _check_key_value_types(obj, key_type, dict)  # check level one

    for level_two_dict in obj.values():
        _check_key_value_types(level_two_dict, key_type, value_type)  # check level two

    return obj

def two_dim_dict_param(obj, param_name, key_type=string_types, value_type=None):
    if not isinstance(obj, dict):
        raise_with_traceback(_param_type_mismatch_exception(obj, dict, param_name))

    return _check_two_dim_key_value_types(obj, key_type, param_name, value_type)

def opt_two_dim_dict_param(obj, param_name, key_type=string_types, value_type=None):
    if obj is not None and not isinstance(obj, dict):
        raise_with_traceback(_param_type_mismatch_exception(obj, dict, param_name))

    if not obj:
        return {}

    return _check_two_dim_key_value_types(obj, key_type, param_name, value_type)

def type_param(obj, param_name):
    if not isinstance(obj, type):
        raise_with_traceback(_not_type_param_subclass_mismatch_exception(obj, param_name))
    return obj

def opt_type_param(obj, param_name, default=None):
    if obj is not None and not isinstance(obj, type):
        raise_with_traceback(_not_type_param_subclass_mismatch_exception(obj, param_name))
    return obj if obj is not None else default

def subclass_param(obj, param_name, superclass):
    type_param(obj, param_name)
    if not issubclass(obj, superclass):
        raise_with_traceback(_param_subclass_mismatch_exception(obj, superclass, param_name))

    return obj

def _element_check_error(key, value, ddict, ttype):
    return ElementCheckError(
        'Value {value} from key {key} is not a {ttype}. Dict: {ddict}'.format(
            key=key, value=repr(value), ddict=repr(ddict), ttype=repr(ttype)
        )
    )

def generator(obj):
    if not inspect.isgenerator(obj):
        raise ParameterCheckError(
            'Not a generator (return value of function that yields) Got {obj} instead'.format(
                obj=obj
            )
        )
    return obj

def opt_generator(obj):
    if obj is not None and not inspect.isgenerator(obj):
        raise ParameterCheckError(
            'Not a generator (return value of function that yields) Got {obj} instead'.format(
                obj=obj
            )
        )
    return obj

def generator_param(obj, param_name):
    if not inspect.isgenerator(obj):
        raise ParameterCheckError(
            (
                'Param "{name}" is not a generator (return value of function that yields) Got '
                '{obj} instead'
            ).format(name=param_name, obj=obj)
        )
    return obj

def opt_generator_param(obj, param_name):
    if obj is not None and not inspect.isgenerator(obj):
        raise ParameterCheckError(
            (
                'Param "{name}" is not a generator (return value of function that yields) Got '
                '{obj} instead'
            ).format(name=param_name, obj=obj)
        )
    return obj

def list_elem(ddict, key):
    dict_param(ddict, 'ddict')
    str_param(key, 'key')

    value = ddict.get(key)
    if not isinstance(value, list):
        raise_with_traceback(_element_check_error(key, value, ddict, list))
    return value

def opt_list_elem(ddict, key):
    dict_param(ddict, 'ddict')
    str_param(key, 'key')

    value = ddict.get(key)

    if value is None:
        return []

    if not isinstance(value, list):
        raise_with_traceback(_element_check_error(key, value, ddict, list))
    return value

def dict_elem(ddict, key):
    dict_param(ddict, 'ddict')
    str_param(key, 'key')

    if key not in ddict:
        raise_with_traceback(
            CheckError('{key} not present in dictionary {ddict}'.format(key=key, ddict=ddict))
        )

    value = ddict[key]
    if not isinstance(value, dict):
        raise_with_traceback(_element_check_error(key, value, ddict, dict))
    return value

def opt_dict_elem(ddict, key):
    dict_param(ddict, 'ddict')
    str_param(key, 'key')

    value = ddict.get(key)

    if value is None:
        return {}

    if not isinstance(value, dict):
        raise_with_traceback(_element_check_error(key, value, ddict, list))

    return value

def bool_elem(ddict, key):
    dict_param(ddict, 'ddict')
    str_param(key, 'key')

    value = ddict[key]
    if not isinstance(value, bool):
        raise_with_traceback(_element_check_error(key, value, ddict, bool))
    return value

def opt_str_elem(ddict, key):
    dict_param(ddict, 'ddict')
    str_param(key, 'key')

    value = ddict.get(key)
    if value is None:
        return None
    if not _is_str(value):
        raise_with_traceback(_element_check_error(key, value, ddict, str))
    return value

def str_elem(ddict, key):
    dict_param(ddict, 'ddict')
    str_param(key, 'key')

    value = ddict[key]
    if not _is_str(value):
        raise_with_traceback(_element_check_error(key, value, ddict, str))
    return value


import json

from dagster import check
from dagster.utils.error import SerializableErrorInfo

from dagster.utils.logging import (
    DEBUG,
    StructuredLoggerHandler,
    JsonEventLoggerHandler,
    StructuredLoggerMessage,
    check_valid_level_param,
    construct_single_handler_logger,
)

class EventRecord(object):
    def __init__(
        self,
        error_info,
        message,
        level,
        user_message,
        run_id,
        timestamp,
        step_key=None,
        pipeline_name=None,
        dagster_event=None,
    ):
        from dagster.core.events import DagsterEvent

        self._error_info = check.opt_inst_param(error_info, 'error_info', SerializableErrorInfo)
        self._message = check.str_param(message, 'message')
        self._level = check_valid_level_param(level)
        self._user_message = check.str_param(user_message, 'user_message')
        self._run_id = check.str_param(run_id, 'run_id')
        self._timestamp = check.float_param(timestamp, 'timestamp')
        self._step_key = check.opt_str_param(step_key, 'step_key')
        self._pipeline_name = check.opt_str_param(pipeline_name, 'pipeline_name')
        self._dagster_event = check.opt_inst_param(dagster_event, 'dagster_event', DagsterEvent)

    @property
    def message(self):
        return self._message

    @property
    def level(self):
        return self._level

    @property
    def user_message(self):
        return self._user_message

    @property
    def is_dagster_event(self):
        return bool(self._dagster_event)

    @property
    def run_id(self):
        return self._run_id

    @property
    def timestamp(self):
        return self._timestamp

    @property
    def error_info(self):
        return self._error_info

    @property
    def step_key(self):
        return self._step_key

    @property
    def pipeline_name(self):
        return self._pipeline_name

    @property
    def dagster_event(self):
        return self._dagster_event

    def to_dict(self):
        return {
            'run_id': self.run_id,
            'message': self.message,
            'level': self.level,
            'user_message': self.user_message,
            'timestamp': self.timestamp,
            'error_info': self.error_info,
            'step_key': self.step_key,
            'dagster_event': self.dagster_event,
            'pipeline_name': self.pipeline_name,
        }

class DagsterEventRecord(EventRecord):
    pass

class LogMessageRecord(EventRecord):
    pass

def construct_error_info(logger_message):
    if 'error_info' not in logger_message.meta:
        return None

    raw_error_info = logger_message.meta['error_info']

    message, stack = json.loads(raw_error_info)

    return SerializableErrorInfo(message, stack)

def construct_event_record(logger_message):
    check.inst_param(logger_message, 'logger_message', StructuredLoggerMessage)

    log_record_cls = LogMessageRecord
    if logger_message.meta.get('dagster_event'):
        log_record_cls = DagsterEventRecord

    return log_record_cls(
        message=logger_message.message,
        level=logger_message.level,
        user_message=logger_message.meta['orig_message'],
        run_id=logger_message.meta['run_id'],
        timestamp=logger_message.record.created,
        step_key=logger_message.meta.get('step_key'),
        pipeline_name=logger_message.meta.get('pipeline_name'),
        dagster_event=logger_message.meta.get('dagster_event'),
        error_info=construct_error_info(logger_message),
    )

def construct_event_logger(event_record_callback):
    '''
    Callback receives a stream of event_records
    '''
    check.callable_param(event_record_callback, 'event_record_callback')

    return construct_single_handler_logger(
        'event-logger',
        DEBUG,
        StructuredLoggerHandler(
            lambda logger_message: event_record_callback(construct_event_record(logger_message))
        ),
    )

def construct_json_event_logger(json_path):
    '''Record a stream of event records to json'''
    check.str_param(json_path, 'json_path')
    return construct_single_handler_logger(
        "json-event-record-logger",
        DEBUG,
        JsonEventLoggerHandler(
            json_path,
            lambda record: construct_event_record(
                StructuredLoggerMessage(
                    name=record.name,
                    message=record.msg,
                    level=record.levelno,
                    meta=record.dagster_meta,
                    record=record,
                )
            ),
        ),
    )