"""Utils"""

def iterate_fields(fields, schema):
    """Recursively iterate over all DictField sub-fields.

    :param fields: Field instance (e.g. input)
    :type fields: dict
    :param schema: Schema instance (e.g. input_schema)
    :type schema: dict

    """
    schema_dict = {val['name']: val for val in schema}
    for field_id, properties in fields.iteritems():
        if 'group' in schema_dict[field_id]:
            for _field_schema, _fields in iterate_fields(properties, schema_dict[field_id]['group']):
                yield (_field_schema, _fields)
        else:
            yield (schema_dict[field_id], fields)

def find_field(schema, field_name):
    """Find field in schema by field name.

    :param schema: Schema instance (e.g. input_schema)
    :type schema: dict
    :param field_name: Field name
    :type field_name: string

    """
    for field in schema:
        if field['name'] == field_name:
            return field

def iterate_schema(fields, schema, path=None):
    """Recursively iterate over all schema sub-fields.

    :param fields: Field instance (e.g. input)
    :type fields: dict
    :param schema: Schema instance (e.g. input_schema)
    :type schema: dict
    :path schema: Field path
    :path schema: string

    """
    for field_schema in schema:
        name = field_schema['name']
        if 'group' in field_schema:
            for rvals in iterate_schema(fields[name] if name in fields else {},
                                        field_schema['group'],
                                        None if path is None else '{}.{}'.format(path, name)):
                yield rvals
        else:
            if path is None:
                yield (field_schema, fields)
            else:
                yield (field_schema, fields, '{}.{}'.format(path, name))


# -*- coding: utf-8 -*-
# Copyright (C) 2012 by Tomasz Wójcik <labs@tomekwojcik.pl>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

"""Generate random `Lorem ipsum` style text."""

import random
import re

from ..dictionaries_loader import get_dictionary

__all__ = [
    'word', 'words', 'title', 'sentence',
    'sentences', 'paragraph', 'paragraphs'
]

_words = None

def word():
    """Random word."""
    return words(quantity=1)

def words(quantity=10, as_list=False):
    """Random words."""
    global _words

    if not _words:
        _words = ' '.join(get_dictionary('lorem_ipsum')).lower().\
            replace('\n', '')
        _words = re.sub(r'\.|,|;/', '', _words)
        _words = _words.split(' ')

    result = random.sample(_words, quantity)

    if as_list:
        return result
    else:
        return ' '.join(result)

def title(words_quantity=4):
    """Random sentence to be used as e.g. an e-mail subject."""
    result = words(quantity=words_quantity)
    result += random.choice('?.!')
    return result.capitalize()

def sentence():
    """Random sentence."""
    return sentences(quantity=1)

def sentences(quantity=2, as_list=False):
    """Random sentences."""
    result = [sntc.strip() for sntc in
              random.sample(get_dictionary('lorem_ipsum'), quantity)]

    if as_list:
        return result
    else:
        return ' '.join(result)

def paragraph(separator='\n\n', wrap_start='', wrap_end='',
              html=False, sentences_quantity=3):
    """Random paragraph."""
    return paragraphs(quantity=1, separator=separator, wrap_start=wrap_start,
                      wrap_end=wrap_end, html=html,
                      sentences_quantity=sentences_quantity)

def paragraphs(quantity=2, separator='\n\n', wrap_start='', wrap_end='',
               html=False, sentences_quantity=3, as_list=False):
    """Random paragraphs."""
    if html:
        wrap_start = '<p>'
        wrap_end = '</p>'
        separator = '\n\n'

    result = []
    for i in xrange(0, quantity):
        result.append(wrap_start + sentences(sentences_quantity) + wrap_end)

    if as_list:
        return result
    else:
        return separator.join(result)


# -*- coding: utf-8 -*-
# Copyright (C) 2012 by Tomasz Wójcik <labs@tomekwojcik.pl>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

"""Generate misc random data."""

import random
import string

HEX_DIGITS = string.hexdigits[:-6].upper()

__all__ = ['hex_color', 'hex_color_short', 'text']

def hex_color():
    """Random HEX color."""
    return ''.join(random.sample(HEX_DIGITS, 6))

def hex_color_short():
    """Random short (e.g. `FFF` color)."""
    return ''.join(random.sample(HEX_DIGITS, 3))

def text(length=None, at_least=10, at_most=15, lowercase=True,
         uppercase=True, digits=True, spaces=True, punctuation=False):
    """
    Random text.

    If `length` is present the text will be exactly this chars long. Else the
    text will be something between `at_least` and `at_most` chars long.
    """
    base_string = ''
    if lowercase:
        base_string += string.ascii_lowercase

    if uppercase:
        base_string += string.ascii_uppercase

    if digits:
        base_string += string.digits

    if spaces:
        base_string += ' '

    if punctuation:
        base_string += string.punctuation

    if len(base_string) == 0:
        return ''

    if not length:
        length = random.randint(at_least, at_most)

    result = ''
    for i in xrange(0, length):
        result += random.choice(base_string)

    return result


# -*- coding: utf-8 -*-
"""
sync action

Syncs completion status of issues and their matched tasks.

"""

import logging
import re
import collections

from .. import transport

from ..action import Action

ASANA_ID_RE = re.compile(r'#(\d{12,16})', re.M)
"""Regular expression for capturing asana IDs."""

_ms_label = lambda x: "_ms:%d"%x
"""Converts a milestone id into an _ms prefixed string"""

class Sync(Action):
    """Syncs completion status of issues and their matched tasks."""

    # name of action
    name = "sync"

    @classmethod
    def add_arguments(cls, parser):
        """Add arguments to the parser for collection in app.args.

        Args:
            parser:
                `argparse.ArgumentParser`. Parser.
                Arguments added here are server on
                self.args.
        """

        parser.add_argument(
            '-c', '--create-missing-tasks',
            action='store_true',
            dest='create_missing_tasks',
            help="[sync] create asana tasks for issues without tasks"
            )

        parser.add_argument(
            '-l', '--sync-labels',
            action='store_true',
            dest='sync_labels',
            help="[sync] sync labels and milestones for each issue"
            )

    def apply_tasks_to_issue(self, issue, tasks, issue_body=None):
        """Applies task numbers to an issue."""
        issue_body = issue_body or issue.body
        task_numbers = transport.format_task_numbers_with_links(tasks)
        if task_numbers:
            new_body = transport.ASANA_SECTION_RE.sub('', issue_body)
            new_body = new_body + "\n## Asana Tasks:\n\n%s" % task_numbers
            transport.issue_edit(issue,
                                 body=new_body)
            return new_body

        return issue_body

    def sync_labels(self, repo):
        """Creates a local map of github labels/milestones to asana tags."""

        logging.info("syncing new github.com labels to tags")

        # create label tag map
        ltm = self.app.data.get("label-tag-map", {})

        # loop over labels, if they don't have tags, make them
        for label in repo.get_labels():
            tag_id = ltm.get(label.name, None)
            if tag_id is None:

                tag = self.app.asana.tags.create(name=label.name,
                                      workspace=self.asana_ws_id,
                                      notes="gh: %s" % label.url
                                      )

                logging.info("\t%s => tag %d", label.name, tag['id'])
                ltm[label.name] = tag['id']

        # loop over milestones, if they don't have tags, make them
        for ms in repo.get_milestones(state="all"):
            tag_id = ltm.get(_ms_label(ms.id), None)
            if tag_id is None:

                tag = self.app.asana.tags.create(name=ms.title,
                                      workspace=self.asana_ws_id,
                                      notes="gh: %s" % ms.url
                                      )

                logging.info("\t%s => tag %d", ms.title, tag['id'])
                ltm[_ms_label(ms.id)] = tag['id']

        self.app.data['label-tag-map'] = ltm
        return ltm

    def run(self):
        app = self.app

        repo, project = self.get_repo_and_project()
        self.asana_ws_id = asana_workspace_id = project['workspace']['id']
        project_id = project['id']

        # Sync project labels <-> asana tags
        if app.args.sync_labels:
            label_tag_map = self.sync_labels(repo)
        else:
            label_tag_map = {}

        # Iterate over the issues in the opposite state as the namespace
        # we are in. We simply want to toggle these guys.
        logging.info("collecting github.com issues")

        # Get the first issue, to limit syncing.
        first_issue = app.data.get('first-issue')

        for issue in repo.get_issues(state="all"):

            # bypass issues < `first-issue` setting.
            if (first_issue is not None and
                issue.number < first_issue):
                logging.debug("stopping at first-issue: %d", first_issue)
                break

            issue_number = str(issue.number)
            issue_body = issue.body
            asana_match = ASANA_ID_RE.search(issue_body)
            multi_match_sections = len(transport.ASANA_SECTION_RE.findall(issue_body)) > 1

            status = "cached"

            # Collect closed and opened tasks known for this issue.
            closed_tasks = \
                app.get_saved_issue_data(issue, 'closed').get('tasks', [])
            open_tasks = \
                app.get_saved_issue_data(issue, 'open').get('tasks', [])

            recorded_tasks = set(open_tasks + closed_tasks)

            # Collect tasks named on github issue
            issue_named_tasks = set()
            for m_grp in ASANA_ID_RE.finditer(issue_body):
                issue_named_tasks.add(int(m_grp.group(1)))

            # Get tasks that are named but missing from cache.
            tasks_to_save_to_this_issue = issue_named_tasks - recorded_tasks
            for task_id in tasks_to_save_to_this_issue:
                status = "collected tasks"
                transport.put_setting("save_issue_data_task",
                                      issue=issue_number,
                                      task_id=task_id,
                                      namespace=issue.state)

            my_tasks = recorded_tasks.union(tasks_to_save_to_this_issue)
            my_tasks = transport.mem.list(my_tasks)

            # Determine if there are multiple groups of ASANA TASKS
            # named.
            if multi_match_sections:
                issue_body = transport.ASANA_SECTION_RE.sub('', issue_body)
                asana_match = None

                issue_body = self.apply_tasks_to_issue(issue, my_tasks,
                    issue_body=issue_body)
                status = "minified issue body"

            # Sync tags and labels
            labels = set()
            if app.args.sync_labels:
                for label in issue.get_labels():
                    labels.add(label.name)
                if issue.milestone:
                    labels.add(_ms_label(issue.milestone.id))

            # If we have tasks already, this issue is cached.
            if recorded_tasks:
                # If the body is missing asana tasks, add all those we know
                # about.
                if not asana_match:
                    # Add tasks if we have any.
                    if recorded_tasks:
                        issue_body = self.apply_tasks_to_issue(issue, my_tasks,
                            issue_body=issue_body)
                        status = "updated with asana #s"

                # If the section isn't formatted... let's reformat it.
                elif not transport.ASANA_SECTION_RE.search(issue_body):
                    issue_body = self.apply_tasks_to_issue(issue, my_tasks,
                        issue_body=issue_body)
                    status = "reformatted asana tasks"

                # Sync tags/labels
                transport.put("sync_tags",
                              tasks=my_tasks,
                              labels=labels,
                              label_tag_map=label_tag_map)

                for task in my_tasks:
                    transport.put('update_task',
                                  task_id=task,
                                  params={'completed': bool(issue.closed_at)})

            # tasks named on issue need to be synced
            elif asana_match and issue_named_tasks:
                status = "connecting tasks"
                self.apply_tasks_to_issue(issue, my_tasks,
                    issue_body=issue_body)

                # Sync tags/labels
                transport.put("sync_tags",
                              tasks=my_tasks,
                              labels=labels,
                              label_tag_map=label_tag_map)

                # Create story
                transport.put("create_story",
                    task_id=task_id,
                    text="Git Issue #%d: \n"
                          "%s" % (
                            issue.number,
                            issue.html_url,
                            )
                    )

                for task in my_tasks:
                    transport.put('update_task',
                                  task_id=task,
                                  params={'completed': bool(issue.closed_at)})

            elif self.args.create_missing_tasks and not issue.pull_request:
                # missing task
                # Create tasks for non-prs
                transport.put("create_missing_task",
                              issue_number=issue.number,
                              issue_state=issue.state,
                              issue_html_url=issue.html_url,
                              issue_body=issue.body,
                              asana_workspace_id=asana_workspace_id,
                              name=issue.title,
                              # TODO: Correct assignee.
                              assignee='me',
                              projects=[project_id],
                              completed=bool(issue.closed_at),
                              tasks=my_tasks,
                              label_tag_map=label_tag_map,
                              labels=labels,
                              )

                status = "new task"
            else:
                status = "no task"

            logging.info("\t%d) %s - %s",
                issue.number, issue.title, status)

        # Flush work.
        app.flush()


from __future__ import unicode_literals
from unittest import TestResult
import sys
import time

from ivoire.compat import indent

class ExampleResult(TestResult):
    """
    Track the outcomes of example runs.

    """

    def __init__(self, formatter):
        super(ExampleResult, self).__init__()
        self.formatter = formatter

    def startTestRun(self):
        super(ExampleResult, self).startTestRun()
        self._start = time.time()

    def enterContext(self, context, depth):
        self.formatter.show(self.formatter.enter_context(context, depth))

    def exitContext(self, depth):
        self.formatter.show(self.formatter.exit_context(depth))

    def enterGroup(self, group):
        self.formatter.show(self.formatter.enter_group(group))

    def addError(self, example, exc_info):
        super(ExampleResult, self).addError(example, exc_info)
        self.formatter.show(self.formatter.error(example, exc_info))

    def addFailure(self, example, exc_info):
        super(ExampleResult, self).addFailure(example, exc_info)
        self.formatter.show(self.formatter.failure(example, exc_info))

    def addSuccess(self, example):
        super(ExampleResult, self).addSuccess(example)
        self.formatter.show(self.formatter.success(example))

    def addSkip(self, example, reason):
        super(ExampleResult, self).addSkip(example, reason)
        self.formatter.show(self.formatter.skip(example, reason))

    def exitGroup(self, group):
        self.formatter.show(self.formatter.exit_group(group))

    def stopTestRun(self):
        super(ExampleResult, self).stopTestRun()
        self.elapsed = time.time() - self._start

        self.formatter.finished()
        self.formatter.show(self.formatter.errors(self.errors))
        self.formatter.show(self.formatter.failures(self.failures))
        self.formatter.show(
            self.formatter.statistics(elapsed=self.elapsed, result=self)
        )

class FormatterMixin(object):
    """
    Provide some higher-level formatting using the child's building blocks.

    """

    def finished(self):
        """
        The run has finished.

        """

        self.show("\n\n")

    def statistics(self, elapsed, result):
        """
        Return output for the combined time and result summary statistics.

        """

        return "\n".join((self.timing(elapsed), self.result_summary(result)))

    def errors(self, errors):
        if not errors:
            return ""

        tracebacks = (self.traceback(error, tb) for error, tb in errors)
        return "\n".join(["Errors:\n", "\n".join(tracebacks), ""])

    def failures(self, failures):
        if not failures:
            return ""

        tracebacks = (self.traceback(fail, tb) for fail, tb in failures)
        return "\n".join(["Failures:\n", "\n".join(tracebacks), ""])

class Colored(FormatterMixin):
    """
    Wrap a formatter to show colored output.

    """

    ANSI = {
        "reset" : "\x1b[0m",
        "black" : "\x1b[30m",
        "red" : "\x1b[31m",
        "green" : "\x1b[32m",
        "yellow" : "\x1b[33m",
        "blue" : "\x1b[34m",
        "magenta" : "\x1b[35m",
        "cyan" : "\x1b[36m",
        "gray" : "\x1b[37m",
    }

    def __init__(self, formatter):
        self._formatter = formatter

    def __getattr__(self, attr):
        """
        Delegate to the wrapped formatter.

        """

        return getattr(self._formatter, attr)

    def color(self, color, text):
        """
        Color some text in the given ANSI color.

        """

        return "{escape}{text}{reset}".format(
            escape=self.ANSI[color], text=text, reset=self.ANSI["reset"],
        )

    def error(self, example, exc_info):
        return self.color("red", self._formatter.error(example, exc_info))

    def failure(self, example, exc_info):
        return self.color("red", self._formatter.failure(example, exc_info))

    def success(self, example):
        return self.color("green", self._formatter.success(example))

    def traceback(self, example, traceback):
        name = str(example.group) + ": " + str(example)
        colored = "\n".join([self.color("blue", name), traceback])
        return indent(colored, 4 * " ")

    def result_summary(self, result):
        output = self._formatter.result_summary(result)

        if result.wasSuccessful():
            return self.color("green", output)
        return self.color("red", output)

class DotsFormatter(FormatterMixin):
    def __init__(self, stream=sys.stderr):
        self.stream = stream

    def show(self, text):
        """
        Write the text to the stream and flush immediately.

        """

        self.stream.write(text)
        self.stream.flush()

    def enter_context(self, context, depth):
        """
        A new context was entered.

        """

        return ""

    def exit_context(self, depth):
        """
        A context was exited.

        """

        return ""

    def enter_group(self, group):
        """
        A new example group was entered.

        """

        return ""

    def exit_group(self, group):
        """
        The example group was entered.

        """

        return ""

    def result_summary(self, result):
        """
        Return a summary of the results.

        """

        return "{} examples, {} errors, {} failures\n".format(
            result.testsRun, len(result.errors), len(result.failures),
        )

    def timing(self, elapsed):
        """
        Return output on the time taken on the examples run.

        """

        return "Finished in {:.6f} seconds.\n".format(elapsed)

    def error(self, example, exc_info):
        """
        An error was encountered.

        """

        return "E"

    def failure(self, example, exc_info):
        """
        A failure was encountered.

        """

        return "F"

    def skip(self, example, reason):
        """
        A skip was encountered.

        """

        return "S"

    def success(self, example):
        """
        A success was encountered.

        """

        return "."

    def traceback(self, example, traceback):
        """
        Format an example and its traceback.

        """

        return "\n".join((str(example), traceback))

class Verbose(FormatterMixin):
    """
    Show verbose output (including example and group descriptions).

    """

    def __init__(self, formatter):
        self._depth = 1
        self._formatter = formatter

    def __getattr__(self, attr):
        return getattr(self._formatter, attr)

    def enter_context(self, context, depth):
        self._depth = depth + 1
        return indent(context.name + "\n", depth * 4 * " ")

    def exit_context(self, depth):
        self._depth = depth + 1
        return ""

    def enter_group(self, group):
        return "{}\n".format(group)

    def finished(self):
        self.show("\n")

    def error(self, example, exc_info):
        return indent(str(example), self._depth * 4 * " ") + " - ERROR\n"

    def failure(self, example, exc_info):
        return indent(str(example), self._depth * 4 * " ") + " - FAIL\n"

    def success(self, example):
        return indent(str(example), self._depth * 4 * " ") + "\n"


"""
The implementation of the Ivoire runner.

"""

import argparse
import runpy
import sys

from ivoire import result
from ivoire.load import load_by_name
from ivoire.transform import ExampleLoader, transform_possible
import ivoire

FORMATTERS = {
    "dots" : result.DotsFormatter,
}

class _ExampleNotRunning(object):
    """
    An error occurred, but no example was running. Mimic an Example object.

    """

    failureException = None
    group = None

    def __str__(self):
        return "<not in example>"

def should_color(when):
    """
    Decide whether to color output.

    """

    if when == "auto":
        return sys.stderr.isatty()
    return when == "always"

def parse(argv=None):
    """
    Parse some arguments using the parser.

    """

    if argv is None:
        argv = sys.argv[1:]

    # Evade http://bugs.python.org/issue9253
    if not argv or argv[0] not in {"run", "transform"}:
        argv = ["run"] + argv

    arguments = _clean(_parser.parse_args(argv))
    return arguments

def _clean(arguments):
    if hasattr(arguments, "color"):
        arguments.color = should_color(arguments.color)
    return arguments

def setup(config):
    """
    Setup the environment for an example run.

    """

    formatter = config.Formatter()

    if config.verbose:
        formatter = result.Verbose(formatter)
    if config.color:
        formatter = result.Colored(formatter)

    current_result = result.ExampleResult(formatter)

    ivoire.current_result = ivoire._manager.result = current_result

def run(config):
    """
    Time to run.

    """

    setup(config)

    if config.exitfirst:
        ivoire.current_result.failfast = True

    ivoire.current_result.startTestRun()

    for spec in config.specs:
        try:
            load_by_name(spec)
        except Exception:
            ivoire.current_result.addError(
                _ExampleNotRunning(), sys.exc_info()
            )

    ivoire.current_result.stopTestRun()

    sys.exit(not ivoire.current_result.wasSuccessful())

def transform(config):
    """
    Run in transform mode.

    """

    if transform_possible:
        ExampleLoader.register()

        args, sys.argv[1:] = sys.argv[1:], config.args
        try:
            return runpy.run_path(config.runner, run_name="__main__")
        finally:
            sys.argv[1:] = args

def main(argv=None):
    arguments = parse(argv)
    arguments.func(arguments)

_parser = argparse.ArgumentParser(description="The Ivoire test runner.")
_subparsers = _parser.add_subparsers()

_run = _subparsers.add_parser(
    "run",
    help="Run Ivoire specs."
)
_run.add_argument(
    "-c", "--color",
    choices=["always", "never", "auto"],
    default="auto",
    dest="color",
    help="Format colored output.",
)
_run.add_argument(
    "-f", "--formatter",
    choices=FORMATTERS,
    default="dots",
    dest="Formatter",
    type=lambda formatter : FORMATTERS[formatter],
    help="Format output with the given formatter.",
)
_run.add_argument(
    "-v", "--verbose",
    action="store_true",
    help="Format verbose output.",
)
_run.add_argument(
    "-x", "--exitfirst",
    action="store_true",
    help="Exit after the first error or failure.",
)
_run.add_argument("specs", nargs="+")
_run.set_defaults(func=run)

_transform = _subparsers.add_parser(
    "transform",
    help="Run an Ivoire spec through another test runner by translating its "
         "source code.",
)
_transform.add_argument(
    "runner",
    help="The command to run the transformed tests with."
)
_transform.add_argument(
    "args",
    nargs=argparse.REMAINDER,
)
_transform.set_defaults(func=transform)


import ast
import sys

from ivoire.compat import FileFinder, SourceFileLoader, transform_possible

class ExampleTransformer(ast.NodeTransformer):
    """
    Transform a module that uses Ivoire into one that uses unittest.

    This is highly experimental, and certain to have bugs (including some
    known ones). Right now it is highly strict and will not properly transform
    all possibilities. File issues if you find things that are wrong.

    Note: None of the methods on this object are public API other than the
    ``transform`` method.

    """

    def transform(self, node):
        transformed = self.visit(node)
        ast.fix_missing_locations(transformed)
        return transformed

    def visit_ImportFrom(self, node):
        if node.module == "ivoire":
            node.module = "unittest"
            node.names[0].name = "TestCase"
        return node

    def visit_With(self, node):
        """
        with describe(thing) as it:
            ...

             |
             v

        class TestThing(TestCase):
            ...

        """

        withitem, = node.items
        context = withitem.context_expr

        if context.func.id == "describe":
            describes = context.args[0].id
            example_group_name = withitem.optional_vars.id
            return self.transform_describe(node, describes, example_group_name)
        else:
            return node

    def transform_describe(self, node, describes, context_variable):
        """
        Transform a describe node into a ``TestCase``.

        ``node`` is the node object.
        ``describes`` is the name of the object being described.
        ``context_variable`` is the name bound in the context manager (usually
        "it").

        """

        body = self.transform_describe_body(node.body, context_variable)
        return ast.ClassDef(
            name="Test" + describes.title(),
            bases=[ast.Name(id="TestCase", ctx=ast.Load())],
            keywords=[],
            starargs=None,
            kwargs=None,
            body=list(body),
            decorator_list=[],
        )

    def transform_describe_body(self, body, group_var):
        """
        Transform the body of an ``ExampleGroup``.

        ``body`` is the body.
        ``group_var`` is the name bound to the example group in the context
        manager (usually "it").

        """

        for node in body:
            withitem, = node.items
            context_expr = withitem.context_expr

            name = context_expr.args[0].s
            context_var = withitem.optional_vars.id

            yield self.transform_example(node, name, context_var, group_var)

    def transform_example(self, node, name, context_variable, group_variable):
        """
        Transform an example node into a test method.

        Returns the unchanged node if it wasn't an ``Example``.

        ``node`` is the node object.
        ``name`` is the name of the example being described.
        ``context_variable`` is the name bound in the context manager (usually
        "test").
        ``group_variable`` is the name bound in the surrounding example group's
        context manager (usually "it").

        """

        test_name = "_".join(["test", group_variable] + name.split())
        body = self.transform_example_body(node.body, context_variable)

        return ast.FunctionDef(
            name=test_name,
            args=self.takes_only_self(),
            body=list(body),
            decorator_list=[],
        )

    def transform_example_body(self, body, context_variable):
        """
        Transform the body of an ``Example`` into the body of a method.

        Replaces instances of ``context_variable`` to refer to ``self``.

        ``body`` is the body.
        ``context_variable`` is the name bound in the surrounding context
        manager to the example (usually "test").

        """

        for node in body:
            for child in ast.walk(node):
                if isinstance(child, ast.Name):
                    if child.id == context_variable:
                        child.id = "self"
            yield node

    def takes_only_self(self):
        """
        Return an argument list node that takes only ``self``.

        """

        return ast.arguments(
            args=[ast.arg(arg="self")],
            defaults=[],
            kw_defaults=[],
            kwonlyargs=[],
        )

class ExampleLoader(SourceFileLoader):

    suffix = "_spec.py"

    @classmethod
    def register(cls):
        """
        Register the path hook.

        """

        cls._finder = FileFinder.path_hook((cls, [cls.suffix]))
        sys.path_hooks.append(cls._finder)

    @classmethod
    def unregister(cls):
        """
        Unregister the path hook.

        """

        sys.path_hooks.remove(cls._finder)

    def source_to_code(self, source_bytes, source_path):
        """
        Transform the source code, then return the code object.

        """

        node = ast.parse(source_bytes)
        transformed = ExampleTransformer().transform(node)
        return compile(transformed, source_path, "exec", dont_inherit=True)


#!/usr/bin/python
import os
from pycoeman.monitor import monitor_cpu_mem_disk

def executeCommandMonitor(commandId, command, diskPath, onlyPrint=False):
    # Define the names of the script that executes the command, the log file and the monitor file
    logFileName = commandId + '.log'
    monitorLogFileName = commandId + '.mon'
    monitorDiskLogFileName = commandId + '.mon.disk'

    #Remove log file if already exists
    for f in (logFileName,monitorLogFileName,monitorDiskLogFileName):
        if os.path.isfile(f):
            os.system('rm ' + f)

    if onlyPrint:
        print(command)
        os.system('touch ' + logFileName)
        os.system('touch ' + monitorLogFileName)
        os.system('touch ' + monitorDiskLogFileName)
    else:
        monitor_cpu_mem_disk.run(command, logFileName, monitorLogFileName, monitorDiskLogFileName, diskPath)
        # TODO: if execution folder is in different file system that source data, right now we only monitor raw data usage
    return (logFileName,monitorLogFileName,monitorDiskLogFileName)

def initExecutionFolderLocal(executionFolder, elementsAbsPaths, resume = False):
    # Create directory for this execution
    executionFolderAbsPath = os.path.abspath(executionFolder)

    if os.path.exists(executionFolderAbsPath):
        if not resume:
            raise Exception(executionFolder + ' already exists!')
    else:
        os.makedirs(executionFolderAbsPath)

    # Create links for the files/folder specifed in require and requirelist XML
    for elementAbsPath in elementsAbsPaths:
        # Existance of the elementAbsPath has been checked before
        os.symlink(elementAbsPath , os.path.join(executionFolderAbsPath, os.path.basename(elementAbsPath)))

def getRequiredList(requiredListFile):
    required = []
    if not os.path.isfile(requiredListFile):
        raise Exception(requiredListFile + ' does not exist!')
    for line in open(requiredListFile, 'r').read().split('\n'):
        if line != '':
            required.append(line)
    return required

def apply_argument_parser(argumentsParser, options=None):
    """ Apply the argument parser. """
    if options is not None:
        args = argumentsParser.parse_args(options)
    else:
        args = argumentsParser.parse_args()
    return args

def parseComponent(componentInfo, dataAbsPath):
    # Mandatory tags
    commandId = componentInfo.find("id").text.strip()
    if commandId.count(' '):
        raise Exception('Command IDs cannot contain whitespaces!')

    command = componentInfo.find("command").text.strip()

    # Optional tags
    requiredTag = componentInfo.find("require")
    requiredElements = []
    if requiredTag != None:
        requiredElements = requiredTag.text.strip().split()

    requiredListTag = componentInfo.find("requirelist")
    if requiredListTag != None:
        requiredListFile = requiredListTag.text.strip()
        requiredElements.extend(getRequiredList(dataAbsPath + '/' + requiredListFile))

    for i, requiredElement in enumerate(requiredElements):
        if requiredElement.endswith('/'):
            requiredElement = requiredElement[:-1]
        if requiredElement.startswith('/'):
            requiredElementAbsPath = requiredElement
        else:
            requiredElementAbsPath = dataAbsPath + '/' + requiredElement

        if not os.path.exists(requiredElementAbsPath):
            raise Exception(requiredElementAbsPath + ' does not exist!')

        requiredElements[i] = requiredElementAbsPath

    outputTag = componentInfo.find("output")
    outputElements = []
    if outputTag != None:
        outputElements = outputTag.text.strip().split()
        for i, outputElement in enumerate(outputElements):
            if outputElement.startswith('/'):
                raise Exception('outputElements must use relative paths!')
            if outputElement.endswith('/'):
                outputElements[i] = outputElement[:-1]

    return (commandId, command, requiredElements, outputElements)

def parseHost(hostInfo):
    hostName = hostInfo.find("name").text.strip()
    hostUser = hostInfo.find("user").text.strip()
    hostSetenv = hostInfo.find("setenv").text.strip()
    hostNumCommands = int(hostInfo.find("numcommands").text.strip())
    hostExeDir = hostInfo.find("exedir").text.strip()
    if hostExeDir[0] != '/':
        raise Exception('remoteexedir must be an absolute path!')
    if hostSetenv[0] != '/':
        raise Exception('setenv must be an absolute path!')
    return (hostName, hostUser, hostSetenv, hostNumCommands, hostExeDir)

def sshExecute(ssh, command):
    _, stdout, stderr = ssh.exec_command(command)
    return '\n'.join(stderr.readlines() + stdout.readlines())


import fnmatch
import imp
import os

def load_by_name(name):
    """
    Load a spec from either a file path or a fully qualified name.

    """

    if os.path.exists(name):
        load_from_path(name)
    else:
        __import__(name)

def load_from_path(path):
    """
    Load a spec from a given path, discovering specs if a directory is given.

    """

    if os.path.isdir(path):
        paths = discover(path)
    else:
        paths = [path]

    for path in paths:
        name = os.path.basename(os.path.splitext(path)[0])
        imp.load_source(name, path)

def filter_specs(paths):
    """
    Filter out only the specs from the given (flat iterable of) paths.

    """

    return fnmatch.filter(paths, "*_spec.py")

def discover(path, filter_specs=filter_specs):
    """
    Discover all of the specs recursively inside ``path``.

    Successively yields the (full) relative paths to each spec.

    """

    for dirpath, _, filenames in os.walk(path):
        for spec in filter_specs(filenames):
            yield os.path.join(dirpath, spec)


# Copyright (c) Moshe Zadka
# See LICENSE for details.
"""ncolony.directory_monitor
============================

Monitor directories for configuration and messages
"""

import functools
import os

from twisted.python import filepath

def checker(location, receiver):
    """Construct a function that checks a directory for process configuration

    The function checks for additions or removals
    of JSON process configuration files and calls the appropriate receiver
    methods.

    :param location: string, the directory to monitor
    :param receiver: IEventReceiver
    :returns: a function with no parameters
    """
    path = filepath.FilePath(location)
    files = set()
    filesContents = {}

    def _check(path):
        currentFiles = set(fname for fname in os.listdir(location)
                           if not fname.endswith('.new'))
        removed = files - currentFiles
        added = currentFiles - files
        for fname in added:
            contents = path.child(fname).getContent()
            filesContents[fname] = contents
            receiver.add(fname, contents)
        for fname in removed:
            receiver.remove(fname)
        same = currentFiles & files
        for fname in same:
            newContents = path.child(fname).getContent()
            oldContents = filesContents[fname]
            if newContents == oldContents:
                continue
            receiver.remove(fname)
            filesContents[fname] = newContents
            receiver.add(fname, newContents)
        files.clear()
        files.update(currentFiles)
    return functools.partial(_check, path)

def messages(location, receiver):
    """Construct a function that checks a directory for messages

    The function checks for new messages and
    calls the appropriate method on the receiver. Sent messages are
    deleted.

    :param location: string, the directory to monitor
    :param receiver: IEventReceiver
    :returns: a function with no parameters
    """
    path = filepath.FilePath(location)

    def _check(path):
        messageFiles = path.globChildren('*')
        for message in messageFiles:
            if message.basename().endswith('.new'):
                continue
            receiver.message(message.getContent())
            message.remove()
    return functools.partial(_check, path)


# Copyright (c) Moshe Zadka
# See LICENSE for details.

"""ncolony.ctllib
===================================================================

This module can be used either as a library from Python or as a
commandline using the wrapper :code:`ctl` via

.. code-block:: bash

   $ python -m ncolony ctl <arguments>

Description of the service's interface is in <figure out how
to do a back-reference>.

The add/remove messages add/remove configuration files for processes.
Since removal of a configuration is equivalent to killing the process,
nothing else nees to be done to rid of needed processes.

All functions which are meant to be used as a library API

Add is the most complicated function, because it needs to be able
to express every aspect of the process. It allows control of the
name, command, arguments, environment variables and uid/gid.

Removal is pretty simple, since it only needs the name.

Restart also needs just the name.

Restart-all does not need even the name, since it restarts
all processes.
"""

import argparse
import collections
import functools
import itertools
import json
import os

from twisted.python import filepath

from ncolony import main as mainlib

NEXT = functools.partial(next, itertools.count(0))

Places = collections.namedtuple('Places', 'config messages')

def _dumps(stuff):
    return json.dumps(stuff).encode('utf-8')

# pylint: disable=too-many-arguments,too-many-locals
def add(places, name, cmd, args, env=None, uid=None, gid=None, extras=None,
        env_inherit=None):
    """Add a process.

    :param places: a Places instance
    :param name: string, the logical name of the process
    :param cmd: string, executable
    :param args: list of strings, command-line arguments
    :param env: dictionary mapping strings to strings
         (will be environment in subprocess)
    :param uid: integer, uid to run the new process as
    :param gid: integer, gid to run the new process as
    :param extras: a dictionary with additional parameters
    :param env_inherit: a list of environment variables to inherit
    :returns: None
    """
    args = [cmd]+args
    config = filepath.FilePath(places.config)
    fle = config.child(name)
    details = dict(args=args)
    if env is not None:
        newEnv = {}
        for thing in env:
            name, value = thing.split('=', 1)
            newEnv[name] = value
        details['env'] = newEnv
    if uid is not None:
        details['uid'] = uid
    if gid is not None:
        details['gid'] = gid
    if env_inherit is not None:
        details['env_inherit'] = env_inherit
    if extras is not None:
        details.update(extras)
    content = _dumps(details)
    fle.setContent(content)
# pylint: enable=too-many-arguments,too-many-locals

def remove(places, name):
    """Remove a process

    :params places: a Places instance
    :params name: string, the logical name of the process
    :returns: None
    """
    config = filepath.FilePath(places.config)
    fle = config.child(name)
    fle.remove()

def _addMessage(places, content):
    messages = filepath.FilePath(places.messages)
    name = '%03dMessage.%s' % (NEXT(), os.getpid())
    message = messages.child(name)
    message.setContent(content)

def restart(places, name):
    """Restart a process

    :params places: a Places instance
    :params name: string, the logical name of the process
    :returns: None
    """
    content = _dumps(dict(type='RESTART', name=name))
    _addMessage(places, content)

def restartAll(places):
    """Restart all processes

    :params places: a Places instance
    :returns: None
    """
    content = _dumps(dict(type='RESTART-ALL'))
    _addMessage(places, content)

def _parseJSON(fname):
    with open(fname) as fp:
        data = fp.read()
    return json.loads(data)

PARSER = argparse.ArgumentParser()
PARSER.add_argument('--messages', required=True)
PARSER.add_argument('--config', required=True)
_subparsers = PARSER.add_subparsers()
_restart_all_parser = _subparsers.add_parser('restart-all')
_restart_all_parser.set_defaults(func=restartAll)
_restart_parser = _subparsers.add_parser('restart')
_restart_parser.add_argument('name')
_restart_parser.set_defaults(func=restart)
_remove_parser = _subparsers.add_parser('remove')
_remove_parser.add_argument('name')
_remove_parser.set_defaults(func=remove)
_add_parser = _subparsers.add_parser('add')
_add_parser.add_argument('name')
_add_parser.add_argument('--cmd', required=True)
_add_parser.add_argument('--arg', dest='args', action='append')
_add_parser.add_argument('--env', action='append')
_add_parser.add_argument('--uid', type=int)
_add_parser.add_argument('--gid', type=int)
_add_parser.add_argument('--extras', type=_parseJSON)
_add_parser.add_argument('--env-inherit', dest='env_inherit', action='append')
_add_parser.set_defaults(func=add)

def call(results):
    """Call results.func on the attributes of results

    :params result: dictionary-like object
    :returns: None
    """
    results = vars(results)
    places = Places(config=results.pop('config'),
                    messages=results.pop('messages'))
    func = results.pop('func')
    func(places, **results)

@mainlib.COMMANDS.register(name='ctl')
def main(argv):
    """command-line entry point

        --messages: messages directory

        --config: configuration directory

    subcommands:
        add:
            name (positional)

            --cmd (required) -- executable

            --arg -- add an argument

            --env -- add an environment variable (VAR=value)

            --uid -- set uid

            --gid -- set gid

            --extras -- a JSON file with more parameters

            --env-inherit -- add an environment variable to inherit

        remove:
            name (positional)
        restart:
            name (positional)
        restart-all:
            no arguments
    """
    ns = PARSER.parse_args(argv[1:])
    call(ns)


# Copyright (c) Moshe Zadka
# See LICENSE for details.
"""ncolony.service
==========================================================================

Implement the 'ncolony' twistd plugin.

.. code-block:: bash

  $ twistd ncolony --config <dir> --messages <dir>

Will run a service that brings up all processes described in files
in the configuration directory (and shuts them down if the files
ago away), and listens for restart messages on the messages directory.
"""

from twisted.python import usage
from twisted.application import service as taservice, internet
from twisted.runner import procmon as procmonlib, procmontap

from ncolony import directory_monitor, process_events

# pylint: disable=too-few-public-methods

class TransportDirectoryDict(dict):

    """Dict-like object that writes the 'pid' value to a directory

    This dict-like object assumes all the values have a 'pid' attribute,
    and writes that attribute into a file named the same as the key
    in the given directory.
    """

    def __init__(self, output):
        """Initialize

        :param output: a {twisted.python.filepath.FilePath} object
        """
        super(TransportDirectoryDict, self).__init__()
        self.output = output

    def __setitem__(self, name, value):
        super(TransportDirectoryDict, self).__setitem__(name, value)
        self.output.child(name).setContent(str(value.pid))

    def __delitem__(self, name):
        super(TransportDirectoryDict, self).__delitem__(name)
        self.output.child(name).remove()

# pylint: enable=too-few-public-methods

def get(config, messages, freq, pidDir=None, reactor=None):
    """Return a service which monitors processes based on directory contents

    Construct and return a service that, when started, will run processes
    based on the contents of the 'config' directory, restarting them
    if file contents change and stopping them if the file is removed.

    It also listens for restart and restart-all messages on the 'messages'
    directory.

    :param config: string, location of configuration directory
    :param messages: string, location of messages directory
    :param freq: number, frequency to check for new messages and configuration
                 updates
    :param pidDir: {twisted.python.filepath.FilePath} or None,
                   location to keep pid files
    :param reactor: something implementing the interfaces
                       {twisted.internet.interfaces.IReactorTime} and
                       {twisted.internet.interfaces.IReactorProcess} and
    :returns: service, {twisted.application.interfaces.IService}
    """
    ret = taservice.MultiService()
    args = ()
    if reactor is not None:
        args = reactor,
    procmon = procmonlib.ProcessMonitor(*args)
    if pidDir is not None:
        protocols = TransportDirectoryDict(pidDir)
        procmon.protocols = protocols
    procmon.setName('procmon')
    receiver = process_events.Receiver(procmon)
    confcheck = directory_monitor.checker(config, receiver)
    confserv = internet.TimerService(freq, confcheck)
    confserv.setServiceParent(ret)
    messagecheck = directory_monitor.messages(messages, receiver)
    messageserv = internet.TimerService(freq, messagecheck)
    messageserv.setServiceParent(ret)
    procmon.setServiceParent(ret)
    return ret

# pylint: disable=too-few-public-methods

class Options(usage.Options):

    """Options for ncolony service"""

    optParameters = [
        ["config", None, None, "Directory for configuration"],
        ["messages", None, None, "Directory for messages"],
        ["frequency", None, 10, "Frequency of checking for updates", float],
        ["pid", None, None, "Directory of PID files"],
    ] + procmontap.Options.optParameters

    def postOptions(self):
        """Checks that required messages/config directories are present"""
        for param in ('messages', 'config'):
            if self[param] is None:
                raise usage.UsageError("Missing required", param)

# pylint: enable=too-few-public-methods

def makeService(opt):
    """Return a service based on parsed command-line options

    :param opt: dict-like object. Relevant keys are config, messages,
                pid, frequency, threshold, killtime, minrestartdelay
                and maxrestartdelay
    :returns: service, {twisted.application.interfaces.IService}
    """
    ret = get(config=opt['config'], messages=opt['messages'],
              pidDir=opt['pid'], freq=opt['frequency'])
    pm = ret.getServiceNamed("procmon")
    pm.threshold = opt["threshold"]
    pm.killTime = opt["killtime"]
    pm.minRestartDelay = opt["minrestartdelay"]
    pm.maxRestartDelay = opt["maxrestartdelay"]
    return ret


import time

import phonon.client
import phonon.connections
from phonon import PHONON_NAMESPACE

class Nodelist(object):
    """
    Keeps track of the nodes currently holding a reference to a particular
    resource by maintaining a hash in redis with the node ids and time the
    session was last refreshed.
    """

    def __init__(self, resource_key):
        """
        :param phonon.connections.AsyncConn conn: The Process to which the instantiating
            reference belongs
        :param string resource_key: An identifier for the instantiating
            reference
        """
        self.resource_key = resource_key
        self.nodelist_key = "{0}_{1}.nodelist".format(PHONON_NAMESPACE, resource_key)
        self.conn = phonon.connections.connection
        self.refresh_session()

    def refresh_session(self, node_id=None):
        """
        Adds or refreshes a particular node in the nodelist, attributing the
        current time with the node_id.

        :param string node_id: optional, the connection id of the node whose
        session should be refreshed
        """
        if not node_id:
            node_id = self.conn.id

        self.conn.client.hset(self.nodelist_key, node_id, int(time.time() * 1000.))

    def find_expired_nodes(self, node_ids=None):
        """
        Detects connections that have held a reference for longer than its
        process_ttl without refreshing its session. This function does not
        actually removed them from the hash. (See remove_expired_nodes.)

        :param list node_ids: optional, a list of ids to check to see if they
            have expired.  If node_ids is not passed in, all nodes in the hash
            will be checked.
        """
        if node_ids:
            nodes = zip(node_ids, [int(t) for t in self.conn.client.hmget(self.nodelist_key, node_ids)])
        else:
            nodes = self.get_all_nodes().items()

        expiration_delta = self.conn.PROCESS_TTL * 1000.
        now = int(time.time() * 1000.)
        return [node_id for (node_id, last_updated) in nodes if (now - last_updated) > expiration_delta]

    def remove_expired_nodes(self, node_ids=None):
        """
        Removes all expired nodes from the nodelist.  If a set of node_ids is
        passed in, those ids are checked to ensure they haven't been refreshed
        prior to a lock being acquired.

        Should only be run with a lock.

        :param list node_ids: optional, a list of node_ids to remove.  They
            will be verified to ensure they haven't been refreshed.

        """
        nodes = self.find_expired_nodes(node_ids)
        if nodes:
            self.conn.client.hdel(self.nodelist_key, *nodes)

    def remove_node(self, node_id=None):
        """
        Removes a particular node from the nodelist.

        :param string node_id: optional, the process id of the node to remove
        """
        if not node_id:
            node_id = self.conn.id

        self.conn.client.hdel(self.nodelist_key, node_id)

    def clear_nodelist(self):
        """
        Removes all nodes from a nodelist.

        Should only be run with a lock.
        """
        self.conn.client.delete(self.nodelist_key)

    def get_last_updated(self, node_id=None):
        """
        Returns the time a particular node has been last refreshed.

        :param string node_id: optional, the connection id of the node to retrieve

        :rtype: int
        :returns: Returns a unix timestamp if it exists, otherwise None
        """
        if not node_id:
            node_id = self.conn.id

        dt = self.conn.client.hget(self.nodelist_key, node_id)
        return int(dt) if dt else None

    def get_all_nodes(self):
        """
        Returns all nodes in the hash with the time they were last refreshed
        as a dictionary.

        :rtype: dict(string, int)
        :returns: A dictionary of strings and corresponding timestamps

        """
        nodes = self.conn.client.hgetall(self.nodelist_key)
        return {node_id: int(dt) for (node_id, dt) in nodes.items()}

    def count(self):
        """
        :rtype: int
        :returns: The number of nodes in the nodelist
        """
        return self.conn.client.hlen(self.nodelist_key)


from phonon import get_logger, PHONON_NAMESPACE, TTL

import time

import phonon.lock
import phonon.nodelist
import phonon.connections

logger = get_logger(__name__)

class Reference(object):
    """
    Represents a reference to some resource in the network. Handles reference
    counting, dereferencing, locking/ownership, and maintaining whether or not
    resources were externally modified.

    This class also provides a layer of abstraction on failover and cleanup. In
    the event of total process failure; a reference on that process will expire
    and be subsequently detected/recovered by this class to the extent updates
    overlap.

    More in-depth:

    When a process acquires a `Reference` to a resource it attempts to lock
    that resource. It will block by default for a total of Process.BLOCKING_TIMEOUT
    seconds (defaults to 500s) before raising a Reference.AlreadyLocked
    exception

    That process/reference will attempt to connect to the backend, and retry
    until either successful completion of the task or python's maximum
    recursion depth has been reached (100 retries), sleeping
    Process.RETRY_SLEEP seconds between each retry (defaults to 0.5s).

    Upon successfully locking; `Reference` will create a record with the key
    "{0}.reflist".format(pid) in redis where the key is the unique id for the
    process this code is running on and the value is the unix timestamp for the
    time the process last modified that entry. If a pid is in that list it is
    assumed they currently have an active session for whatever resource is
    locked. If the record was created longer ago than the session length it is
    assumed the process "fell over". It's reference will be cleaned up and the
    reference count will be decremented.

    Reference counts are calculated by the length of the json structure
    described above after process failures have been resolved.

    """

    def __init__(self, resource):
        """
        :param Process process: The Process to which this reference belongs
        :param str resource: An identifier for the resource. For example:
            Buzz.12345

        """
        self.resource_key = resource
        self.nodelist = phonon.nodelist.Nodelist(resource)
        self.times_modified_key = "{}_{}.times_modified".format(PHONON_NAMESPACE, resource)
        self.refcount_key = "{}_{}.refcount".format(PHONON_NAMESPACE, resource)
        self.force_expiry = False
        self.conn = phonon.connections.connection
        if self.resource_key not in self.conn.local_registry:
            self.conn.client.incr(self.refcount_key)
            self.conn.add_to_registry(self.resource_key)

        self.refresh_session()

    def lock(self):
        """
        Locks the resource managed by this reference.
        """
        return phonon.lock.Lock(self.resource_key)

    def refresh_session(self):
        """
        Update the session for this node. Specifically; lock on the reflist,
        then update the time  this node acquired the reference.

        This method should only be called while the reference is locked.
        """
        expired_nodes = self.nodelist.find_expired_nodes()
        if expired_nodes:
            self.nodelist.remove_expired_nodes(expired_nodes)
        self.nodelist.refresh_session()

    def increment_times_modified(self):
        """
        Increments the number of times this resource has been modified by all
        processes.
        """
        rc = self.conn.client.incr(self.times_modified_key)
        self.conn.client.pexpire(self.times_modified_key,
                                 phonon.s_to_ms(TTL))  # ttl is in ms

    def get_times_modified(self):
        """
        :returns: The total number of times increment_times_modified has been called for this resource by all processes.
        :rtype: int
        """
        times_modified = self.conn.client.get(self.times_modified_key)
        if times_modified is None:
            return 0
        return int(times_modified)

    def count(self):
        """
        :returns: The total number of elements in the reference list.
        :rtype: int
        """
        references = self.conn.client.get(self.refcount_key)
        if references is None:
            return 0
        return int(references)

    def dereference(self, callback=None, args=None, kwargs=None):
        """
        This method should only be called while the reference is locked.

        Decrements the reference count for the resource. If this process holds
        the only reference at the time we finish dereferencing it; True is
        returned. Operating on the resource after it has been dereferenced is
        undefined behavior.

        Dereference queries the value stored in the backend, if any, iff (if
        and only if) this instance is the last reference to that resource. e.g.
        self.count() == 0

        :param function callback: A function to execute iff it's determined
            this process holds the only reference to the resource. When there
            is a failure communicating with the backend in the cleanup step the
            callback function will be called an additional time for that
            failure and each subsequent one thereafter. Ensure your callback
            handles this properly.
        :param tuple args: Positional arguments to pass your callback.
        :param dict kwargs: keyword arguments to pass your callback.

        :returns: Whether or not there are no more references among all
            processes. True if this was the last reference. False otherwise.
        :rtype: bool
        """
        if args is None:
            args = tuple()
        if kwargs is None:
            kwargs = {}

        client = self.conn.client

        should_execute = False
        if self.force_expiry:
            should_execute = True

        if not should_execute:
            self.nodelist.remove_node(self.conn.id)
            self.nodelist.remove_expired_nodes()

            updated_refcount = client.incr(self.refcount_key, -1)
            should_execute = (updated_refcount <= 0)  # When we force expiry this will be -1

        try:
            if callable(callback) and should_execute:
                callback(*args, **kwargs)
        finally:
            if should_execute:
                client.delete(self.resource_key,
                              self.nodelist.nodelist_key,
                              self.times_modified_key,
                              self.refcount_key)

            self.conn.remove_from_registry(self.resource_key)
        return should_execute


from __future__ import unicode_literals, absolute_import

def delimit(values, delimiter=', '):
    "Returns a list of tokens interleaved with the delimiter."
    toks = []

    if not values:
        return toks

    if not isinstance(delimiter, (list, tuple)):
        delimiter = [delimiter]

    last = len(values) - 1

    for i, value in enumerate(values):
        toks.append(value)

        if i < last:
            toks.extend(delimiter)

    return toks


# Copyright (c) Moshe Zadka
# See LICENSE for details.

"""ncolony.beatcheck
====================

Check heartbeats of processes that should beat.

Usually used as

$ twistd -n ncolony_beatcheck --config config --messages messages

It will watch the configurations, and send a restart message
for any process that does not beat within its heartbeat.
Processes are encouraged to try and beat about 3-4 times
faster than the minimum, so that they can miss one beat, and
account for slight timer inaccuracies, and still not be considered
unhealthy.
"""

import functools
import json
import time

from twisted.python import filepath, usage

from twisted.application import internet as tainternet

from ncolony import ctllib
from ncolony.client import heart

def check(path, start, now):
    """check which processes need to be restarted

    :params path: a twisted.python.filepath.FilePath with configurations
    :params start: when the checker started running
    :params now: current time
    :returns: list of strings
    """
    return [child.basename() for child in path.children()
            if _isbad(child, start, now)]

def _isbad(child, start, now):
    content = child.getContent()
    parsed = json.loads(content)
    params = parsed.get('ncolony.beatcheck')
    if params is None:
        return False
    period = params['period']
    grace = params['grace']
    mtime = max(child.getModificationTime(), start)
    if mtime + period*grace >= now:
        return False
    status = params['status']
    statusPath = child.clonePath(status)
    if not statusPath.exists():
        return True
    if statusPath.isdir():
        statusPath = statusPath.child(child.basename())
    statusMtime = statusPath.getModificationTime()
    return (statusMtime + period) < now

def run(restarter, checker, timer):
    """Run restarter on the checker's output

    :params restarter: something to run on the output of the checker
    :params checker: a function expected to get one argument (current time)
                     and return a list of stale names
    :params timer: a function of zero arguments, intended to return current
                   time
    :returns: None
    """
    for bad in checker(timer()):
        restarter(bad)

def parseConfig(opt):
    """Parse configuration

    :params opt: dict-like object with config and messages keys
    :returns: restarter, path
    """
    places = ctllib.Places(config=opt['config'], messages=opt['messages'])
    restarter = functools.partial(ctllib.restart, places)
    path = filepath.FilePath(opt['config'])
    return restarter, path

def makeService(opt):
    """Make a service

    :params opt: dictionary-like object with 'freq', 'config' and 'messages'
    :returns: twisted.application.internet.TimerService that at opt['freq']
              checks for stale processes in opt['config'], and sends
              restart messages through opt['messages']
    """
    restarter, path = parseConfig(opt)
    now = time.time()
    checker = functools.partial(check, path, now)
    beatcheck = tainternet.TimerService(opt['freq'], run, restarter,
                                        checker, time.time)
    beatcheck.setName('beatcheck')
    return heart.wrapHeart(beatcheck)

# pylint: disable=too-few-public-methods

class Options(usage.Options):

    """Options for ncolony beatcheck service"""

    optParameters = [
        ["messages", None, None, "Directory for messages"],
        ["config", None, None, "Directory for configuration"],
        ["freq", None, 10, "Frequency of checking for updates", float],
    ]

    def postOptions(self):
        """Checks that required messages/config directories are present"""
        for param in ('messages', 'config'):
            if self[param] is None:
                raise usage.UsageError("Missing required", param)

# pylint: enable=too-few-public-methods


import re
from io import StringIO
from typing import Generic, Sequence, TypeVar, Callable, Optional, Tuple, Union  # noqa: F401

Input = TypeVar('Input')
Output = TypeVar('Output')
Convert = TypeVar('Convert')

class Reader(Generic[Input]):
    """Abstract base class for readers.

    A ``Reader`` is an immutable object holding the state of parsing.

    Attributes:
        first (Input): The element at the current position.
        rest (Sequence[Input]): A ``Reader`` at the next position.
        position (int): How many characters into the source the parsing has
            gone.
        finished (bool): Indicates if the source is at the end. It is an error
            to access ``first`` or ``rest`` if ``finished`` is ``True``.
    """
    first = NotImplemented  # type: Input
    rest = NotImplemented  # type: Reader[Input]
    position = NotImplemented  # type: int
    finished = NotImplemented  # type: bool

    def next_token(self):
        return self.first

    def expected_error(self, expected: str) -> str:
        """Generate a basic error to include the current state.

        A parser can supply only a representation of what it is expecting to
        this method and the reader will provide the context, including the index
        to the error.

        Args:
            expected: A representation of what the parser is currently expecting

        Returns:
            A full error message
        """

        if self.finished:
            return 'Expected {} but found end of source'.format(expected)
        else:
            return 'Expected {} but found {} at index {}'.format(expected, self.next_token(), self.position)

    def recursion_error(self, repeated_parser: str):
        """Generate an error to indicate that infinite recursion was encountered.

        A parser can supply a representation of itself to this method and the
        reader will supply the context, including the location where the
        parser stalled.

        Args:
            repeated_parser: A representation of the repeated parser

        Returns:
            A full error message
        """

        if self.finished:
            return 'Infinite recursion detected in {}; empty string was matched and will be matched forever at ' \
                   'end of source'.format(repeated_parser)
        else:
            return 'Infinite recursion detected in {}; empty string was matched and will be matched forever at ' \
                   'index {} before {}'.format(repeated_parser, self.position, self.next_token())

    def __repr__(self):
        if self.finished:
            return 'Reader(finished)'
        else:
            return 'Reader({}@{})'.format(self.first, self.position)

class SequenceReader(Reader):
    """A reader for sequences that should not be sliced.

    Python makes a copy when a sequence is sliced. This reader avoids making
    that copy by keeping one copy of source, passing it to the new reader when
    calling rest and using position to determine where in the source the reader
    should read from.

    Attributes:
        source (Sequence[Input]): What will be parsed.
    """

    def __init__(self, source: Sequence[Input], position: int = 0):
        self.source = source
        self.position = position

    @property
    def first(self) -> Input:
        return self.source[self.position]

    @property
    def rest(self) -> 'SequenceReader[Input]':
        return SequenceReader(self.source, self.position + 1)

    @property
    def finished(self) -> bool:
        return self.position >= len(self.source)

# Python lacks character type, so "str" will be used for both the sequence and the elements
class StringReader(Reader[str]):
    """A reader for strings.

    Python's regular expressions and string operations only work on strings,
    not abstract "readers". This class defines the source as a string so that
    regular expressions (and string literals) can safely and efficiently work
    directly on the source using the position to determine where to start.

    Attributes:
        source (str): What will be parsed.
    """

    def __init__(self, source: str, position: int = 0):
        self.source = source
        self.position = position

    @property
    def first(self) -> str:
        return self.source[self.position]

    @property
    def rest(self) -> 'StringReader[str]':
        return StringReader(self.source, self.position + 1)

    @property
    def finished(self) -> bool:
        return self.position >= len(self.source)

    def drop(self, count: int) -> 'StringReader[Input]':
        return StringReader(self.source, self.position + count)

    next_token_regex = re.compile(r'[\(\)\[\]\{\}\"\']|\w+|[^\w\s\(\)\[\]\{\}\"\']+|\s+')

    def next_token(self) -> str:
        match = self.next_token_regex.match(self.source, self.position)
        return self.source[match.start():match.end()]

    def current_line(self):
        characters_consumed = 0
        for line_index, line in enumerate(StringIO(self.source)):
            if characters_consumed + len(line) > self.position:
                # The line with the error has been found
                character_index = self.position - characters_consumed

                # This creates a line like this '        ^                  '
                pointer = (' ' * character_index) + '^' + (' ' * (len(line) - character_index - 1))

                # This adds a newline to line in case it is the end of the file
                if line[-1] != '\n':
                    line = line + '\n'

                # Add one to indexes to account for 0-indexes
                return line_index + 1, character_index + 1, line, pointer
            else:
                characters_consumed += len(line)

    def expected_error(self, expected: str) -> str:
        """Generate a basic error to include the current state.

        A parser can supply only a representation of what it is expecting to
        this method and the reader will provide the context, including the line
        and character positions.

        Args:
            expected: A representation of what the parser is currently expecting

        Returns:
            A full error message
        """

        if self.finished:
            return super().expected_error(expected)
        else:
            line_index, character_index, line, pointer = self.current_line()

            return 'Expected {} but found {}\nLine {}, character {}\n\n{}{}'.format(
                expected, repr(self.next_token()), line_index, character_index, line, pointer)

    def recursion_error(self, repeated_parser: str):
        """Generate an error to indicate that infinite recursion was encountered.

        A parser can supply a representation of itself to this method and the
        reader will supply the context, including the location where the
        parser stalled.

        Args:
            repeated_parser: A representation of the repeated parser

        Returns:
            A full error message
        """
        if self.finished:
            return super().recursion_error(repeated_parser)
        else:
            line_index, character_index, line, pointer = self.current_line()

            return 'Infinite recursion detected in {}; empty string was matched and will be matched forever\n' \
                   'Line {}, character {}\n\n{}{}'.format(repeated_parser, line_index, character_index, line, pointer)

    def __repr__(self):
        if self.finished:
            return 'StringReader(finished)'
        else:
            return 'StringReader({}@{})'.format(self.next_token(), self.position)

class Result(Generic[Output]):
    """Abstract algebraic base class for ``Success`` and ``Failure``.

    The class of all values returned from Parser.parse.
    """

    def or_die(self):
        raise NotImplementedError()

class Success(Generic[Output], Result[Output]):
    """Parsing succeeded.

    Returned from Parser.parse when the parser matched the source entirely.

    Attributes:
        value (Output): The value returned from the parser.
    """

    def __init__(self, value: Output):
        self.value = value

    def or_die(self):
        return self.value

    def __eq__(self, other):
        if isinstance(other, Success):
            return self.value == other.value
        else:
            return NotImplemented

    def __repr__(self):
        return 'Success({})'.format(repr(self.value))

class Failure(Generic[Output], Result[Output]):
    """Parsing failed.

    Returned from Parser.parse when the parser did not match the source or the
    source was not completely consumed.

    Attributes:
        message (str): A human-readable error from the farthest point reached
            during parsing.
    """

    def __init__(self, message: str):
        self.message = message

    def or_die(self):
        raise ParseError(self.message)

    def __eq__(self, other):
        if isinstance(other, Failure):
            return self.message == other.message
        else:
            return NotImplemented

    def __repr__(self):
        return 'Failure({})'.format(repr(self.message))

class ParseError(Exception):
    """Parsing failure converted to an exception.

    Raised when ``or_die`` method on ``Failure`` is called.

    Attributes:
        message (str): A human-readable error message
    """
    def __init__(self, message: str):
        self.message = message

    def __str__(self):
        return self.message

    def __repr__(self):
        return 'ParseError({})'.format(repr(self.message))

class Status(Generic[Input, Output]):
    farthest = None  # type: Optional[Reader]
    expected = ()  # type: Tuple[Callable[[], str]]

    def merge(self, status: 'Status[Input, Output]') -> 'Status[Input, Output]':
        """Merge the failure message from another status into this one.

        Whichever status represents parsing that has gone the farthest is
        retained. If both statuses have gone the same distance, then the
        expected values from both are retained.

        Args:
            status: The status to merge into this one.

        Returns:
            This ``Status`` which may have ``farthest`` and ``expected``
            updated accordingly.
        """
        if status is None or status.farthest is None:
            # No new message; simply return unchanged
            pass
        elif self.farthest is None:
            # No current message to compare to; use the message from status
            self.farthest = status.farthest
            self.expected = status.expected
        elif status.farthest.position < self.farthest.position:
            # New message is not farther; keep current message
            pass
        elif status.farthest.position > self.farthest.position:
            # New message is farther than current message; replace with new message
            self.farthest = status.farthest
            self.expected = status.expected
        else:
            # New message and current message are equally far; merge messages
            self.expected = status.expected + self.expected

        return self

class Continue(Generic[Input, Output], Status[Input, Output]):
    def __init__(self, remainder: Reader[Input], value: Output):
        self.remainder = remainder
        self.value = value

    def __repr__(self):
        return 'Continue({}, {})'.format(repr(self.value), repr(self.remainder))

class Backtrack(Generic[Input], Status[Input, None]):
    def __init__(self, farthest: Reader[Input], expected: Callable[[], str]):
        self.farthest = farthest
        self.expected = (expected,)

    def __repr__(self):
        return 'Backtrack({}, {})'.format(repr(self.farthest), list(map(lambda x: x(), self.expected)))

__all__ = ['Input', 'Output', 'Convert',
           'Reader', 'SequenceReader', 'StringReader',
           'Result', 'Success', 'Failure', 'ParseError', 'Status', 'Continue', 'Backtrack']


from __future__ import unicode_literals, absolute_import
from .syntax import *  # noqa

def exists(value):
    "Query to test if a value exists."
    if not isinstance(value, Token):
        raise TypeError('value must be a token')

    if not hasattr(value, 'identifier'):
        raise TypeError('value must support an identifier')

    if not value.identifier:
        value = value.__class__(**value.__dict__)
        value.identifier = 'v'

    ident = Identifier(value.identifier)

    return Query([
        OptionalMatch(value),
        Return(Predicate(ident, 'IS NOT NULL')),
        Limit(1),
    ])

def get(value):
    "Query to get the value."
    if not isinstance(value, Token):
        raise TypeError('value must be a token')

    if not hasattr(value, 'identifier'):
        raise TypeError('value must support an identifier')

    if not value.identifier:
        value = value.__class__(**value.__dict__)
        value.identifier = 'v'

    ident = Identifier(value.identifier)

    return Query([
        Match(value),
        Return(ident)
    ])


from typing import Callable, TypeVar, Iterable

A = TypeVar('Input')

def constant(x: A) -> Callable[..., A]:
    """Produce a function that always returns a supplied value.

    Args:
        x: Any object.

    Returns:
        A function that accepts any number of positional and keyword arguments, discards them, and returns ``x``.
    """

    def constanted(*args, **kwargs):
        return x

    return constanted

def splat(f: Callable[..., A]) -> Callable[[Iterable], A]:
    """Convert a function taking multiple arguments into a function taking a single iterable argument.

    Args:
        f: Any function

    Returns:
        A function that accepts a single iterable argument. Each element of this iterable argument is passed as an
        argument to ``f``.

    Example:
        $ def f(a, b, c):
        $     return a + b + c
        $
        $ f(1, 2, 3)  # 6
        $ g = splat(f)
        $ g([1, 2, 3])  # 6
    """

    def splatted(args):
        return f(*args)

    return splatted

def unsplat(f: Callable[[Iterable], A]) -> Callable[..., A]:
    """Convert a function taking a single iterable argument into a function taking multiple arguments.

    Args:
        f: Any function taking a single iterable argument

    Returns:
        A function that accepts multiple arguments. Each argument of this function is passed as an element of an
        iterable to ``f``.

    Example:
        $ def f(a):
        $     return a[0] + a[1] + a[2]
        $
        $ f([1, 2, 3])  # 6
        $ g = unsplat(f)
        $ g(1, 2, 3)  # 6
    """

    def unsplatted(*args):
        return f(args)

    return unsplatted

__all__ = ['constant', 'splat', 'unsplat']


# Copyright (c) Moshe Zadka
# See LICENSE for details.
"""ncolony.schedulelib
========================

Construct a Twisted service for process scheduling.

.. code-block:: bash

   $ twistd -n ncolonysched --timeout 2 --grace 1 --frequency 10
            --arg /bin/echo --arg hello
"""
from __future__ import print_function

import os

from zope import interface

from twisted.python import usage

from twisted.internet import interfaces as tiinterfaces
from twisted.internet import defer
from twisted.internet import error as tierror
from twisted.internet import reactor as tireactor

from twisted.application import internet as tainternet, service

from ncolony.client import heart

@interface.implementer(tiinterfaces.IProcessProtocol)
class ProcessProtocol(object):

    """Process protocol that manages short-lived processes"""

    def __init__(self, deferred):
        self.deferred = deferred

    # pylint: disable=no-self-use
    def childDataReceived(self, fd, data):
        """Log data from process

        :params fd: File descriptor data is coming from
        :params data: The bytes the process returned
        """
        for line in data.splitlines():
            print('[%d]' % fd, line)
    # pylint: enable=no-self-use

    def processEnded(self, reason):
        """Report process end to deferred

        :params reason: a Failure
        """
        self.deferred.errback(reason)

    def processExited(self, reason):
        """Ignore processExited"""
        pass

    def childConnectionLost(self, reason):
        """Ignore childConnectionLoss"""
        pass

    def makeConnection(self, transport):
        """Ignore makeConnection"""
        pass

def runProcess(args, timeout, grace, reactor):
    """Run a process, return a deferred that fires when it is done

    :params args: Process arguments
    :params timeout: Time before terminating process
    :params grace: Time before killing process after terminating it
    :params reactor: IReactorProcess and IReactorTime
    :returns: deferred that fires with success when the process ends,
              or fails if there was a problem spawning/terminating
              the process
    """
    deferred = defer.Deferred()
    protocol = ProcessProtocol(deferred)
    process = reactor.spawnProcess(protocol, args[0], args, env=os.environ)

    def _logEnded(err):
        err.trap(tierror.ProcessDone, tierror.ProcessTerminated)
        print(err.value)
    deferred.addErrback(_logEnded)

    def _cancelTermination(dummy):
        for termination in terminations:
            if termination.active():
                termination.cancel()
    deferred.addCallback(_cancelTermination)
    terminations = []
    terminations.append(reactor.callLater(timeout, process.signalProcess,
                                          "TERM"))
    terminations.append(reactor.callLater(timeout+grace,
                                          process.signalProcess, "KILL"))
    return deferred

class Options(usage.Options):

    """Options for scheduler service"""

    optParameters = [
        ['timeout', None, None, 'Time before terminating the command', int],
        ['grace', None, None,
         'Time between terminating the command and sending an umaskable kill',
         int],
        ['frequency', None, None, 'How often to run the command', int],
    ]

    def __init__(self):
        usage.Options.__init__(self)
        self['args'] = []

    def opt_arg(self, arg):
        """Argument"""
        self['args'].append(arg)

    def postOptions(self):
        for elem in ['args', 'timeout', 'grace', 'frequency']:
            if not self[elem]:
                raise ValueError(elem)

def makeService(opts):
    """Make scheduler service

    :params opts: dict-like object.
       keys: frequency, args, timeout, grace
    """
    ser = tainternet.TimerService(opts['frequency'], runProcess, opts['args'],
                                  opts['timeout'], opts['grace'], tireactor)
    ret = service.MultiService()
    ser.setName('scheduler')
    ser.setServiceParent(ret)
    heart.maybeAddHeart(ret)
    return ret