# coding: utf-8

"""
Copyright 2015 SmartBear Software

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

   ref: https://github.com/swagger-api/swagger-codegen
"""

from __future__ import absolute_import
from . import models
from .rest import RESTClientObject
from .rest import ApiException

import os
import re
import sys
import urllib
import json
import mimetypes
import random
import tempfile
import threading

from datetime import datetime
from datetime import date

# python 2 and python 3 compatibility library
from six import iteritems

try:
    # for python3
    from urllib.parse import quote
except ImportError:
    # for python2
    from urllib import quote

from .configuration import Configuration

class ApiClient(object):
    """
    Generic API client for Swagger client library builds.

    Swagger generic API client. This client handles the client-
    server communication, and is invariant across implementations. Specifics of
    the methods and models for each application are generated from the Swagger
    templates.

    NOTE: This class is auto generated by the swagger code generator program.
    Ref: https://github.com/swagger-api/swagger-codegen
    Do not edit the class manually.

    :param host: The base path for the server to call.
    :param header_name: a header to pass when making calls to the API.
    :param header_value: a header value to pass when making calls to the API.
    """
    def __init__(self, host=None, header_name=None, header_value=None, cookie=None):

        """
        Constructor of the class.
        """
        self.rest_client = RESTClientObject()
        self.default_headers = {}
        if header_name is not None:
            self.default_headers[header_name] = header_value
        if host is None:
            self.host = Configuration().host
        else:
            self.host = host
        self.cookie = cookie
        # Set default User-Agent.
        self.user_agent = 'Python-Swagger/1.0.0'

    @property
    def user_agent(self):
        """
        Gets user agent.
        """
        return self.default_headers['User-Agent']

    @user_agent.setter
    def user_agent(self, value):
        """
        Sets user agent.
        """
        self.default_headers['User-Agent'] = value

    def set_default_header(self, header_name, header_value):
        self.default_headers[header_name] = header_value

    def __call_api(self, resource_path, method,
                   path_params=None, query_params=None, header_params=None,
                   body=None, post_params=None, files=None,
                   response_type=None, auth_settings=None, callback=None):

        # headers parameters
        header_params = header_params or {}
        header_params.update(self.default_headers)
        if self.cookie:
            header_params['Cookie'] = self.cookie
        if header_params:
            header_params = self.sanitize_for_serialization(header_params)

        # path parameters
        if path_params:
            path_params = self.sanitize_for_serialization(path_params)
            for k, v in iteritems(path_params):
                replacement = quote(str(self.to_path_value(v)))
                resource_path = resource_path.\
                    replace('{' + k + '}', replacement)

        # query parameters
        if query_params:
            query_params = self.sanitize_for_serialization(query_params)
            query_params = {k: self.to_path_value(v)
                            for k, v in iteritems(query_params)}

        # post parameters
        if post_params or files:
            post_params = self.prepare_post_parameters(post_params, files)
            post_params = self.sanitize_for_serialization(post_params)

        # auth setting
        self.update_params_for_auth(header_params, query_params, auth_settings)

        # body
        if body:
            body = self.sanitize_for_serialization(body)

        # request url
        url = self.host + resource_path

        # perform request and return response
        response_data = self.request(method, url,
                                     query_params=query_params,
                                     headers=header_params,
                                     post_params=post_params, body=body)

        self.last_response = response_data

        # deserialize response data
        if response_type:
            deserialized_data = self.deserialize(response_data, response_type)
        else:
            deserialized_data = None

        if callback:
            callback(deserialized_data)
        else:
            return deserialized_data

    def to_path_value(self, obj):
        """
        Takes value and turn it into a string suitable for inclusion in
        the path, by url-encoding.

        :param obj: object or string value.

        :return string: quoted value.
        """
        if type(obj) == list:
            return ','.join(obj)
        else:
            return str(obj)

    def sanitize_for_serialization(self, obj):
        """
        Builds a JSON POST object.

        If obj is None, return None.
        If obj is str, int, float, bool, return directly.
        If obj is datetime.datetime, datetime.date
            convert to string in iso8601 format.
        If obj is list, sanitize each element in the list.
        If obj is dict, return the dict.
        If obj is swagger model, return the properties dict.

        :param obj: The data to serialize.
        :return: The serialized form of data.
        """
        types = (str, int, float, bool, tuple)
        if sys.version_info < (3,0):
            types = types + (unicode,)
        if isinstance(obj, type(None)):
            return None
        elif isinstance(obj, types):
            return obj
        elif isinstance(obj, list):
            return [self.sanitize_for_serialization(sub_obj)
                    for sub_obj in obj]
        elif isinstance(obj, (datetime, date)):
            return obj.isoformat()
        else:
            if isinstance(obj, dict):
                obj_dict = obj
            else:
                # Convert model obj to dict except
                # attributes `swagger_types`, `attribute_map`
                # and attributes which value is not None.
                # Convert attribute name to json key in
                # model definition for request.
                obj_dict = {obj.attribute_map[attr]: getattr(obj, attr)
                            for attr, _ in iteritems(obj.swagger_types)
                            if getattr(obj, attr) is not None}

            return {key: self.sanitize_for_serialization(val)
                    for key, val in iteritems(obj_dict)}

    def deserialize(self, response, response_type):
        """
        Deserializes response into an object.

        :param response: RESTResponse object to be deserialized.
        :param response_type: class literal for
            deserialzied object, or string of class name.

        :return: deserialized object.
        """
        # handle file downloading
        # save response body into a tmp file and return the instance
        if "file" == response_type:
            return self.__deserialize_file(response)

        # fetch data from response object
        try:
            data = json.loads(response.data)
        except ValueError:
            data = response.data

        return self.__deserialize(data, response_type)

    def __deserialize(self, data, klass):
        """
        Deserializes dict, list, str into an object.

        :param data: dict, list or str.
        :param klass: class literal, or string of class name.

        :return: object.
        """
        if data is None:
            return None

        if type(klass) == str:
            if klass.startswith('list['):
                sub_kls = re.match('list\[(.*)\]', klass).group(1)
                return [self.__deserialize(sub_data, sub_kls)
                        for sub_data in data]

            if klass.startswith('dict('):
                sub_kls = re.match('dict\(([^,]*), (.*)\)', klass).group(2)
                return {k: self.__deserialize(v, sub_kls)
                        for k, v in iteritems(data)}

            # convert str to class
            # for native types
            if klass in ['int', 'float', 'str', 'bool',
                         "date", 'datetime', "object"]:
                klass = eval(klass)
            # for model types
            else:
                klass = eval('models.' + klass)

        if klass in [int, float, str, bool]:
            return self.__deserialize_primitive(data, klass)
        elif klass == object:
            return self.__deserialize_object(data)
        elif klass == date:
            return self.__deserialize_date(data)
        elif klass == datetime:
            return self.__deserialize_datatime(data)
        else:
            return self.__deserialize_model(data, klass)

    def call_api(self, resource_path, method,
                 path_params=None, query_params=None, header_params=None,
                 body=None, post_params=None, files=None,
                 response_type=None, auth_settings=None, callback=None):
        """
        Makes the HTTP request (synchronous) and return the deserialized data.
        To make an async request, define a function for callback.

        :param resource_path: Path to method endpoint.
        :param method: Method to call.
        :param path_params: Path parameters in the url.
        :param query_params: Query parameters in the url.
        :param header_params: Header parameters to be
            placed in the request header.
        :param body: Request body.
        :param post_params dict: Request post form parameters,
            for `application/x-www-form-urlencoded`, `multipart/form-data`.
        :param auth_settings list: Auth Settings names for the request.
        :param response: Response data type.
        :param files dict: key -> filename, value -> filepath,
            for `multipart/form-data`.
        :param callback function: Callback function for asynchronous request.
            If provide this parameter,
            the request will be called asynchronously.
        :return:
            If provide parameter callback,
            the request will be called asynchronously.
            The method will return the request thread.
            If parameter callback is None,
            then the method will return the response directly.
        """
        if callback is None:
            return self.__call_api(resource_path, method,
                                   path_params, query_params, header_params,
                                   body, post_params, files,
                                   response_type, auth_settings, callback)
        else:
            thread = threading.Thread(target=self.__call_api,
                                      args=(resource_path, method,
                                            path_params, query_params,
                                            header_params, body,
                                            post_params, files,
                                            response_type, auth_settings,
                                            callback))
        thread.start()
        return thread

    def request(self, method, url, query_params=None, headers=None,
                post_params=None, body=None):
        """
        Makes the HTTP request using RESTClient.
        """
        if method == "GET":
            return self.rest_client.GET(url,
                                        query_params=query_params,
                                        headers=headers)
        elif method == "HEAD":
            return self.rest_client.HEAD(url,
                                         query_params=query_params,
                                         headers=headers)
        elif method == "OPTIONS":
            return self.rest_client.OPTIONS(url,
                                            query_params=query_params,
                                            headers=headers,
                                            post_params=post_params,
                                            body=body)
        elif method == "POST":
            return self.rest_client.POST(url,
                                         query_params=query_params,
                                         headers=headers,
                                         post_params=post_params,
                                         body=body)
        elif method == "PUT":
            return self.rest_client.PUT(url,
                                        query_params=query_params,
                                        headers=headers,
                                        post_params=post_params,
                                        body=body)
        elif method == "PATCH":
            return self.rest_client.PATCH(url,
                                          query_params=query_params,
                                          headers=headers,
                                          post_params=post_params,
                                          body=body)
        elif method == "DELETE":
            return self.rest_client.DELETE(url,
                                           query_params=query_params,
                                           headers=headers)
        else:
            raise ValueError(
                "http method must be `GET`, `HEAD`,"
                " `POST`, `PATCH`, `PUT` or `DELETE`."
            )

    def prepare_post_parameters(self, post_params=None, files=None):
        """
        Builds form parameters.

        :param post_params: Normal form parameters.
        :param files: File parameters.
        :return: Form parameters with files.
        """
        params = {}

        if post_params:
            params.update(post_params)

        if files:
            for k, v in iteritems(files):
                if not v:
                    continue

                with open(v, 'rb') as f:
                    filename = os.path.basename(f.name)
                    filedata = f.read()
                    mimetype = mimetypes.\
                        guess_type(filename)[0] or 'application/octet-stream'
                    params[k] = tuple([filename, filedata, mimetype])

        return params

    def select_header_accept(self, accepts):
        """
        Returns `Accept` based on an array of accepts provided.

        :param accepts: List of headers.
        :return: Accept (e.g. application/json).
        """
        if not accepts:
            return

        accepts = list(map(lambda x: x.lower(), accepts))

        if 'application/json' in accepts:
            return 'application/json'
        else:
            return ', '.join(accepts)

    def select_header_content_type(self, content_types):
        """
        Returns `Content-Type` based on an array of content_types provided.

        :param content_types: List of content-types.
        :return: Content-Type (e.g. application/json).
        """
        if not content_types:
            return 'application/json'

        content_types = list(map(lambda x: x.lower(), content_types))

        if 'application/json' in content_types:
            return 'application/json'
        else:
            return content_types[0]

    def update_params_for_auth(self, headers, querys, auth_settings):
        """
        Updates header and query params based on authentication setting.

        :param headers: Header parameters dict to be updated.
        :param querys: Query parameters dict to be updated.
        :param auth_settings: Authentication setting identifiers list.
        """
        config = Configuration()

        if not auth_settings:
            return

        for auth in auth_settings:
            auth_setting = config.auth_settings().get(auth)
            if auth_setting:
                if not auth_setting['value']:
                    continue
                elif auth_setting['in'] == 'header':
                    headers[auth_setting['key']] = auth_setting['value']
                elif auth_setting['in'] == 'query':
                    querys[auth_setting['key']] = auth_setting['value']
                else:
                    raise ValueError(
                        'Authentication token must be in `query` or `header`'
                    )

    def __deserialize_file(self, response):
        """
        Saves response body into a file in a temporary folder,
        using the filename from the `Content-Disposition` header if provided.

        :param response:  RESTResponse.
        :return: file path.
        """
        config = Configuration()

        fd, path = tempfile.mkstemp(dir=config.temp_folder_path)
        os.close(fd)
        os.remove(path)

        content_disposition = response.getheader("Content-Disposition")
        if content_disposition:
            filename = re.\
                search(r'filename=[\'"]?([^\'"\s]+)[\'"]?', content_disposition).\
                group(1)
            path = os.path.join(os.path.dirname(path), filename)

        with open(path, "w") as f:
            f.write(response.data)

        return path

    def __deserialize_primitive(self, data, klass):
        """
        Deserializes string to primitive type.

        :param data: str.
        :param klass: class literal.

        :return: int, float, str, bool.
        """
        try:
            value = klass(data)
        except UnicodeEncodeError:
            value = unicode(data)
        except TypeError:
            value = data
        return value

    def __deserialize_object(self, value):
        """
        Return a original value.

        :return: object.
        """
        return value

    def __deserialize_date(self, string):
        """
        Deserializes string to date.

        :param string: str.
        :return: date.
        """
        try:
            from dateutil.parser import parse
            return parse(string).date()
        except ImportError:
            return string
        except ValueError:
            raise ApiException(
                status=0,
                reason="Failed to parse `{0}` into a date object"
                .format(string)
            )

    def __deserialize_datatime(self, string):
        """
        Deserializes string to datetime.

        The string should be in iso8601 datetime format.

        :param string: str.
        :return: datetime.
        """
        try:
            from dateutil.parser import parse
            return parse(string)
        except ImportError:
            return string
        except ValueError:
            raise ApiException(
                status=0,
                reason="Failed to parse `{0}` into a datetime object".
                format(string)
            )

    def __deserialize_model(self, data, klass):
        """
        Deserializes list or dict to model.

        :param data: dict, list.
        :param klass: class literal.
        :return: model object.
        """
        instance = klass()

        for attr, attr_type in iteritems(instance.swagger_types):
            if data is not None \
               and instance.attribute_map[attr] in data\
               and isinstance(data, (list, dict)):
                value = data[instance.attribute_map[attr]]
                setattr(instance, attr, self.__deserialize(value, attr_type))

        return instance


# -*- coding: utf-8 -*-
# vim:fenc=utf-8

'''
  Core Flask RESTFul app
  ----------------------

  :copyright (c) 2014 Xavier Bruhiere.
  :license: MIT, see LICENSE for more details.
'''

import os
from docopt import docopt
import flask
import mongoengine
from flask.ext import restful
from flask.ext.mongoengine import MongoEngine
import dna.logging

log = dna.logging.logger(__name__)

DEFAULT_DOC = '''Apy api

Usage:
  api -h | --help
  api --version
  api [--bind=<ip>] [--port=<port>] [-d | --debug] [--log=<level>]

Options:
  -h --help       Show this screen.
  --version       Show version.
  --debug         Activates Flask debug
  --bind=<ip>     Listens on the given ip [default: 127.0.0.1]
  --port=<port>   Listens on the given port [default: 5000]
  --log=<level>   Log output level [default: debug]
'''

class App(object):
    '''
    Setup a flask server
    '''

    def __init__(self, sentry_dns=None):
        ''' Setup default Flask app and sentry '''
        self.app = flask.Flask(__name__)
        self.app.config.update(
            PROPAGATE_EXCEPTIONS=False,
            PRESERVE_CONTEXT_ON_EXCEPTION=True,
            SECRET_KEY='vbrzbbzrrbz'
        )

        # Optionally activate Sentry (https://getsentry.com/welcome/)
        sentry_dns = sentry_dns or os.environ.get('SENTRY_DNS')
        if sentry_dns:
            from raven.contrib.flask import Sentry
            Sentry(self.app, dsn=sentry_dns)

    def setup_routes(self, routes):
        api = restful.Api(self.app)
        for endpoint, resource in routes.iteritems():
            api.add_resource(resource, endpoint)

    def setup_db(self):
        self.app.config.update(
            MONGODB_SETTINGS={
                'DB': os.environ.get('MONGODB_NAME', 'apy'),
                'HOST': os.environ.get('MONGODB_HOST', 'localhost'),
                'PORT': int(os.environ.get('MONGODB_PORT', 27017))
            }
        )
        try:
            self.db = MongoEngine(self.app)
        except mongoengine.connection.ConnectionError as error:
            log.error(error)
            raise

    def serve(self, app_docopt=DEFAULT_DOC, description=''):
        ''' Configure from cli and run the server '''

        exit_status = 0
        if isinstance(app_docopt, str):
            args = docopt(app_docopt, version=description)
        elif isinstance(app_docopt, dict):
            args = app_docopt
        else:
            raise ValueError('unknown configuration object ({})'
                             .format(type(app_docopt)))
        log_level = args.get('--log', 'debug')
        is_debug = args.get('--debug', False)
        # TODO More serious default
        log_output = 'stdout' if is_debug else 'apy.log'
        safe_bind = args.get('--bind', '127.0.0.1')
        safe_port = int(args.get('--port', 5000))
        log_setup = dna.logging.setup(level=log_level, output=log_output)

        with log_setup.applicationbound():
            try:
                log.info('server ready',
                         version=description,
                         log=log_level,
                         debug=is_debug,
                         bind='{}:{}'.format(safe_bind, safe_port))

                self.app.run(host=safe_bind,
                             port=safe_port,
                             debug=is_debug)

            except Exception as error:
                if is_debug:
                    raise
                log.error('{}: {}'.format(type(error).__name__, str(error)))
                exit_status = 1

            finally:
                log.info('session ended with status {}'.format(exit_status))

        return exit_status

Application = App()

if __name__ == '__main__':
    import dna.apy.resources as resources
    routes = {
        '/users/<string:username>': resources.User,
        '/test': resources.TestRestricted
    }
    Application.setup_routes(routes)
    Application.serve()


from django import forms
from django.template.loader import render_to_string

__all__ = ('WYSIWYGWidget', )

class WYSIWYGWidget(forms.Textarea):
    """Custom bootstrap-wysiwyg widget."""
    template_name = 'secure_input/widgets/full_wysiwyg.html'

    def render(self, name, value, attrs=None):
        """Include a hidden input to stored the serialized upload value."""
        context = attrs or {}
        context.update({'name': name, 'value': value, })
        return render_to_string(self.template_name, context)

    class Media:
        css = {
            'all': (
                'secure_input/bootstrap/css/bootstrap.css',
                'secure_input/font-awesome/css/font-awesome.css',
                'secure_input/css/basicEditor.css',
            )
        }
        js = (
            'secure_input/js/libs/jquery.js',
            'secure_input/bootstrap/js/bootstrap.js',
            'secure_input/bootstrap-wysiwyg/external/jquery.hotkeys.js',
            'secure_input/bootstrap-wysiwyg/bootstrap-wysiwyg.js',
            'secure_input/js/plugin.js',
        )

class MiniWYSIWYGWidget(WYSIWYGWidget):
    """Renders a bootstrap-wysiwyg widget with very few tools."""
    template_name = 'secure_input/widgets/mini_wysiwyg.html'


# Copyright (c) 2015, Narrative Science
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
from __future__ import print_function
import os
import shlex
import subprocess
import sys

from threading import Thread

from lsi.utils.term import random_color, get_color_hash

DEFAULT_COLOR = None

def decode(string):
    """Ensure that `string` is in unicode format."""
    if hasattr(string, "decode"):
        return string.decode("utf-8")
    return string

def stream_command(command, formatter=None, write_stdin=None, ignore_empty=False):
    """
    Starts `command` in a subprocess. Prints every line the command prints,
    prefaced with `description`.

    :param command: The bash command to run. Must use fully-qualified paths.
    :type command: ``str``
    :param formatter: An optional formatting function to apply to each line.
    :type formatter: ``function`` or ``NoneType``
    :param write_stdin: An optional string to write to the process' stdin.
    :type write_stdin: ``str`` or ``NoneType``
    :param ignore_empty: If true, empty or whitespace-only lines will be skipped.
    :type ignore_empty: ``bool``
    """
    command_list = shlex.split(command)
    try:
        proc = subprocess.Popen(command_list, stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT, stdin=subprocess.PIPE)
    except Exception as e:
        raise IOError('Encountered error: {0} when running command {1}'
                      .format(e.message, ' '.join(command_list)))
    if write_stdin is not None:
        proc.stdin.write(write_stdin)
        proc.stdin.flush()

    while proc.poll() is None:
        try:
            line = proc.stdout.readline()
        except KeyboardInterrupt:
            sys.exit('Keyboard interrupt while running {}'.format(command))
        if len(line.strip()) == 0 and ignore_empty is True:
            continue
        elif 'killed by signal 1' in decode(line).lower():
            continue
        elif 'to the list of known hosts' in decode(line).lower():
            continue
        if formatter is not None:
            line = formatter(line)
        sys.stdout.write(line)
    result = proc.poll()
    return result

def stream_command_dicts(commands, parallel=False):
    """
    Takes a list of dictionaries with keys corresponding to ``stream_command``
    arguments, and runs all concurrently.

    :param commands: A list of dictionaries, the keys of which should line up
                     with the arguments to ``stream_command`` function.
    :type commands: ``list`` of ``dict``
    :param parallel: If true, commands will be run in parallel.
    :type parallel: ``bool``
    """
    if parallel is True:
        threads = []
        for command in commands:
            target = lambda: stream_command(**command)
            thread = Thread(target=target)
            thread.start()
            threads.append(thread)
        for t in threads:
            t.join()
    else:
        for command in commands:
            stream_command(**command)

def _format_with_description(description):
    def _fmat(line):
        return '[{0}]: {1}'.format(description, line)
    return _fmat

def stream_commands(commands, hash_colors=True, parallel=False):
    """
    Runs multiple commands, optionally in parallel. Each command should be
    a dictionary with a 'command' key and optionally 'description' and
    'write_stdin' keys.
    """
    def _get_color(string):
        if hash_colors is True:
            return get_color_hash(string)
        else:
            return DEFAULT_COLOR
    fixed_commands = []
    for command in commands:
        cmd_text = command['command']
        description = command.get('description')
        color = _get_color(description or '')
        write_stdin = command.get('write_stdin')
        description = color(description) if color is not None else description
        formatter = _format_with_description(description)
        fixed_commands.append({
            'command': cmd_text,
            'formatter': formatter,
            'write_stdin': write_stdin,
            'ignore_empty': True
        })
    stream_command_dicts(fixed_commands, parallel=parallel)


from rhcalendar.locales import locales
import workdays

__version__ = '1.1.0'

def networkdays(from_date, to_date, locale='en-US'):
    """ Return the net work days according to RH's calendar. """
    holidays = locales[locale]
    return workdays.networkdays(from_date, to_date, holidays)


import os
import json
import collections
import copy
import functools

yaml = None
xmltodict = None

class Dicto(object):

    def __init__(self, dict_=None, **kwargs):

        if dict_ is None:
            dict_ = {}

        if not isinstance(dict_, dict):
            raise ValueError("dict_ parameters is not a python dict")

        dict_.update(kwargs)

        to_dicto(dict_, dicto = self)

    def __setitem__(self, key, item):
        # self._dict[key] = item
        setattr(self, key, item)

    def __getitem__(self, key):
        return getattr(self, key)

    def __repr__(self):
        return repr(self.__dict__)

    def __len__(self):
        return len(self.__dict__)

    def __delitem__(self, key):
        delattr(self, key)

    def __contains__(self, item):
        return hasattr(self, item)

    def __iter__(self):
        return iter(self.__dict__)

def to_dicto(obj, dicto = None):

    if isinstance(obj, Dicto):
        return obj

    elif isinstance(obj, dict):

        if dicto is None:
            dicto = Dicto()

        for key, value in obj.items():

            value = to_dicto(value)

            setattr(dicto, key, value)

        return dicto

    elif isinstance(obj, str):
        return obj

    elif isinstance(obj, list):
        return [ to_dicto(x) for x in obj ]

    elif isinstance(obj, tuple):
        return tuple([ to_dicto(x) for x in obj ])

    elif hasattr(obj, "__iter__"):
        return ( to_dicto(x) for x in obj )

    else:
        return obj

def to_dict(obj, dict_ = None):

    if isinstance(obj, dict):
        return obj

    elif isinstance(obj, Dicto):

        if dict_ is None:
            dict_ = dict()

        for key, value in obj.__dict__.items():

            dict_[key] = to_dict(value)

        return dict_

    elif isinstance(obj, str):
        return obj

    elif isinstance(obj, list):
        return [ to_dict(x) for x in obj ]

    elif isinstance(obj, tuple):
        return tuple([ to_dict(x) for x in obj ])

    elif hasattr(obj, "__iter__"):
        return ( to_dict(x) for x in obj )

    else:
        return obj

def merge(dicto, other):
    """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of
    updating only top-level keys, dict_merge recurses down into dicts nested
    to an arbitrary depth, updating keys. The ``other`` is merged into
    ``dicto``.
    :param dicto: dict onto which the merge is executed
    :param other: dict that is going to merged into dicto
    :return: None
    """
    if not isinstance(dicto, Dicto):
        dicto = Dicto(dicto)

    if not isinstance(other, Dicto):
        other = Dicto(other)

    for k, v in other.__dict__.items():
        if k in dicto and isinstance(dicto[k], Dicto) and isinstance(other[k], Dicto):
            dicto[k] = merge(dicto[k], other[k])
        else:
            dicto[k] = other[k]

    return dicto

def load(filepath, as_dicto = True):
    filepath = os.path.realpath(filepath)

    if filepath.endswith(".yaml") or filepath.endswith(".yml"):
        global yaml

        if not yaml:
            import yaml as mod
            yaml = mod

        with open(filepath, 'r') as stream:
            dict_ = yaml.load(stream)

    elif filepath.endswith(".json"):

        with open(filepath, 'r') as stream:
            dict_ = json.load(stream)

    elif filepath.endswith(".xml"):
        global xmltodict

        if not xmltodict:
            import xmltodict as mod
            xmltodict = mod

        with open(filepath, 'r') as stream:
            dict_ = xmltodict.parse(stream.read())

    else:
        raise Exception("File type not supported.")

    if as_dicto:
        return to_dicto(dict_)
    else:
        return dict_

def dump(dicto, filepath):

    filepath = os.path.realpath(filepath)
    obj = to_dict(dicto)

    if filepath.endswith(".yaml") or filepath.endswith(".yml"):
        with open(filepath, 'w') as stream:
            yaml.safe_dump(obj, stream, default_flow_style=False)
    elif filepath.endswith(".json"):
        with open(filepath, 'w') as stream:
            json.dump(obj, stream)
    else:
        raise Exception("File type not supported.")

def fire_options(config_path, single_argument = None, as_dicto = True, use_environment = False):
    import fire

    if isinstance(config_path, dict):
        dict_ = config_path
    else:
        dict_ = load(config_path, as_dicto=False)

    if not isinstance(dict_, dict):
        raise TypeError("File {config_path} was loaded as a {type}, expected a dict.".format(config_path=config_path, type=type(dict_)))

    if use_environment:
        for key in dict_:
            if key in os.environ:
                value = os.environ[key]
                dict_[key] = fire.parser.DefaultParseValue(value)
            elif key.upper() in os.environ:
                value = os.environ[key.upper()]
                dict_[key] = fire.parser.DefaultParseValue(value)

    def decorator(f):

        @functools.wraps(f)
        def final_f(*args, **kwargs):

            if single_argument is not None:

                params = Dicto(dict_) if as_dicto else dict(dict_)

                for flag in dict_:
                    if flag in kwargs:
                        params[flag] = kwargs.pop(flag)

                kwargs[single_argument] = params

            else:
                dict_.update(kwargs)
                kwargs = dict_

            return f(*args, **kwargs)

        return final_f

    return decorator

def kwargs_dicto(arg_name):

    def decorator(f):

        @functools.wraps(f)
        def final_f(*args, **kwargs):

            kwargs = {arg_name: Dicto(kwargs)}

            return f(*args, **kwargs)

        return final_f

    return decorator

def click_options(config_path, single_argument = None, as_dicto = True, underscore_to_dash = True):
    import click

    if isinstance(config_path, dict):
        dict_ = config_path
    else:
        dict_ = load(config_path, as_dicto=False)

    if not isinstance(dict_, dict):
        raise TypeError("File {config_path} was loaded as a {type}, expected a dict.".format(config_path=config_path, type=type(dict_)))

    def decorator(f):

        for flag, kwargs in dict_.items():

            op_flag = flag.replace('_', '-') if underscore_to_dash else flag
            op_flag = "--" + op_flag

            if not isinstance(kwargs, dict):
                kwargs = dict(default = kwargs)

            if "default" in kwargs and not "type" in kwargs:
                kwargs["type"] = type(kwargs["default"])

            f = click.option(op_flag, **kwargs)(f)

        if single_argument is not None:
            params = Dicto() if as_dicto else dict()

            def final_f(*args, **final_kwargs):

                for kwarg in dict_:
                    kwarg = kwarg.replace("-", "_")

                    if kwarg in final_kwargs:
                        params[kwarg] = final_kwargs.pop(kwarg)

                final_kwargs[single_argument] = params

                return f(*args, **final_kwargs)

            final_f.__click_params__ = f.__click_params__
        else:
            final_f = f

        final_f = functools.update_wrapper(final_f, f)

        return final_f

    return decorator

# legacy
click_options_config = click_options


# Copyright (c) 2015-2016, Narrative Science
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#!/usr/bin/env python

"""
A simple script for listing (sorted by tag:Name) all running instances
within ns's ec2 east set. It will also return the public dns name of
an instance if the `--host` argument is used. Writes results to a cache
(just a text file) to speed up repeated queries. However, if the latest
list is needed, you can pass `--latest` (`-l`) as an argument.

Any additional command-line arguments will act as regex filters on the list of
instances (similar to running `grep`). Excluding patterns can be provided with
the `-v` (or `--exclude`) argument.

This also provides the ability to SSH into an instance. Passing the argument
`-s` (or `--ssh`) will present the list of instances and let the user choose
which to SSH into. The user can choose to log in a specific user by passing
the argument `-u` (or `--username`), and can specify an identity file with
`-i` (or `--identity_file`).

::
    # Use cache, if it exists and is not too old.
    > list_instances.py

    # Pull latest results from AWS, regardless of the cache.
    > list_instances.py --latest

    # Returns the public DNS name of <host>.
    > list_instances.py --host <host>

    # Filters with `wkfl` and `stg`, and runs the SSH dialog.
    > list_instances.py wkfl stg -s

    # Filters with `wkfl` and `stg`, excludes `batch`, and runs the SSH
    # dialog. Will connect as user `ubuntu`.
    > list_instances.py wkfl stg -e batch -s -u ubuntu
"""
from __future__ import print_function
import argparse
import os
from os.path import dirname, join, expanduser, exists
import random as py_random
import sys
import subprocess

# The configparser module was renamed in python3. For the other
# imports, they just seem to not work the same in python2 and
# python3. Oh well.
if sys.version_info >= (3, 0):
    from configparser import ConfigParser
    from lsi.utils.hosts import (HostEntry, get_entries, filter_entries,
                                 get_host)
    from lsi.utils.term import (green, yellow, cyan, blue, get_input)
    from lsi.utils.system import get_current_username
    from lsi.utils.stream import stream_commands
else:
    from ConfigParser import ConfigParser
    from utils.hosts import (HostEntry, get_entries, filter_entries, get_host)
    from utils.term import (green, yellow, cyan, blue, get_input)
    from utils.system import get_current_username
    from utils.stream import stream_commands

# String displayed at the command prompt.
COMMANDS_STRING = \
'''{commands}:
  h: Show this message
  <number {n}>: Connect to the {n}th instance in the list
  p {profile}: Use profile {profile}
  u {username}: Change SSH username to {username} (currently {_username})
  i {idfile}: Change identity file to {idfile} (currently {_idfile})
  f <one or more {pattern}s>: Restrict results to those with {pattern}s
  e <one or more {pattern}s>: Restrict results to those without {pattern}s
  limit {n}: Limit output to first {n} lines
  sort <{attribute}>: Sort the list by {attribute}
  show <one or more {attribute}s>: Additionally show those {attribute}s
  c <{command}>: Set ssh command to run on matching hosts (currently {cur_cmd})
  x: Execute the above command on the above host(s)
  {q}: Quit
'''.format(commands=green('Commands'),
                          n=cyan('n'),
                          profile=cyan('profile'),
                          attribute=cyan('attribute'),
                          username=cyan('username'), _username='{username}',
                          idfile=cyan('idfile'), _idfile='{idfile}',
                          pattern=cyan('pattern'), command=cyan('command'),
                          cur_cmd='{cur_cmd}',
                          q=cyan('q'))

# Path to where the SSH known hosts will be stored.
_KNOWN_HOSTS_FILE = os.path.expanduser(os.environ.get('LSI_KNOWN_HOSTS',
                                                      '~/.lsi-known-hosts'))

# Maps bash commands to their fully qualified paths, e.g. 'ls' -> '/bin/ls'
_PATHS = {}

class LsiProfile(object):
    """Describes the arguments to be found in an LSI profile."""
    def __init__(self, username=None, identity_file=None, filters=None,
                 exclude=None, command=None, no_prompt=False):
        self.username = username
        self.identity_file = identity_file
        self.filters = filters or []
        self.exclude = exclude or []
        self.command = command
        self.no_prompt = no_prompt

    def override(self, attrib, value):
        """Overrides an attribute, unless the overriding value is None."""
        if value is not None:
            setattr(self, attrib, value)

    class LoadError(ValueError):
        """Thrown when a profile fails to load."""
        pass

    @classmethod
    def load(cls, profile_name=None):
        """Loads the user's LSI profile, or provides a default."""
        lsi_location = os.path.expanduser('~/.lsi')
        if not os.path.exists(lsi_location):
            return LsiProfile()
        cfg_parser = ConfigParser()
        cfg_parser.read(lsi_location)
        if profile_name is None:
            # Load the default profile if one exists; otherwise return empty.
            if cfg_parser.has_section('default'):
                profile_name = 'default'
            else:
                return cls()
        elif not cfg_parser.has_section(profile_name):
            raise cls.LoadError('No such profile {}'.format(profile_name))
        def _get(option, alt=None):
            """Gets an option if it exists; else returns `alt`."""
            if cfg_parser.has_option(profile_name, option):
                return cfg_parser.get(profile_name, option)
            else:
                return alt
        if cfg_parser.has_option(profile_name, 'inherit'):
            profile = cls.load(cfg_parser.get(profile_name, 'inherit'))
        else:
            profile = cls()
        profile.override('username', _get('username'))
        profile.override('identity_file', _get('identity file'))
        profile.override('command', _get('command'))
        filters = [s for s in _get('filters', '').split(',') if len(s) > 0]
        exclude = [s for s in _get('exclude', '').split(',') if len(s) > 0]
        profile.filters.extend(filters)
        profile.exclude.extend(exclude)
        return profile

    @staticmethod
    def from_args(args):
        """Takes arguments parsed from argparse and returns a profile."""
        # If the args specify a username explicitly, don't load from file.
        if args.username is not None or args.identity_file is not None:
            profile = LsiProfile()
        else:
            profile = LsiProfile.load(args.profile)
        profile.override('username', args.username)
        profile.override('identity_file', args.identity_file)
        profile.override('command', args.command)
        profile.no_prompt = args.no_prompt
        profile.filters.extend(args.filters)
        profile.exclude.extend(args.exclude)
        if profile.identity_file is not None:
            profile.identity_file = os.path.expanduser(profile.identity_file)
        return profile

def _run_ssh(entries, username, idfile, no_prompt=False, command=None,
             show=None, only=None, sort_by=None, limit=None, tunnel=None,
             num=None, random=False):
    """
    Lets the user choose which instance to SSH into.

    :param entries: The list of host entries.
    :type entries: [:py:class:`HostEntry`]
    :param username: The SSH username to use. Defaults to current user.
    :type username: ``str`` or ``NoneType``
    :param idfile: The identity file to use. Optional.
    :type idfile: ``str`` or ``NoneType``
    :param no_prompt: Whether to disable confirmation for SSH command.
    :type no_prompt: ``bool``
    :param command: SSH command to run on matching instances.
    :type command: ``str`` or ``NoneType``
    :param show: Instance attributes to show in addition to defaults.
    :type show: ``NoneType`` or ``list`` of ``str``
    :param only: If not ``None``, will *only* show these attributes.
    :type only: ``NoneType`` or ``list`` of ``str``
    :param sort_by: What to sort columns by. By default, sort by 'name'.
    :type sort_by: ``str``
    :param limit: At most how many results to show.
    :type limit: ``int`` or ``NoneType``
    :param num: If not None, choose the given entry from within filters.
    :type num: ``int`` or ``NoneType``
    :param random: If true, choose a random entry from within filters.
    :type random: ``bool``
    """
    _print_entries = num is None
    _print_help = False
    if len(entries) == 0:
        exit('No entries matched the filters.')
    if no_prompt is True and command is not None:
        return _run_ssh_command(entries, username, idfile, command, tunnel)
    elif len(entries) == 1 or random is True:
        entry = py_random.choice(entries)
        if command is None:
            return _connect_ssh(entry, username, idfile, tunnel)
        else:
            return _run_ssh_command([entry], username, idfile, command, tunnel)
    elif command is not None:
        print(HostEntry.render_entries(entries,
                                       additional_columns=show,
                                       only_show=only, numbers=True))
        if no_prompt is False:
            get_input("Press enter to run command {} on the {} "
                      "above machines (Ctrl-C to cancel)"
                      .format(cyan(command), len(entries)))
        return _run_ssh_command(entries, username, idfile, command, tunnel)
    else:
        while True:
            if sort_by is not None:
                entries = HostEntry.sort_by(entries, sort_by)
            if limit is not None:
                entries = entries[:limit]
            if _print_entries is True:
                print(HostEntry.render_entries(entries,
                                               additional_columns=show,
                                               only_show=only, numbers=True))
                print('%s matching entries.' % len(entries))
                _print_entries = False
            if _print_help is True:
                cmd_str = green(command) if command is not None else 'none set'
                msg = COMMANDS_STRING.format(username=username or 'none set',
                                             idfile=idfile or 'none set',
                                             cur_cmd=cmd_str)
                print(msg)
                _print_help = False
            elif command is not None:
                print('Set to run ssh command: %s' % cyan(command))
            msg = 'Enter command (%s for help, %s to quit): ' % (cyan('h'),
                                                                 cyan('q'))
            if num is not None:
                choice = num
            else:
                choice = get_input(msg)
            if isinstance(choice, int):
                if 0 <= choice <= len(entries):
                    break
                else:
                    if num is not None:
                        exit("Invalid number {}: must be between 0 and {}"
                             .format(num, len(entries) - 1))
                    else:
                        msg = "Invalid number: must be between 0 and %s"
                        print(msg % (len(entries) - 1))
            elif choice == 'x':
                if command is None:
                    print('No command has been set. Set command with `c`')
                else:
                    return _run_ssh_command(entries, username, idfile,
                                            command, tunnel)
            elif choice == 'h':
                _print_help = True
            elif choice in ['q', 'quit', 'exit']:
                print('bye!')
                return
            else:
                # All of these commands take one or more arguments, so the
                # split length must be at least 2.
                commands = choice.split()
                if len(commands) < 2:
                    print(yellow('Unknown command "%s".' % choice))
                else:
                    cmd = commands[0]
                    if cmd in ['u', 'i', 'p']:
                        if cmd == 'u':
                            username = commands[1]
                        elif cmd == 'i':
                            _idfile = commands[1]
                            if not os.path.exists(_idfile):
                                print(yellow('No such file: %s' % _idfile))
                                continue
                            idfile = _idfile
                        elif cmd == 'p':
                            p = commands[1]
                            try:
                                profile = LsiProfile.load(p)
                                _username = profile.username
                                _idfile = expanduser(profile.identity_file)
                            except LsiProfile.LoadError:
                                print(yellow('No such profile: %s' % repr(p)))
                                continue
                            username = _username
                            idfile = _idfile
                        print('username: %s' % green(repr(username)))
                        print('identity file: %s' % green(repr(idfile)))
                    elif cmd == 'f':
                        entries = filter_entries(entries, commands[1:], [])
                        _print_entries = True
                    elif cmd == 'e':
                        entries = filter_entries(entries, [], commands[1:])
                        _print_entries = True
                    elif cmd == 'c':
                        command = ' '.join(commands[1:])
                    elif cmd == 'limit':
                        try:
                            limit = int(commands[1])
                            _print_entries = True
                        except ValueError:
                            print(yellow('Invalid limit (must be an integer)'))
                    elif cmd == 'sort':
                        sort_by = commands[1]
                        if sort_by not in show:
                            show.append(sort_by)
                        _print_entries = True
                    elif cmd == 'show':
                        if show is None:
                            show = commands[1:]
                        else:
                            show.extend(commands[1:])
                        _print_entries = True
                    else:
                        print(yellow('Unknown command "%s".' % cmd))
        return _connect_ssh(entries[choice], username, idfile, tunnel)

def _get_path(cmd):
    """Queries bash to find the path to a commmand on the system."""
    if cmd in _PATHS:
        return _PATHS[cmd]
    out = subprocess.check_output('which {}'.format(cmd), shell=True)
    _PATHS[cmd] = out.decode("utf-8").strip()
    return _PATHS[cmd]

def _build_ssh_command(hostname, username, idfile, ssh_command, tunnel):
    """Uses hostname and other info to construct an SSH command."""
    command = [_get_path('ssh'),
               '-o', 'StrictHostKeyChecking=no',
               '-o', 'ConnectTimeout=5']
    if idfile is not None:
        command.extend(['-i', idfile])
    if tunnel is not None:
        # If there's a tunnel, run the ssh command on the tunneled host.
        command.extend(['-A', '-t', tunnel, 'ssh', '-A', '-t'])
    if username is not None:
        command.append('{}@{}'.format(username, hostname))
    else:
        command.append(hostname)
    if ssh_command is not None:
        command.append(repr(ssh_command))
    return(' '.join(command))

def _build_scp_command(hostname, username, idfile, is_get,
                       local_path, remote_path):
    """
    Uses hostname and other info to construct an SCP command.

    :param hostname: The hostname of the remote machine.
    :type hostname: ``str``
    :param username: The username to use on the remote machine.
    :type username: ``str``
    :param idfile: A path to the identity file to use.
    :type idfile: ``str``
    :param is_get: If true, we are getting a file rather than putting a file.
    :type is_get: ``bool``
    :param local_path: The path on the local file system.
    :type local_path: ``str``
    :param remote_path: The path on the remote file system.
    :type remote_path: ``str``
    """
    if hostname.strip() == '' or hostname is None:
        raise ValueError('Empty hostname')
    command = [_get_path('scp'),
               '-o', 'StrictHostKeyChecking=no',
               '-o', 'ConnectTimeout=5',
               '-o', 'UserKnownHostsFile={}'.format(_KNOWN_HOSTS_FILE)]
    if idfile is not None:
        command.extend(['-i', idfile])
    if username is not None:
        hostname = '%s@%s' % (username, hostname)
    remote_path = '{}:{}'.format(hostname, remote_path)
    if is_get:
        command.extend([remote_path, local_path])
    else:
        command.extend([local_path, remote_path])
    return ' '.join(command)

def _copy_to(entries, remote_path, local_path, profile):
    """
    Performs an SCP command where the remote_path is the target and the
    local_path is the source.

    :param entries: A list of entries.
    :type entries: ``list`` of :py:class:`HostEntry`
    :param remote_path: The target path on the remote machine(s).
    :type remote_path: ``str``
    :param local_path: The source path on the local machine.
    :type local_path: ``str``
    :param profile: The profile, holding username/idfile info, etc.
    :type profile: :py:class:`Profile`
    """
    commands = []
    for entry in entries:
        hname = entry.hostname or entry.public_ip
        cmd = _build_scp_command(hname, profile.username,
                                 profile.identity_file,
                                 is_get=False,
                                 local_path=local_path,
                                 remote_path=remote_path)
        print('Command:', cmd)
        commands.append({
            'command': cmd,
            'description': entry.display()
        })
    stream_commands(commands)
    print(green('Finished copying'))

def _copy_from(entries, remote_path, local_path, profile):
    """
    Performs an SCP command where the remote_path is the source and the
    local_path is a format string, formatted individually for each host
    being copied from so as to create one or more distinct paths on the
    local system.

    :param entries: A list of entries.
    :type entries: ``list`` of :py:class:`HostEntry`
    :param remote_path: The source path on the remote machine(s).
    :type remote_path: ``str``
    :param local_path: A format string for the path on the local machine.
    :type local_path: ``str``
    :param profile: The profile, holding username/idfile info, etc.
    :type profile: :py:class:`Profile`
    """
    commands = []
    paths = set()
    for entry in entries:
        hname = entry.hostname or entry.public_ip
        _local_path = entry.format_string(local_path)
        if _local_path in paths:
            raise ValueError('Duplicate local paths: one or more paths '
                             'had value {} after formatting.'
                             .format(local_path))
        paths.add(_local_path)
        # If the path references a folder, create the folder if it doesn't
        # exist.
        _folder = os.path.split(_local_path)[0]
        if len(_folder) > 0:
            if not os.path.exists(_folder):
                print('Creating directory ' + _folder)
                os.makedirs(_folder)
        cmd = _build_scp_command(hname, profile.username,
                                 profile.identity_file,
                                 is_get=True,
                                 local_path=_local_path,
                                 remote_path=remote_path)
        print('Command:', cmd)
        commands.append({
            'command': cmd,
            'description': entry.display()
        })
    stream_commands(commands)
    print(green('Finished copying'))

def _run_ssh_command(entries, username, idfile, command, tunnel,
                     parallel=False):
    """
    Runs the given command over SSH in parallel on all hosts in `entries`.

    :param entries: The host entries the hostnames from.
    :type entries: ``list`` of :py:class:`HostEntry`
    :param username: To use a specific username.
    :type username: ``str`` or ``NoneType``
    :param idfile: The SSH identity file to use, or none.
    :type idfile: ``str`` or ``NoneType``
    :param command: The command to run.
    :type command: ``str``
    :param parallel: If true, commands will be run in parallel.
    :type parallel: ``bool``
    """
    if len(entries) == 0:
        print('(No hosts to run command on)')
        return 1
    if command.strip() == '' or command is None:
        raise ValueError('No command given')
    print('Running command {0} on {1} matching hosts'
          .format(green(repr(command)), len(entries)))
    shell_cmds = []
    for entry in entries:
        hname = entry.hostname or entry.public_ip
        cmd = _build_ssh_command(hname, username, idfile, command, tunnel)
        shell_cmds.append({
            'command': cmd,
            'description': entry.display()
        })
    stream_commands(shell_cmds, parallel=parallel)
    print(green('All commands finished'))

def _connect_ssh(entry, username, idfile, tunnel=None):
    """
    SSH into to a host.

    :param entry: The host entry to pull the hostname from.
    :type entry: :py:class:`HostEntry`
    :param username: To use a specific username.
    :type username: ``str`` or ``NoneType``
    :param idfile: The SSH identity file to use, if supplying a username.
    :type idfile: ``str`` or ``NoneType``
    :param tunnel: Host to tunnel SSH command through.
    :type tunnel: ``str`` or ``NoneType``

    :return: An exit status code.
    :rtype: ``int``
    """
    if entry.hostname != "" and entry.hostname is not None:
        _host = entry.hostname
    elif entry.public_ip != "" and entry.public_ip is not None:
        _host = entry.public_ip
    elif entry.private_ip != "" and entry.private_ip is not None:
        if tunnel is None:
            raise ValueError("Entry does not have a hostname or public IP. "
                             "You can connect via private IP if you use a "
                             "tunnel.")
        _host = entry.private_ip
    else:
        raise ValueError("No hostname, public IP or private IP information "
                         "found on host entry. I don't know how to connect.")
    command = _build_ssh_command(_host, username, idfile, None, tunnel)
    print('Connecting to %s...' % cyan(entry.display()))
    print('SSH command: %s' % green(command))
    proc = subprocess.Popen(command, shell=True)
    return proc.wait()

def _print_version():
    """Print the version and exit."""
    from __init__ import __version__
    print(__version__)
    sys.exit(0)

def _get_args():
    """Parse command-line arguments."""
    parser = argparse.ArgumentParser(description='List EC2 instances')
    parser.add_argument('-l', '--latest', action='store_true', default=False,
                        help='Query AWS for latest instances')
    parser.add_argument('--version', action='store_true', default=False,
                        help='Print version and exit')
    parser.add_argument('--refresh-only', action='store_true', default=False,
                        help='Refresh cache and exit')
    parser.add_argument('--host', help='Specific host to list',
                        default=None)
    parser.add_argument('-s', '--ssh', action='store_true',
                        help='SSH to instance', default=False)
    parser.add_argument('-i', '--identity-file', help='SSH identify file',
                        default=None)
    parser.add_argument('-u', '--username', default=None,
                        help='Log in as this user')
    parser.add_argument('filters', nargs='*',
                        help='Text filters for output lines')
    parser.add_argument('-v', '--exclude', nargs='+',
                        help='Exclude results that match these')
    parser.add_argument('-c', '--command', type=str,
                        help='Command to run on matching instance(s)')
    parser.add_argument('-y', '--no-prompt', action='store_true', default=False,
                        help="Don't ask for confirmation before running a "
                             "command")
    parser.add_argument('-p', '--profile', type=str,
                        help='Profile to use (defined in ~/.lsi)')
    parser.add_argument('--show', nargs='+', default=None,
                        help='Instance attributes to show')
    parser.add_argument('--only', nargs='+', default=None,
                        help='Show ONLY these instance attributes')
    parser.add_argument('--sep', type=str, default=None,
                        help='Simple output with given separator')
    parser.add_argument('--sort-by', type=str, default="name",
                        help='What to sort list by')
    parser.add_argument('-L', '--limit', type=int, default=None,
                        help='Show at most this many entries')
    parser.add_argument('--attributes', action='store_true',
                        help='Show all available attributes')
    parser.add_argument('--get', nargs=2, default=None,
                        help='Get files from matching instances, must be '
                             'followed by the source and destination filenames')
    parser.add_argument('--put', nargs=2, default=None,
                        help='Put a local file on matching instances, must be '
                             'followed by the source and destination filenames')
    parser.add_argument('-t', '--tunnel', default=None,
                        help='Connect via the tunneled host.')
    parser.add_argument("-r", "--random", action="store_true", default=False,
                        help="Choose a random instance from within results.")
    parser.add_argument("-n", "--num", type=int, default=None,
                        help="Choose the given number from within results.")
    args = parser.parse_args()
    if args.exclude is None:
        args.exclude = []
    # Presumably, if someone is sorting by something, they want to show that
    # thing...
    if args.sort_by is not None:
        args.show = (args.show or []) + [args.sort_by]
    return args

def main(progname=sys.argv[0]):
    args = _get_args()
    if args.version is True:
        _print_version()
    profile = LsiProfile.from_args(args)
    if args.host is not None:
        return get_host(args.host)
    # Either of these directives should force a refresh.
    latest = args.latest or args.refresh_only
    entries = get_entries(latest, profile.filters, profile.exclude)
    if args.refresh_only is True:
        print('Refreshed cache')
        return
    entries = HostEntry.sort_by(entries, args.sort_by)
    if args.get is not None:
        _copy_from(entries, remote_path=args.get[0],
                   local_path=args.get[1],
                   profile=profile)
    elif args.put is not None:
        if args.num is not None:
            entries = [entries[args.num]]
        _copy_to(entries, local_path=args.put[0],
                 remote_path=args.put[1],
                 profile=profile)
    elif args.ssh is True or \
           args.username is not None or args.identity_file is not None or \
           args.profile is not None or profile.command is not None or \
           args.random is True or args.num is not None:
        _run_ssh(
            entries=entries,
            username=profile.username,
            idfile=profile.identity_file,
            no_prompt=profile.no_prompt,
            command=profile.command,
            show=args.show,
            only=args.only,
            sort_by=args.sort_by,
            limit=args.limit,
            tunnel=args.tunnel,
            random=args.random,
            num=args.num,
        )
    elif args.sep is not None:
        for e in entries:
            print(e.repr_as_line(additional_columns=args.show,
                                 only_show=args.only,
                                 sep=args.sep))
    elif args.attributes is True:
        attribs = HostEntry.list_attributes()
        print('The following attributes are available: {}'
              .format(', '.join(attribs)))
    else:
        if args.limit is not None:
            entries = entries[:args.limit]
        print(HostEntry.render_entries(entries, additional_columns=args.show,
                                       only_show=args.only))
        print('%s matching entries.' % len(entries))

if __name__ == '__main__':
    main()


import os
import json
import collections

yaml = None
xmltodict = None

class dicto(dict):

    def __init__(self, *args, **kwargs):
        super(dicto, self).__init__(*args, **kwargs)

        for key, value in self.items():
            if isinstance(value, dict):
                self[key] = dicto(value)

            elif isinstance(value, str):
                pass

            elif hasattr(value, "__iter__"):
                self[key] = [ dicto(e) if isinstance(e, dict) else e for e in value ]

    def __getattr__(self, attr):
        if attr in self:
            return self[attr]
        else:
            raise AttributeError(attr)

    def __setattr__(self, attr, value):
        self[attr] = value

    def dict_(self):
        d = dict(self)

        for key, value in d.items():
            if isinstance(value, dicto):
                d[key] = value.dict_()

            elif isinstance(value, str):
                pass

            elif isinstance(value, dict):
                pass

            elif hasattr(value, "__iter__"):
                d[key] = [ e.dict_() if isinstance(e, dicto) else e for e in value ]

        return d

    def merge_(self, merge_dct):
        """ Recursive dict merge. Inspired by :meth:``dict.update()``, instead of
        updating only top-level keys, dict_merge recurses down into dicts nested
        to an arbitrary depth, updating keys. The ``merge_dct`` is merged into
        ``self``.
        :param self: dict onto which the merge is executed
        :param merge_dct: self merged into self
        :return: None
        """

        for k, v in merge_dct.items():
            if (k in self and isinstance(self[k], dict) and isinstance(merge_dct[k], collections.Mapping)):
                self[k].merge_(dicto(merge_dct[k]))
            else:
                self[k] = merge_dct[k]

        return self

    def merge(self, *args, **kwargs):
        return self.merge_(*args, **kwargs)

    @classmethod
    def load_(cls, filepath):
        filepath = os.path.realpath(filepath)

        if filepath.endswith(".yaml") or filepath.endswith(".yml"):
            global yaml

            if yaml is None:
                import yaml as mod
                yaml = mod

            with open(filepath, 'r') as stream:
                dict_ = yaml.load(stream)
        elif filepath.endswith(".json"):
            with open(filepath, 'r') as stream:
                dict_ = json.load(stream)
        elif filepath.endswith(".xml"):
            global xmltodict

            if xmltodict is None:
                import xmltodict as mod
                xmltodict = mod

            with open(filepath, 'r') as stream:
                dict_ = xmltodict.parse(stream.read())
        else:
            raise Exception("File type not supported.")

        return cls(dict_)

    def dump_(self, filepath):

        filepath = os.path.realpath(filepath)
        obj = self.dict_()

        if filepath.endswith(".yaml") or filepath.endswith(".yml"):
            global yaml

            if yaml is None:
                import yaml as mod
                yaml = mod

            with open(filepath, 'w') as stream:
                yaml.safe_dump(obj, stream, default_flow_style=False)
        elif filepath.endswith(".json"):
            with open(filepath, 'w') as stream:
                json.dump(obj, stream)
        else:
            raise Exception("File type not supported.")


"""An Open Packaging Conventions implementation.
"""

import os
import posixpath
import datetime
import logging
import collections
import codecs
from collections import defaultdict
from jaraco.collections import FoldedCaseKeyedDict

import six
from lxml.etree import Element, fromstring, tostring
from lxml.builder import ElementMaker as _ElementMaker

from .util import validator, parse_tag, get_ext

log = logging.getLogger(__name__)

# even though Element looks like a class name, it's actually a function. To
#   get the actual class, construct an instance and grab its class.
ElementClass = Element('__').__class__

ooxml_namespaces = dict(
	cp='http://schemas.openxmlformats.org/package/2006/metadata/'
  		'core-properties',
	dc='http://purl.org/dc/elements/1.1/',
	dcterms='http://purl.org/dc/terms/',
	dcmitype='http://purl.org/dc/dcmitype/',
	xsi='http://www.w3.org/2001/XMLSchema-instance',
)

class Relational(object):
	"A mixin class for packages and parts; both support relationships."

	def relate(self, part, id=None):
		"""Relate this package component to the supplied part."""
		assert part.name.startswith(self.base)
		name = part.name[len(self.base):].lstrip('/')
		rel = Relationship(self, name, part.rel_type, id=id)
		self.relationships.add(rel)
		return rel

	def related(self, reltype):
		"""Return a list of parts related to this one via reltype."""
		parts = []
		package = getattr(self, 'package', None) or self
		for rel in self.relationships.types.get(reltype, []):
			parts.append(package[posixpath.join(self.base, rel.target)])
		return parts

	def _load_rels(self, source):
		"""Load relationships from source XML."""
		# don't get confused here - the original source is string data;
		#  the parameter source below is a Part object
		self.relationships.load(source=self, data=source)

class Package(collections.MutableMapping, Relational):
	"""A base class for an OPC package.

	Handles processing provided XML into the core components of a Package:
	 * Relationships
	 * Content-Types
	 * Parts

	Subclasses should overload open(...) and save(...) to read and write
	data to and from various physical package formats.

	Instances of this class support dict-style access to parts via their
	part-names.
	"""

	def __init__(self):
		self.parts = {}
		self.base = '/'
		self.relationships = rels = Relationships(self, self)
		self[rels.name] = rels
		self.content_types = ContentTypes()
		self.content_types.add(ContentType.Default(rels.content_type, 'rels'))

	def __setitem__(self, name, part):
		self._validate_part(name, part)
		self.parts[part.name] = part
		try:
			if part.relationships:
				self.parts[part.relationships.name] = part.relationships
		except ValueError:
			pass

	def __getitem__(self, name):
		"""Returns a part from the given URL."""
		return self.parts[name]

	def __delitem__(self, name):
		del self.parts[name]

	def __iter__(self):
		return iter(self.parts)

	def __len__(self):
		return len(self.parts)

	def keys(self):
		return self.parts.keys()

	def add(self, part, override=True):
		"""Add a part to the package.

		It will also add a content-type - by default an override.  If
		override is False then it will add a content-type for the extension
		if one isn't already present.
		"""
		ct_add_method = [
			self.content_types.add_default,
			self.content_types.add_override,
                ][override]
		self[part.name] = part
		ct_add_method(part)

	@validator
	def _validate_part(self, name, part):
		# 8.1.1.1 -   A package implementer shall neither create nor
		# recognize a part with a part name derived from another part name by
		# appending segments to it
		for cname in self:
			assert not cname.startswith(name), \
                            'The name %s is a derivative of %s' % (name, cname)
		assert name == part.name, "%s != %s" % (name, part.name)
		return part

	def save(self):
		raise NotImplementedError("Subclasses must implement save.")

	def open(self):
		raise NotImplementedError("Subclasses must implement open.")
		# something like this:
		"""
		self._load_content_types(ct_xml)
		self._load_package_rels(rel_xml)
		for rel in self.relationships:
			self._load_part(rel.name, some_file_like_object)
		"""

	def _load_content_types(self, source):
		"""Load up the content_types object with value from source XML."""
		self.content_types.update(ContentTypes.load(source))

	def _load_part(self, rel_type, name, data):
		"""
		Load a part into this package based on its relationship type
		"""
		if self.content_types.find_for(name) is None:
			log.warning('no content type found for part %(name)s' % vars())
			return
		cls = Part.classes_by_rel_type[rel_type]
		part = cls(self, name)
		part.load(data)
		self[name] = part
		return part

	def __repr__(self):
		return "Package-%s" % id(self)

	def get_parts_by_class(self, cls):
		"""
		Return all parts of this package that are instances of cls
		(where cls is passed directly to isinstance, so can be a class
		or sequence of classes).
		"""
		return (part for part in self.parts.values() if isinstance(part, cls))

	def get_parts_by_content_type(self, content_type):
		# first find any parts who's registered type matches or who's
		#  content_type attribute matches
		return (
			part
			for part in self.parts.values()
			if self.content_types.find_for(part.name) == content_type
			or part.content_type == content_type
                )

	@property
	def core_properties(self):
		return next(self.get_parts_by_class(CoreProperties))

class DefaultNamed(object):
	"""
	Mix-in for Parts that have a default name. Subclasses should include
	a 'default_name' attribute, which will be used during construction
	if a name is not explicitly defined.
	"""

	def __init__(self, package, name=None, *args, **kwargs):
		name = name or self.default_name
		super(DefaultNamed, self).__init__(package, name, *args, **kwargs)

class RelationshipTypeHandler(type):
	"""
	A metaclass designed to register new Part classes that handle
	particular relationship types. Whenever a new subclass of Part is
	created, its rel_type attribute will be mapped to that class.

	Subsequently, Part.classes_by_rel_type will be a mapping of
	relationship-type to the appropriate class for that rel-type.
	"""
	def __new__(mcs, name, bases, attrs):
		"""
		This is called when a new class is created of this type
		"""
		# Allow the new class to be created
		cls = type.__new__(mcs, name, bases, attrs)
		# if the class (or its parent) doesn't already have a mapping
		#  of relationship type to class, create one (with this new
		#  class being the default).
		if not hasattr(cls, 'classes_by_rel_type'):
			cls.classes_by_rel_type = defaultdict(lambda: cls)
		rt = attrs.get('rel_type', None)
		if rt:
			cls.classes_by_rel_type[rt] = cls
		return cls

@six.add_metaclass(RelationshipTypeHandler)
class Part(Relational):
	"""
	Parts are the building blocks of OOXML files.

	All Part subclasses need to define their content-type in a
	content_type attribute.  Most will also need a relationship-type
	(defined in the rel_type attribute).  See the documentation for the
	part that you are implementing for the proper values for those attributes.
	"""
	content_type = None
	rel_type = None

	def __init__(self, package, name, **kwargs):
		#map(functools.partial(setattr, self), *kwargs.items())
		for key, value in kwargs.items():
			setattr(self, key, value)
		self.name = name
		self.package = package
		if not isinstance(self, Relationships):
			self.relationships = Relationships(self.package, self)

	@property
	def base(self):
		return posixpath.dirname(self.name)

	@validator
	def _set_name(self, name):
		assert name
		assert name[0] == '/', "%s does not start with a '/'" % name
		assert name[-1] != '/', "%s ends with a '/'" % name
		# TODO: test for empty segments
		# TODO: test for only pchar characters (RFC 3986)
		for segment in name[1:].split('/'):
			# TODO: test for percent encoded slash and unreserved chars
			assert segment[-1] != '.'
			assert segment != '.'
		self._name = name

	def _get_name(self):
		return self._name

	name = property(_get_name, _set_name)

	def __iter__(self):
		"""Should return an iterator for the underlying content."""
		return iter(self.data or [])

	def dump(self):
		"""Return the raw bytes of the Part."""
		data = self.data
		if isinstance(data, ElementClass):
			data = tostring(data, encoding='utf-8', pretty_print=True)
		if isinstance(data, six.text_type):
			return data.encode('utf-8')
		return data

	def load(self, data):
		self.data = data

class Relationship(object):
	"""Represents an OPC relationship between a Package/Part and another Part.

	source : a Package or Part instance
	target : the URL for the target part
	reltype : the type that defines the relationship
	id : a unique identifier for this relationship
	mode : should be one of "Internal" or "External"
	"""

	def __init__(self, source, target, reltype, id=None, mode=None):
		self.source = self._validate_source(source)
		self.target = self._validate_target(target)
		self.id = self._validate_id(id)
		self.mode = self._validate_mode(mode or "Internal")
		self.type = reltype

	def __repr__(self):
		args = (self.source, self.target, self.type, self.id, self.mode)
		return "Relationship(%r, %r, %r, %r, %r)" % args

	@validator
	def _validate_source(self, source):
		assert isinstance(source, (Package, Part))
		return source

	@validator
	def _validate_target(self, target):
		assert isinstance(target, six.string_types), "target must be a part name"
		return target

	@validator
	def _validate_mode(self, mode):
		assert mode in ("Internal", "External")
		return mode

	@validator
	def _validate_id(self, id):
		if id is None:
			return self._generate_id()
		# TODO: The Id type is xsd:ID and it shall conform to the naming
		# restrictions for xsd:ID as specified in the W3C Recommendation
		# "XML Schema Part 2: Datatypes."
		return id

	@staticmethod
	def _generate_id():
		return "d%s" % codecs.encode(os.urandom(4), 'hex').decode()

class Relationships(Part):
	"""A collection of Package or Part Relationships."""
	xmlns = "{http://schemas.openxmlformats.org/package/2006/relationships}"
	content_type = "application/vnd.openxmlformats-package.relationships+xml"
	rel_type = None

	def __init__(self, package, source, encoding=None):
		"""
		@param source package or part where from which the relationship is
		       derived
		@ptype source Package or Part
		"""
		name = self._name_from_source(source)
		Part.__init__(self, package, name)
		self.ids = set()
		self.children = set()
		self.types = {}
		self.encoding = encoding or 'utf-8'

	class relationships(object):
		def __get__(self, instance, owner):
			raise ValueError("Relationship parts have no relationships.")

		def __set__(self, instance, value):
			return
	relationships = relationships()

	def dump(self):
		rels = Element(self.xmlns + 'Relationships',
                 nsmap={None: self.xmlns.strip('{}')})
		for rel in self:
			rels.append(Element(
				'Relationship',
				TargetMode=rel.mode,
				Target=rel.target,
				Type=rel.type,
				Id=rel.id,
			))
		return tostring(rels, encoding=self.encoding)

	def load(self, source, data):
		"""
		@param source The source Part for each relationship in this
		              collection
		@ptype source Part
		@param data Relationship XML from a previous dump operation
		@ptype data string
		"""
		elem = fromstring(data)
		for rel in elem:
			mode = rel.get('TargetMode')
			target = rel.get('Target')
			rtype = rel.get('Type')
			id = rel.get('Id')
			relationship = Relationship(source, target, rtype, id, mode)
			self.add(relationship)

	def __iter__(self):
		return iter(self.children)

	def __repr__(self):
		return "\n".join([repr(c) for c in self.children])

	def add(self, rel):
		self.ids.add(self._validate_id(rel.id))
		self.children.add(rel)
		self.types.setdefault(rel.type, []).append(rel)

	@validator
	def _validate_id(self, id):
		# The value of the Id attribute shall be
		# unique within the Relationships part.
		assert id not in self.ids
		return id

	def _name_from_source(self, source):
		if isinstance(source, Package):
			return "/_rels/.rels"
		base, item = posixpath.split(source.name)
		return posixpath.join(base, '_rels/%s.rels' % item)

class ContentTypes(set):
	"""A container for managing Package content types."""

	xmlns = '{http://schemas.openxmlformats.org/package/2006/content-types}'

	def add_override(self, part):
		ct = ContentType.Override(part.content_type, part.name)
		self.add(ct)
		return ct

	def add_default(self, part):
		ext = get_ext(part.name)
		ct = ContentType.Default(part.content_type, ext)
		self.add(ct)
		return ct

	def dump(self, encoding='utf-8'):
		return tostring(self.to_element(), encoding=encoding)

	@classmethod
	def load(cls, source):
		elem = fromstring(source)
		return cls.from_element(elem)

	def to_element(self):
		elem = Element(self.xmlns + 'Types',
                 nsmap={None: self.xmlns.strip('{}')})
		elem.extend(ct.to_element() for ct in self)
		return elem

	@classmethod
	def from_element(cls, elem):
		ns, tag = parse_tag(elem.tag)
		assert tag == 'Types'
		return cls(map(ContentType.from_element, elem))

	def _item_map(self, class_filter=object):
		return FoldedCaseKeyedDict(
			(item.key, item)
			for item in self
			if isinstance(item, class_filter)
		)
	items = property(_item_map)

	def find_for(self, name):
		"""
		Get the correct content type for a given name
		"""
		map = self.items
		# first search the overrides (by name)
		# then fall back to the defaults (by extension)
		# finally, return None if unmatched
		return map.get(name, None) or map.get(get_ext(name) or None, None)

	# a couple of properties for backward compatibility - please don't
	#  try to write to the resultant collections
	@property
	def defaults(self):
		return self._item_map(ContentType.Default)

	@property
	def overrides(self):
		return self._item_map(ContentType.Override)

class ContentType(object):
	"""
	An abstract content type.
	Each content type has a name, which is a mime-type like
	 application/xml, and a key which refers to the content type.
	"""

	def __init__(self, name, key):
		self.name = name
		self.key = key

	def to_element(self):
		element_name = self.__class__.__name__
		elem = Element(
			element_name,
			ContentType=self.name,
		)
		elem.set(self.key_name, self.key)
		return elem

	@classmethod
	def from_element(cls, element):
		"given an element, parse out the proper ContentType"
		# disambiguate the subclass
		ns, class_name = parse_tag(element.tag)
		class_ = getattr(ContentType, class_name)
		if not class_:
			msg = 'Invalid Types child element: %(class_name)s' % vars()
			raise ValueError(msg)
		# construct the subclass
		key = element.get(class_.key_name)
		name = element.get('ContentType')
		return class_(name, key)

	def __repr__(self):
		params = dict(
			class_name=self.__class__.__name__,
			**self.__dict__
                )
		return "%(class_name)s(%(name)r, %(key)r)" % params

	def __eq__(self, other):
		"""
		This object is equal to another if the other is of the
		same class and the name and key match.
		"""
		attrs = 'name key'.split()
		all_attrs_eq = all(
			getattr(self, attr, None) == getattr(other, attr, None)
			for attr in attrs
                )
		return isinstance(other, self.__class__) and all_attrs_eq

	def __hash__(self):
		return hash((self.name, self.key))

class Default(ContentType):
	"""A Default content type, based on a file extension."""
	key_name = 'Extension'

ContentType.Default = Default
del Default

class Override(ContentType):
	"""An Override content type, based on a part name."""
	key_name = 'PartName'

ContentType.Override = Override
del Override

# Construct E, a convient namespace for making elements in the OOXML
# namespaces.
E = type('E', (object,), dict(
	(key, _ElementMaker(namespace=namespace, nsmap=ooxml_namespaces))
	for key, namespace in ooxml_namespaces.items()
))

class CoreProperties(Part):
	"""
	Core properties on a package, has attributes like 'title', and 'subject'
	"""
	content_type = ("application/"
                 "vnd.openxmlformats-package.core-properties+xml")
	rel_type = ("http://schemas.openxmlformats.org/package/2006/"
             "relationships/metadata/core-properties")
	title = ''
	subject = ''
	creator = ''
	keywords = ''
	description = ''
	last_modified_by = ''
	revision = 1
	created = None
	modified = None
	dt_format = '%Y-%m-%dT%H:%M:%SZ'

	def __init__(self, package, name):
		Part.__init__(self, package, name)

	def load(self, data):
		xml = fromstring(data)

		def DC(tag): return '{%(dc)s}' % ooxml_namespaces + tag

		def CP(tag): return '{%(cp)s}' % ooxml_namespaces + tag

		def DCTERMS(tag): return '{%(dcterms)s}' % ooxml_namespaces + tag

		def identity(x): return x

		def set_attr_if_tag(tag, attr=None, transform=identity):
			if attr is None:
				ns, attr = parse_tag(tag)
			elem = xml.find(tag)
			if elem is not None and elem.text:
				value = transform(elem.text)
				setattr(self, attr, value)
		map(set_attr_if_tag, (
			DC('title'),
			DC('subject'),
			DC('creator'),
			CP('keywords'),
			DC('description'),
                ))
		set_attr_if_tag(CP('revision'), transform=int)
		set_attr_if_tag(CP('lastModifiedBy'), 'last_modified_by')

		def parse_datetime(str):
			try:
				result = datetime.datetime.strptime(str, self.dt_format)
			except ValueError:
				result = str
			return result
		set_attr_if_tag(DCTERMS('created'), transform=parse_datetime)
		set_attr_if_tag(DCTERMS('modified'), transform=parse_datetime)

	def dump(self, encoding='utf-8'):
		return tostring(self.to_element(), encoding=encoding)

	def to_element(self):
		# some datetime handling
		now = datetime.datetime.now()
		if self.created is None:
			self.created = now
		if self.modified is None:
			self.modified = now
		created_str = self.created.strftime(self.dt_format)
		created = E.dcterms.created(created_str)
		created.set('{%(xsi)s}type' % ooxml_namespaces, 'dcterms:W3CDTF')
		modified_str = self.modified.strftime(self.dt_format)
		modified = E.dcterms.modified(modified_str)
		modified.set('{%(xsi)s}type' % ooxml_namespaces, 'dcterms:W3CDTF')
		# create the element
		element = E.cp.coreProperties(
			E.dc.title(self.title),
			E.dc.subject(self.subject),
			E.dc.creator(self.creator),
			E.cp.keywords(self.keywords),
			E.dc.description(self.description),
			E.cp.revision(str(self.revision)),
			E.cp.lastModifiedBy(self.last_modified_by),
			created,
			modified,
		)
		return element

	element = property(to_element)


from arpeggio import PTNodeVisitor
from arpeggio.cleanpeg import ParserPEG, visit_parse_tree

parser = ParserPEG("""
root = (intent_definition / entity_definition / synonym_definition / comment / EOL)+

EOL = r'\\n|\\r\\n'
indent = r'[ ]*'
sentence = r'[^@^~^%^#^\\n^\\r\\n]+'

element_name = r'[^]]+'

entity_alias = "@[" element_name "]"
synonym_alias = "~[" element_name "]"

prop_key = r'[^=^\n^\r\n]*'
prop_value = r'[^,^)^\n^\r\n]*'
prop = prop_key "=" prop_value indent r'[,]?' indent
props = "(" prop+ ")"

entity_data = indent (synonym_alias / sentence) EOL?
entity_definition = "@[" element_name "]" props? EOL
  entity_data*

intent_data = indent (entity_alias / synonym_alias / sentence)+ EOL?
intent_definition = "%[" element_name "]" props? EOL
  intent_data*

synonym_data = indent sentence+ EOL?
synonym_definition = "~[" element_name "]" props? EOL
  synonym_data*

comment = "#" sentence? EOL?

""", 'root', skipws=False)

def parse(input_string, prefix=''):
  """Parses the given DSL string and returns parsed results.

  Args:
    input_string (str): DSL string
    prefix (str): Optional prefix to add to every element name, useful to namespace things

  Returns:
    dict: Parsed content

  """

  tree = parser.parse(input_string)
  visitor = ChatlVisitor(prefix)

  visit_parse_tree(tree, visitor)

  return visitor.parsed

class ChatlVisitor (PTNodeVisitor):
  """Custom visitor to traverse the tree given by Arpeggio and output data that are
  usable for adapters.

  It could be better but it works already well.

  """

  def __init__(self, prefix=''):
    super(ChatlVisitor, self).__init__ ()

    self._cur_element = None
    self._cur_name = None
    self._cur_training_data = []
    self._cur_raw_data = []
    self._cur_prop_key = None
    self._cur_prop_value = None
    self._cur_props = {}

    self.prefix = prefix
    self.parsed = {
      'intents': {},
      'entities': {},
      'synonyms': {},
    }

  def _append_and_reset(self, key, flatten_data=False):
    splitted = self._cur_element.split('#')
    data = [d[0] for d in self._cur_training_data] if flatten_data else self._cur_training_data
    name = self._cur_element
    variant = None

    if len(splitted) > 1:
      name, variant = splitted

    if name not in self.parsed[key]:
      initial_data = { 'props': {}, 'data': [] }

      # Add the variants key for entities
      if key == 'entities':
        initial_data['variants'] = {}

      self.parsed[key][name] = initial_data

    self.parsed[key][name]['props'].update(self._cur_props)

    if variant:
      self.parsed[key][name]['variants'][variant] = data
    else:
      self.parsed[key][name]['data'].extend(data)

    self._reset()

  def _reset(self):
    self._cur_training_data = []
    self._cur_raw_data = []
    self._cur_element = None
    self._cur_props = {}

  def _append_cur_data(self):
    self._cur_training_data.append (self._cur_raw_data)
    self._cur_raw_data = []

  def visit_prop_key(self, node, children):
    self._cur_prop_key = node.value

  def visit_prop_value(self, node, children):
    self._cur_prop_value = node.value
    self._cur_props[self._cur_prop_key] = self._cur_prop_value

  def visit_element_name(self, node, children):
    if self._cur_element:
      self._cur_name = self.prefix + node.value
    else:
      self._reset()
      self._cur_element = self.prefix + node.value

  def visit_sentence(self, node, children):
    self._cur_raw_data.append({
      'type': 'text',
      'value': node.value,
    })

  def visit_intent_definition(self, node, children):
    self._append_and_reset('intents')

  def visit_entity_definition(self, node, children):
    self._append_and_reset('entities', True)

  def visit_synonym_definition(self, node, children):
    self._append_and_reset('synonyms', True)

  def visit_entity_alias(self, node, children):
    splitted = self._cur_name.split('#')
    value = self._cur_name
    variant = None

    if len (splitted) > 1:
      value, variant = splitted

    self._cur_raw_data.append({
      'type': 'entity',
      'value': value,
      'variant': variant,
    })

  def visit_synonym_alias(self, node, children):
    self._cur_raw_data.append({
      'type': 'synonym',
      'value': self._cur_name,
    })

  def visit_intent_data(self, node, children):
    self._append_cur_data()

  def visit_entity_data(self, node, children):
    self._append_cur_data()

  def visit_synonym_data(self, node, children):
    self._append_cur_data()


import calendar
import time
import uuid

from jwcrypto import jwe, jwk
from jwcrypto.common import base64url_encode, json_encode

VIEW_IDENTIFIERS_CLAIM_NAME = 'http://reflect.io/s/v3/vid'
PARAMETERS_CLAIM_NAME = 'http://reflect.io/s/v3/p'
ATTRIBUTES_CLAIM_NAME = 'http://reflect.io/s/v3/a'

class ProjectTokenBuilder(object):
    """Builder for encrypted tokens.

    Provides a way to create secure, reusable tokens that enable particular
    functionality in the Reflect API while disallowing tampering.
    """

    def __init__(self, access_key):
        """Constructs a new token builder with the given access key.

        :param access_key: The access key to use.
        """
        self._access_key = access_key
        self._expiration = None

        self._view_identifiers = []
        self._parameters = []
        self._attributes = {}

    def set_expiration(self, expiration):
        """Sets the expiration for the constructed token to the given time.

        After this time, the token will no longer be valid. All requests made
        using an expired token will fail.

        :param expiration(datetime.datetime): The time object to use for expiration. Should be aware or in UTC.
        """
        self._expiration = expiration
        return self

    def add_view_identifier(self, identifier):
        """Adds the given view identifier to the list of view identifiers
        permitted by this token.

        If no view identifiers are added to this builder, all views in the
        given access key's project will be able to be loaded. Otherwise, only
        those added will be able to be loaded.

        :param identifier(string): The view identifier to restrict to.
        """
        self._view_identifiers.append(identifier)
        return self

    def add_parameter(self, parameter):
        """Adds a data-filtering parameter to this token.

        :param parameter(Parameter): The parameter to add.
        """
        self._parameters.append(parameter)
        return self

    def set_attribute(self, name, value):
        """Sets the given attribute in this token.

        :param name(string): The attribute slug.
        :param value: The attribute's value, which must be serializable to JSON.
        """
        self._attributes[name] = value
        return self

    def build(self, secret_key):
        """Builds a final copy of the token using the given secret key.

        :param secret_key(string): The secret key that corresponds to this builder's access key.
        """
        key = jwk.JWK(
            kty='oct',
            k=base64url_encode(uuid.UUID(secret_key).bytes),
        )

        header = {
            'alg': 'dir',
            'enc': 'A128GCM',
            'zip': 'DEF',
            'cty': 'JWT',
            'kid': self._access_key,
        }

        now = int(time.time())

        payload = {
            'iat': now,
            'nbf': now,
        }

        if self._expiration is not None:
            payload['exp'] = int(calendar.timegm(self._expiration.utctimetuple()))

        if len(self._view_identifiers) > 0:
            payload[VIEW_IDENTIFIERS_CLAIM_NAME] = self._view_identifiers

        if len(self._parameters) > 0:
            parameters = []
            for parameter in self._parameters:
                serialized = {
                    'field': parameter.field,
                    'op': parameter.op,
                }

                if hasattr(parameter, '__iter__'):
                    serialized['any'] = list(parameter.value)
                else:
                    serialized['value'] = parameter.value

                parameters.append(serialized)

            payload[PARAMETERS_CLAIM_NAME] = parameters

        if len(self._attributes) > 0:
            payload[ATTRIBUTES_CLAIM_NAME] = self._attributes

        tok = jwe.JWE(json_encode(payload), protected=header)
        tok.add_recipient(key)

        return tok.serialize(compact=True)