# -*- coding: utf-8 -*-
"""Tools for inspecting Python objects.

Uses syntax highlighting for presenting the various information elements.

Similar in spirit to the inspect module, but all calls take a name argument to
reference the name under which an object is being read.
"""

#*****************************************************************************
#       Copyright (C) 2001-2004 Fernando Perez <fperez@colorado.edu>
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#*****************************************************************************

__all__ = ['Inspector','InspectColors']

# stdlib modules
import __builtin__
import inspect
import linecache
import os
import sys
import types
from collections import namedtuple
try:
    from itertools import izip_longest
except ImportError:
    from itertools import zip_longest as izip_longest

# IPython's own
from IPython.core import page
from IPython.testing.skipdoctest import skip_doctest_py3
from IPython.utils import PyColorize
from IPython.utils import io
from IPython.utils import py3compat
from IPython.utils.text import indent
from IPython.utils.wildcard import list_namespace
from IPython.utils.coloransi import *

#****************************************************************************
# Builtin color schemes

Colors = TermColors  # just a shorthand

# Build a few color schemes
NoColor = ColorScheme(
    'NoColor',{
    'header' : Colors.NoColor,
    'normal' : Colors.NoColor  # color off (usu. Colors.Normal)
    }  )

LinuxColors = ColorScheme(
    'Linux',{
    'header' : Colors.LightRed,
    'normal' : Colors.Normal  # color off (usu. Colors.Normal)
    } )

LightBGColors = ColorScheme(
    'LightBG',{
    'header' : Colors.Red,
    'normal' : Colors.Normal  # color off (usu. Colors.Normal)
    }  )

# Build table of color schemes (needed by the parser)
InspectColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors],
                                 'Linux')

#****************************************************************************
# Auxiliary functions and objects

# See the messaging spec for the definition of all these fields.  This list
# effectively defines the order of display
info_fields = ['type_name', 'base_class', 'string_form', 'namespace',
               'length', 'file', 'definition', 'docstring', 'source',
               'init_definition', 'class_docstring', 'init_docstring',
               'call_def', 'call_docstring',
               # These won't be printed but will be used to determine how to
               # format the object
               'ismagic', 'isalias', 'isclass', 'argspec', 'found', 'name'
               ]

def object_info(**kw):
    """Make an object info dict with all fields present."""
    infodict = dict(izip_longest(info_fields, [None]))
    infodict.update(kw)
    return infodict

def getdoc(obj):
    """Stable wrapper around inspect.getdoc.

    This can't crash because of attribute problems.

    It also attempts to call a getdoc() method on the given object.  This
    allows objects which provide their docstrings via non-standard mechanisms
    (like Pyro proxies) to still be inspected by ipython's ? system."""
    # Allow objects to offer customized documentation via a getdoc method:
    try:
        ds = obj.getdoc()
    except Exception:
        pass
    else:
        # if we get extra info, we add it to the normal docstring.
        if isinstance(ds, basestring):
            return inspect.cleandoc(ds)

    try:
        return inspect.getdoc(obj)
    except Exception:
        # Harden against an inspect failure, which can occur with
        # SWIG-wrapped extensions.
        return None

def getsource(obj,is_binary=False):
    """Wrapper around inspect.getsource.

    This can be modified by other projects to provide customized source
    extraction.

    Inputs:

    - obj: an object whose source code we will attempt to extract.

    Optional inputs:

    - is_binary: whether the object is known to come from a binary source.
    This implementation will skip returning any output for binary objects, but
    custom extractors may know how to meaningfully process them."""

    if is_binary:
        return None
    else:
        # get source if obj was decorated with @decorator
        if hasattr(obj,"__wrapped__"):
            obj = obj.__wrapped__
        try:
            src = inspect.getsource(obj)
        except TypeError:
            if hasattr(obj,'__class__'):
                src = inspect.getsource(obj.__class__)
        return src

def getargspec(obj):
    """Get the names and default values of a function's arguments.

    A tuple of four things is returned: (args, varargs, varkw, defaults).
    'args' is a list of the argument names (it may contain nested lists).
    'varargs' and 'varkw' are the names of the * and ** arguments or None.
    'defaults' is an n-tuple of the default values of the last n arguments.

    Modified version of inspect.getargspec from the Python Standard
    Library."""

    if inspect.isfunction(obj):
        func_obj = obj
    elif inspect.ismethod(obj):
        func_obj = obj.im_func
    elif hasattr(obj, '__call__'):
        func_obj = obj.__call__
    else:
        raise TypeError('arg is not a Python function')
    args, varargs, varkw = inspect.getargs(func_obj.func_code)
    return args, varargs, varkw, func_obj.func_defaults

def format_argspec(argspec):
    """Format argspect, convenience wrapper around inspect's.

    This takes a dict instead of ordered arguments and calls
    inspect.format_argspec with the arguments in the necessary order.
    """
    return inspect.formatargspec(argspec['args'], argspec['varargs'],
                                 argspec['varkw'], argspec['defaults'])

def call_tip(oinfo, format_call=True):
    """Extract call tip data from an oinfo dict.

    Parameters
    ----------
    oinfo : dict

    format_call : bool, optional
      If True, the call line is formatted and returned as a string.  If not, a
      tuple of (name, argspec) is returned.

    Returns
    -------
    call_info : None, str or (str, dict) tuple.
      When format_call is True, the whole call information is formattted as a
      single string.  Otherwise, the object's name and its argspec dict are
      returned.  If no call information is available, None is returned.

    docstring : str or None
      The most relevant docstring for calling purposes is returned, if
      available.  The priority is: call docstring for callable instances, then
      constructor docstring for classes, then main object's docstring otherwise
      (regular functions).
    """
    # Get call definition
    argspec = oinfo.get('argspec')
    if argspec is None:
        call_line = None
    else:
        # Callable objects will have 'self' as their first argument, prune
        # it out if it's there for clarity (since users do *not* pass an
        # extra first argument explicitly).
        try:
            has_self = argspec['args'][0] == 'self'
        except (KeyError, IndexError):
            pass
        else:
            if has_self:
                argspec['args'] = argspec['args'][1:]

        call_line = oinfo['name']+format_argspec(argspec)

    # Now get docstring.
    # The priority is: call docstring, constructor docstring, main one.
    doc = oinfo.get('call_docstring')
    if doc is None:
        doc = oinfo.get('init_docstring')
    if doc is None:
        doc = oinfo.get('docstring','')

    return call_line, doc

def find_file(obj):
    """Find the absolute path to the file where an object was defined.

    This is essentially a robust wrapper around `inspect.getabsfile`.

    Returns None if no file can be found.

    Parameters
    ----------
    obj : any Python object

    Returns
    -------
    fname : str
      The absolute path to the file where the object was defined.
    """
    # get source if obj was decorated with @decorator
    if hasattr(obj, '__wrapped__'):
        obj = obj.__wrapped__

    fname = None
    try:
        fname = inspect.getabsfile(obj)
    except TypeError:
        # For an instance, the file that matters is where its class was
        # declared.
        if hasattr(obj, '__class__'):
            try:
                fname = inspect.getabsfile(obj.__class__)
            except TypeError:
                # Can happen for builtins
                pass
    except:
        pass
    return fname

def find_source_lines(obj):
    """Find the line number in a file where an object was defined.

    This is essentially a robust wrapper around `inspect.getsourcelines`.

    Returns None if no file can be found.

    Parameters
    ----------
    obj : any Python object

    Returns
    -------
    lineno : int
      The line number where the object definition starts.
    """
    # get source if obj was decorated with @decorator
    if hasattr(obj, '__wrapped__'):
        obj = obj.__wrapped__

    try:
        try:
            lineno = inspect.getsourcelines(obj)[1]
        except TypeError:
            # For instances, try the class object like getsource() does
            if hasattr(obj, '__class__'):
                lineno = inspect.getsourcelines(obj.__class__)[1]
    except:
        return None

    return lineno

class Inspector:
    def __init__(self, color_table=InspectColors,
                 code_color_table=PyColorize.ANSICodeColors,
                 scheme='NoColor',
                 str_detail_level=0):
        self.color_table = color_table
        self.parser = PyColorize.Parser(code_color_table,out='str')
        self.format = self.parser.format
        self.str_detail_level = str_detail_level
        self.set_active_scheme(scheme)

    def _getdef(self,obj,oname=''):
        """Return the definition header for any callable object.

        If any exception is generated, None is returned instead and the
        exception is suppressed."""

        try:
            # We need a plain string here, NOT unicode!
            hdef = oname + inspect.formatargspec(*getargspec(obj))
            return py3compat.unicode_to_str(hdef, 'ascii')
        except:
            return None

    def __head(self,h):
        """Return a header string with proper colors."""
        return '%s%s%s' % (self.color_table.active_colors.header,h,
                           self.color_table.active_colors.normal)

    def set_active_scheme(self, scheme):
        self.color_table.set_active_scheme(scheme)
        self.parser.color_table.set_active_scheme(scheme)

    def noinfo(self, msg, oname):
        """Generic message when no information is found."""
        print 'No %s found' % msg,
        if oname:
            print 'for %s' % oname
        else:
            print

    def pdef(self, obj, oname=''):
        """Print the definition header for any callable object.

        If the object is a class, print the constructor information."""

        if not callable(obj):
            print 'Object is not callable.'
            return

        header = ''

        if inspect.isclass(obj):
            header = self.__head('Class constructor information:\n')
            obj = obj.__init__
        elif (not py3compat.PY3) and type(obj) is types.InstanceType:
            obj = obj.__call__

        output = self._getdef(obj,oname)
        if output is None:
            self.noinfo('definition header',oname)
        else:
            print >>io.stdout, header,self.format(output),

    # In Python 3, all classes are new-style, so they all have __init__.
    @skip_doctest_py3
    def pdoc(self,obj,oname='',formatter = None):
        """Print the docstring for any object.

        Optional:
        -formatter: a function to run the docstring through for specially
        formatted docstrings.

        Examples
        --------

        In [1]: class NoInit:
           ...:     pass

        In [2]: class NoDoc:
           ...:     def __init__(self):
           ...:         pass

        In [3]: %pdoc NoDoc
        No documentation found for NoDoc

        In [4]: %pdoc NoInit
        No documentation found for NoInit

        In [5]: obj = NoInit()

        In [6]: %pdoc obj
        No documentation found for obj

        In [5]: obj2 = NoDoc()

        In [6]: %pdoc obj2
        No documentation found for obj2
        """

        head = self.__head  # For convenience
        lines = []
        ds = getdoc(obj)
        if formatter:
            ds = formatter(ds)
        if ds:
            lines.append(head("Class Docstring:"))
            lines.append(indent(ds))
        if inspect.isclass(obj) and hasattr(obj, '__init__'):
            init_ds = getdoc(obj.__init__)
            if init_ds is not None:
                lines.append(head("Constructor Docstring:"))
                lines.append(indent(init_ds))
        elif hasattr(obj,'__call__'):
            call_ds = getdoc(obj.__call__)
            if call_ds:
                lines.append(head("Calling Docstring:"))
                lines.append(indent(call_ds))

        if not lines:
            self.noinfo('documentation',oname)
        else:
            page.page('\n'.join(lines))

    def psource(self,obj,oname=''):
        """Print the source code for an object."""

        # Flush the source cache because inspect can return out-of-date source
        linecache.checkcache()
        try:
            src = getsource(obj)
        except:
            self.noinfo('source',oname)
        else:
            page.page(self.format(py3compat.unicode_to_str(src)))

    def pfile(self, obj, oname=''):
        """Show the whole file where an object was defined."""

        lineno = find_source_lines(obj)
        if lineno is None:
            self.noinfo('file', oname)
            return

        ofile = find_file(obj)
        # run contents of file through pager starting at line where the object
        # is defined, as long as the file isn't binary and is actually on the
        # filesystem.
        if ofile.endswith(('.so', '.dll', '.pyd')):
            print 'File %r is binary, not printing.' % ofile
        elif not os.path.isfile(ofile):
            print 'File %r does not exist, not printing.' % ofile
        else:
            # Print only text files, not extension binaries.  Note that
            # getsourcelines returns lineno with 1-offset and page() uses
            # 0-offset, so we must adjust.
            page.page(self.format(open(ofile).read()), lineno-1)

    def _format_fields(self, fields, title_width=12):
        """Formats a list of fields for display.

        Parameters
        ----------
        fields : list
          A list of 2-tuples: (field_title, field_content)
        title_width : int
          How many characters to pad titles to. Default 12.
        """
        out = []
        header = self.__head
        for title, content in fields:
            if len(content.splitlines()) > 1:
                title = header(title + ":") + "\n"
            else:
                title = header((title+":").ljust(title_width))
            out.append(title + content)
        return "\n".join(out)

    # The fields to be displayed by pinfo: (fancy_name, key_in_info_dict)
    pinfo_fields1 = [("Type", "type_name"),
                    ]

    pinfo_fields2 = [("String Form", "string_form"),
                    ]

    pinfo_fields3 = [("Length", "length"),
                    ("File", "file"),
                    ("Definition", "definition"),
                    ]

    pinfo_fields_obj = [("Class Docstring", "class_docstring"),
                        ("Constructor Docstring","init_docstring"),
                        ("Call def", "call_def"),
                        ("Call docstring", "call_docstring")]

    def pinfo(self,obj,oname='',formatter=None,info=None,detail_level=0):
        """Show detailed information about an object.

        Optional arguments:

        - oname: name of the variable pointing to the object.

        - formatter: special formatter for docstrings (see pdoc)

        - info: a structure with some information fields which may have been
        precomputed already.

        - detail_level: if set to 1, more information is given.
        """
        info = self.info(obj, oname=oname, formatter=formatter,
                            info=info, detail_level=detail_level)
        displayfields = []
        def add_fields(fields):
            for title, key in fields:
                field = info[key]
                if field is not None:
                    displayfields.append((title, field.rstrip()))

        add_fields(self.pinfo_fields1)

        # Base class for old-style instances
        if (not py3compat.PY3) and isinstance(obj, types.InstanceType) and info['base_class']:
            displayfields.append(("Base Class", info['base_class'].rstrip()))

        add_fields(self.pinfo_fields2)

        # Namespace
        if info['namespace'] != 'Interactive':
            displayfields.append(("Namespace", info['namespace'].rstrip()))

        add_fields(self.pinfo_fields3)

        # Source or docstring, depending on detail level and whether
        # source found.
        if detail_level > 0 and info['source'] is not None:
            displayfields.append(("Source", self.format(py3compat.cast_bytes_py2(info['source']))))
        elif info['docstring'] is not None:
            displayfields.append(("Docstring", info["docstring"]))

        # Constructor info for classes
        if info['isclass']:
            if info['init_definition'] or info['init_docstring']:
                displayfields.append(("Constructor information", ""))
                if info['init_definition'] is not None:
                    displayfields.append((" Definition",
                                    info['init_definition'].rstrip()))
                if info['init_docstring'] is not None:
                    displayfields.append((" Docstring",
                                        indent(info['init_docstring'])))

        # Info for objects:
        else:
            add_fields(self.pinfo_fields_obj)

        # Finally send to printer/pager:
        if displayfields:
            page.page(self._format_fields(displayfields))

    def info(self, obj, oname='', formatter=None, info=None, detail_level=0):
        """Compute a dict with detailed information about an object.

        Optional arguments:

        - oname: name of the variable pointing to the object.

        - formatter: special formatter for docstrings (see pdoc)

        - info: a structure with some information fields which may have been
        precomputed already.

        - detail_level: if set to 1, more information is given.
        """

        obj_type = type(obj)

        header = self.__head
        if info is None:
            ismagic = 0
            isalias = 0
            ospace = ''
        else:
            ismagic = info.ismagic
            isalias = info.isalias
            ospace = info.namespace

        # Get docstring, special-casing aliases:
        if isalias:
            if not callable(obj):
                try:
                    ds = "Alias to the system command:\n  %s" % obj[1]
                except:
                    ds = "Alias: " + str(obj)
            else:
                ds = "Alias to " + str(obj)
                if obj.__doc__:
                    ds += "\nDocstring:\n" + obj.__doc__
        else:
            ds = getdoc(obj)
            if ds is None:
                ds = '<no docstring>'
        if formatter is not None:
            ds = formatter(ds)

        # store output in a dict, we initialize it here and fill it as we go
        out = dict(name=oname, found=True, isalias=isalias, ismagic=ismagic)

        string_max = 200 # max size of strings to show (snipped if longer)
        shalf = int((string_max -5)/2)

        if ismagic:
            obj_type_name = 'Magic function'
        elif isalias:
            obj_type_name = 'System alias'
        else:
            obj_type_name = obj_type.__name__
        out['type_name'] = obj_type_name

        try:
            bclass = obj.__class__
            out['base_class'] = str(bclass)
        except: pass

        # String form, but snip if too long in ? form (full in ??)
        if detail_level >= self.str_detail_level:
            try:
                ostr = str(obj)
                str_head = 'string_form'
                if not detail_level and len(ostr)>string_max:
                    ostr = ostr[:shalf] + ' <...> ' + ostr[-shalf:]
                    ostr = ("\n" + " " * len(str_head.expandtabs())).\
                            join(q.strip() for q in ostr.split("\n"))
                out[str_head] = ostr
            except:
                pass

        if ospace:
            out['namespace'] = ospace

        # Length (for strings and lists)
        try:
            out['length'] = str(len(obj))
        except: pass

        # Filename where object was defined
        binary_file = False
        fname = find_file(obj)
        if fname is None:
            # if anything goes wrong, we don't want to show source, so it's as
            # if the file was binary
            binary_file = True
        else:
            if fname.endswith(('.so', '.dll', '.pyd')):
                binary_file = True
            elif fname.endswith('<string>'):
                fname = 'Dynamically generated function. No source code available.'
            out['file'] = fname

        # reconstruct the function definition and print it:
        defln = self._getdef(obj, oname)
        if defln:
            out['definition'] = self.format(defln)

        # Docstrings only in detail 0 mode, since source contains them (we
        # avoid repetitions).  If source fails, we add them back, see below.
        if ds and detail_level == 0:
                out['docstring'] = ds

        # Original source code for any callable
        if detail_level:
            # Flush the source cache because inspect can return out-of-date
            # source
            linecache.checkcache()
            source = None
            try:
                try:
                    source = getsource(obj, binary_file)
                except TypeError:
                    if hasattr(obj, '__class__'):
                        source = getsource(obj.__class__, binary_file)
                if source is not None:
                    out['source'] = source.rstrip()
            except Exception:
                pass

            if ds and source is None:
                out['docstring'] = ds

        # Constructor docstring for classes
        if inspect.isclass(obj):
            out['isclass'] = True
            # reconstruct the function definition and print it:
            try:
                obj_init =  obj.__init__
            except AttributeError:
                init_def = init_ds = None
            else:
                init_def = self._getdef(obj_init,oname)
                init_ds  = getdoc(obj_init)
                # Skip Python's auto-generated docstrings
                if init_ds and \
                       init_ds.startswith('x.__init__(...) initializes'):
                    init_ds = None

            if init_def or init_ds:
                if init_def:
                    out['init_definition'] = self.format(init_def)
                if init_ds:
                    out['init_docstring'] = init_ds

        # and class docstring for instances:
        else:
            # First, check whether the instance docstring is identical to the
            # class one, and print it separately if they don't coincide.  In
            # most cases they will, but it's nice to print all the info for
            # objects which use instance-customized docstrings.
            if ds:
                try:
                    cls = getattr(obj,'__class__')
                except:
                    class_ds = None
                else:
                    class_ds = getdoc(cls)
                # Skip Python's auto-generated docstrings
                if class_ds and \
                       (class_ds.startswith('function(code, globals[,') or \
                   class_ds.startswith('instancemethod(function, instance,') or \
                   class_ds.startswith('module(name[,') ):
                    class_ds = None
                if class_ds and ds != class_ds:
                    out['class_docstring'] = class_ds

            # Next, try to show constructor docstrings
            try:
                init_ds = getdoc(obj.__init__)
                # Skip Python's auto-generated docstrings
                if init_ds and \
                       init_ds.startswith('x.__init__(...) initializes'):
                    init_ds = None
            except AttributeError:
                init_ds = None
            if init_ds:
                out['init_docstring'] = init_ds

            # Call form docstring for callable instances
            if hasattr(obj, '__call__'):
                call_def = self._getdef(obj.__call__, oname)
                if call_def is not None:
                    out['call_def'] = self.format(call_def)
                call_ds = getdoc(obj.__call__)
                # Skip Python's auto-generated docstrings
                if call_ds and call_ds.startswith('x.__call__(...) <==> x(...)'):
                    call_ds = None
                if call_ds:
                    out['call_docstring'] = call_ds

        # Compute the object's argspec as a callable.  The key is to decide
        # whether to pull it from the object itself, from its __init__ or
        # from its __call__ method.

        if inspect.isclass(obj):
            # Old-style classes need not have an __init__
            callable_obj = getattr(obj, "__init__", None)
        elif callable(obj):
            callable_obj = obj
        else:
            callable_obj = None

        if callable_obj:
            try:
                args,  varargs, varkw, defaults = getargspec(callable_obj)
            except (TypeError, AttributeError):
                # For extensions/builtins we can't retrieve the argspec
                pass
            else:
                out['argspec'] = dict(args=args, varargs=varargs,
                                      varkw=varkw, defaults=defaults)

        return object_info(**out)

    def psearch(self,pattern,ns_table,ns_search=[],
                ignore_case=False,show_all=False):
        """Search namespaces with wildcards for objects.

        Arguments:

        - pattern: string containing shell-like wildcards to use in namespace
        searches and optionally a type specification to narrow the search to
        objects of that type.

        - ns_table: dict of name->namespaces for search.

        Optional arguments:

          - ns_search: list of namespace names to include in search.

          - ignore_case(False): make the search case-insensitive.

          - show_all(False): show all names, including those starting with
          underscores.
        """
        #print 'ps pattern:<%r>' % pattern # dbg

        # defaults
        type_pattern = 'all'
        filter = ''

        cmds = pattern.split()
        len_cmds  =  len(cmds)
        if len_cmds == 1:
            # Only filter pattern given
            filter = cmds[0]
        elif len_cmds == 2:
            # Both filter and type specified
            filter,type_pattern = cmds
        else:
            raise ValueError('invalid argument string for psearch: <%s>' %
                             pattern)

        # filter search namespaces
        for name in ns_search:
            if name not in ns_table:
                raise ValueError('invalid namespace <%s>. Valid names: %s' %
                                 (name,ns_table.keys()))

        #print 'type_pattern:',type_pattern # dbg
        search_result, namespaces_seen = set(), set()
        for ns_name in ns_search:
            ns = ns_table[ns_name]
            # Normally, locals and globals are the same, so we just check one.
            if id(ns) in namespaces_seen:
                continue
            namespaces_seen.add(id(ns))
            tmp_res = list_namespace(ns, type_pattern, filter,
                                    ignore_case=ignore_case, show_all=show_all)
            search_result.update(tmp_res)

        page.page('\n'.join(sorted(search_result)))


"""
Twisted integration
-------------------

This module provides a very simple way to integrate your tests with the
Twisted_ event loop.

You must import this module *before* importing anything from Twisted itself!

Example::

  from nose.twistedtools import reactor, deferred

  @deferred()
  def test_resolve():
      return reactor.resolve("www.python.org")

Or, more realistically::

  @deferred(timeout=5.0)
  def test_resolve():
      d = reactor.resolve("www.python.org")
      def check_ip(ip):
          assert ip == "67.15.36.43"
      d.addCallback(check_ip)
      return d

.. _Twisted: http://twistedmatrix.com/trac/
"""

import sys
from Queue import Queue, Empty
from nose.tools import make_decorator, TimeExpired

__all__ = [
    'threaded_reactor', 'reactor', 'deferred', 'TimeExpired',
    'stop_reactor'
]

_twisted_thread = None

def threaded_reactor():
    """
    Start the Twisted reactor in a separate thread, if not already done.
    Returns the reactor.
    The thread will automatically be destroyed when all the tests are done.
    """
    global _twisted_thread
    try:
        from twisted.internet import reactor
    except ImportError:
        return None, None
    if not _twisted_thread:
        from twisted.python import threadable
        from threading import Thread
        _twisted_thread = Thread(target=lambda: reactor.run( \
                installSignalHandlers=False))
        _twisted_thread.setDaemon(True)
        _twisted_thread.start()
    return reactor, _twisted_thread

# Export global reactor variable, as Twisted does
reactor, reactor_thread = threaded_reactor()

def stop_reactor():
    """Stop the reactor and join the reactor thread until it stops.
    Call this function in teardown at the module or package level to
    reset the twisted system after your tests. You *must* do this if
    you mix tests using these tools and tests using twisted.trial.
    """
    global _twisted_thread

    def stop_reactor():
        '''Helper for calling stop from withing the thread.'''
        reactor.stop()

    reactor.callFromThread(stop_reactor)
    reactor_thread.join()
    for p in reactor.getDelayedCalls():
        if p.active():
            p.cancel()
    _twisted_thread = None

def deferred(timeout=None):
    """
    By wrapping a test function with this decorator, you can return a
    twisted Deferred and the test will wait for the deferred to be triggered.
    The whole test function will run inside the Twisted event loop.

    The optional timeout parameter specifies the maximum duration of the test.
    The difference with timed() is that timed() will still wait for the test
    to end, while deferred() will stop the test when its timeout has expired.
    The latter is more desireable when dealing with network tests, because
    the result may actually never arrive.

    If the callback is triggered, the test has passed.
    If the errback is triggered or the timeout expires, the test has failed.

    Example::

        @deferred(timeout=5.0)
        def test_resolve():
            return reactor.resolve("www.python.org")

    Attention! If you combine this decorator with other decorators (like
    "raises"), deferred() must be called *first*!

    In other words, this is good::

        @raises(DNSLookupError)
        @deferred()
        def test_error():
            return reactor.resolve("xxxjhjhj.biz")

    and this is bad::

        @deferred()
        @raises(DNSLookupError)
        def test_error():
            return reactor.resolve("xxxjhjhj.biz")
    """
    reactor, reactor_thread = threaded_reactor()
    if reactor is None:
        raise ImportError("twisted is not available or could not be imported")
    # Check for common syntax mistake
    # (otherwise, tests can be silently ignored
    # if one writes "@deferred" instead of "@deferred()")
    try:
        timeout is None or timeout + 0
    except TypeError:
        raise TypeError("'timeout' argument must be a number or None")

    def decorate(func):
        def wrapper(*args, **kargs):
            q = Queue()
            def callback(value):
                q.put(None)
            def errback(failure):
                # Retrieve and save full exception info
                try:
                    failure.raiseException()
                except:
                    q.put(sys.exc_info())
            def g():
                try:
                    d = func(*args, **kargs)
                    try:
                        d.addCallbacks(callback, errback)
                    # Check for a common mistake and display a nice error
                    # message
                    except AttributeError:
                        raise TypeError("you must return a twisted Deferred "
                                        "from your test case!")
                # Catch exceptions raised in the test body (from the
                # Twisted thread)
                except:
                    q.put(sys.exc_info())
            reactor.callFromThread(g)
            try:
                error = q.get(timeout=timeout)
            except Empty:
                raise TimeExpired("timeout expired before end of test (%f s.)"
                                  % timeout)
            # Re-raise all exceptions
            if error is not None:
                exc_type, exc_value, tb = error
                raise exc_type, exc_value, tb
        wrapper = make_decorator(func)(wrapper)
        return wrapper
    return decorate


"""
https://stackoverflow.com/questions/36013295/find-best-substring-match
"""
from difflib import SequenceMatcher
from .utils import round_decimal

def find_best_string(query,
                     corpus,
                     step=4,
                     flex=3,
                     case_sensitive=False):
    """Return best matching substring of corpus.

    Parameters
    ----------
    query : str
    corpus : str
    step : int
        Step size of first match-value scan through corpus. Can be thought of
        as a sort of "scan resolution". Should not exceed length of query.
    flex : int
        Max. left/right substring position adjustment value. Should not
        exceed length of query / 2.

    Outputs
    -------
    output0 : str
        Best matching substring.
    output1 : float
        Match ratio of best matching substring. 1 is perfect match.

    """

    def ratio(a, b):
        """Compact alias for SequenceMatcher."""
        return SequenceMatcher(None, a, b).ratio()

    def scan_corpus(step):
        """Return list of match values from corpus-wide scan."""
        match_values = []
        m = 0
        while m + qlen - step <= len(corpus):
            match_values.append(ratio(query, corpus[m : m-1+qlen]))
            m += step
        return match_values

    def index_max(v):
        """Return index of max value."""
        return max(range(len(v)), key=v.__getitem__)

    def adjust_left_right_positions():
        """Return left/right positions for best string match."""
        # bp_* is synonym for 'Best Position Left/Right' and are adjusted
        # to optimize bmv_*
        p_l, bp_l = [pos] * 2
        p_r, bp_r = [pos + qlen] * 2

        # bmv_* are declared here in case they are untouched in optimization
        bmv_l = match_values[round_decimal(p_l / step)]
        bmv_r = match_values[round_decimal(p_r / step)]

        for f in range(flex):
            ll = ratio(query, corpus[p_l - f: p_r])
            if ll > bmv_l:
                bmv_l = ll
                bp_l = p_l - f

            lr = ratio(query, corpus[p_l + f: p_r])
            if lr > bmv_l:
                bmv_l = lr
                bp_l = p_l + f

            rl = ratio(query, corpus[p_l: p_r - f])
            if rl > bmv_r:
                bmv_r = rl
                bp_r = p_r - f

            rr = ratio(query, corpus[p_l: p_r + f])
            if rr > bmv_r:
                bmv_r = rr
                bp_r = p_r + f

        return bp_l, bp_r, ratio(query, corpus[bp_l : bp_r])

    if not case_sensitive:
        query = query.lower()
        corpus = corpus.lower()

    qlen = len(query)

    if flex >= qlen/2:
        print("Warning: flex exceeds length of query / 2. Setting to default.")
        flex = 3

    match_values = scan_corpus(step)
    pos = index_max(match_values) * step

    pos_left, pos_right, match_value = adjust_left_right_positions()

    return corpus[pos_left: pos_right].strip(), match_value


"""Code coverage measurement for Python.

Ned Batchelder
http://nedbatchelder.com/code/coverage

"""

from coverage.version import __version__, __url__

from coverage.control import coverage, process_startup
from coverage.data import CoverageData
from coverage.cmdline import main, CoverageScript
from coverage.misc import CoverageException

# Module-level functions.  The original API to this module was based on
# functions defined directly in the module, with a singleton of the coverage()
# class.  That design hampered programmability, so the current api uses
# explicitly-created coverage objects.  But for backward compatibility, here we
# define the top-level functions to create the singleton when they are first
# called.

# Singleton object for use with module-level functions.  The singleton is
# created as needed when one of the module-level functions is called.
_the_coverage = None

def _singleton_method(name):
    """Return a function to the `name` method on a singleton `coverage` object.

    The singleton object is created the first time one of these functions is
    called.

    """
    # Disable pylint msg W0612, because a bunch of variables look unused, but
    # they're accessed via locals().
    # pylint: disable=W0612

    def wrapper(*args, **kwargs):
        """Singleton wrapper around a coverage method."""
        global _the_coverage
        if not _the_coverage:
            _the_coverage = coverage(auto_data=True)
        return getattr(_the_coverage, name)(*args, **kwargs)

    import inspect
    meth = getattr(coverage, name)
    args, varargs, kw, defaults = inspect.getargspec(meth)
    argspec = inspect.formatargspec(args[1:], varargs, kw, defaults)
    docstring = meth.__doc__
    wrapper.__doc__ = ("""\
        A first-use-singleton wrapper around coverage.%(name)s.

        This wrapper is provided for backward compatibility with legacy code.
        New code should use coverage.%(name)s directly.

        %(name)s%(argspec)s:

        %(docstring)s
        """ % locals()
        )

    return wrapper

# Define the module-level functions.
use_cache = _singleton_method('use_cache')
start =     _singleton_method('start')
stop =      _singleton_method('stop')
erase =     _singleton_method('erase')
exclude =   _singleton_method('exclude')
analysis =  _singleton_method('analysis')
analysis2 = _singleton_method('analysis2')
report =    _singleton_method('report')
annotate =  _singleton_method('annotate')

# On Windows, we encode and decode deep enough that something goes wrong and
# the encodings.utf_8 module is loaded and then unloaded, I don't know why.
# Adding a reference here prevents it from being unloaded.  Yuk.
import encodings.utf_8

# Because of the "from coverage.control import fooey" lines at the top of the
# file, there's an entry for coverage.coverage in sys.modules, mapped to None.
# This makes some inspection tools (like pydoc) unable to find the class
# coverage.coverage.  So remove that entry.
import sys
try:
    del sys.modules['coverage.coverage']
except KeyError:
    pass

# COPYRIGHT AND LICENSE
#
# Copyright 2001 Gareth Rees.  All rights reserved.
# Copyright 2004-2013 Ned Batchelder.  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the
#    distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
# DAMAGE.


import cgi
import re
from uuid import UUID

from lxml import etree

try:
    from io import BytesIO  # python 3
except ImportError:
    try:
        from cStringIO import StringIO as BytesIO
    except ImportError:
        from StringIO import StringIO as BytesIO

__all__ = (
    'XMLEncoder',
    )

class XMLEncoder(object):
    """The main constructor method which accepts the value
    of ``data`` to be later converted to XML.

    The document element ``doc_el`` can be changed from the
    default 'document' to any valid XML element name.

    The ``encoding`` defaults to 'UTF-8', but can be changed
    to any value accepted by ``lxml.etree``.

    Setting ``strict_errors`` to ``True`` will cause a ``TypeError``
    to be raised for unsupported types.

    Any other keyword arguments are passed directly to the
    set-up method of ``lxml.etree``.
    """

    _is_uuid = re.compile(
        r'^\{?([0-9a-f]{8}\-[0-9a-f]{4}\-[0-9a-f]{4}'
        r'\-[0-9a-f]{4}\-[0-9a-f]{12})\}?$',
        re.I
    )

    def __init__(self, data,
                 doc_el='document', encoding='UTF-8', strict_errors=False,
                 **params):
        self.data = data
        self.document = etree.Element(doc_el, **params)
        self.encoding = encoding
        self.strict_errors = strict_errors

    def to_string(self, indent=True, declaration=True):
        """Encodes the stored ``data`` to XML and returns a
        ``string``.

        Setting ``indent`` to ``False`` will forego any pretty-printing
        and return a condensed value.

        Setting ``declaration`` to ``False`` will skip inserting the
        XML declaration.
        """
        return etree.tostring(self.to_xml(),
                              encoding=self.encoding,
                              xml_declaration=declaration,
                              pretty_print=indent
                              )

    def to_xml(self):
        """Encodes the stored ``data`` to XML and returns
        an ``lxml.etree`` value.
        """
        if self.data:
            self.document = self._update_document(self.document, self.data)

        return self.document

    def from_string(self, string):
        """Parses a ``string`` value which
        replaces the internal ``data`` value."""
        self.document = etree.parse(BytesIO(string))

    def _update_document(self, node, data):
        if data is None:
            node.text = None

        elif data is True:
            node.set('nodetype', u'boolean')
            node.text = u"true"

        elif data is False:
            node.set('nodetype', u'boolean')
            node.text = u"false"

        elif (isinstance(data, basestring)
              and len(data) in (36, 38)
              and self._is_uuid.match(data)):

            try:
                UUID(data)
            except (ValueError, AttributeError):
                pass
            else:
                node.set('nodetype', u'uuid')
            finally:
                node.text = self._to_unicode(data)

        elif hasattr(data, 'isoformat'):
            try:
                node.text = data.isoformat()
                node.set('nodetype', u'timestamp')
            except TypeError:
                pass

        elif self._is_scalar(data):
            node.text = self._to_unicode(data)

        elif isinstance(data, BytesIO):
            node.text = self._to_unicode(data.getvalue())

        elif hasattr(data, 'iteritems'):
            for name, items in data.iteritems():
                try:
                    if (isinstance(name, basestring)
                        and name
                        and str(name[0]) is '?'):
                        #  processing instruction
                        self._add_processing_instruction(node, items)

                    elif (isinstance(name, basestring)
                          and name
                          and str(name[0]) is '!'):
                        # doctypes not implemented
                        # self._add_doctype(node, items)
                        pass

                    elif (isinstance(name, basestring)
                          and name
                          and not name[0].isalpha()):
                        child = etree.SubElement(node, u'node',
                                                 name=unicode(name))

                    elif isinstance(name, basestring) and name:
                        child = etree.SubElement(node, unicode(name))

                    else:
                        # node name is invalid, use <node name="{name}">
                        child = etree.SubElement(node, u"node",
                                                 name=unicode(name))

                except ValueError:
                    # node name is invalid, use <node name="{name}">
                    child = etree.SubElement(node, u"node", name=unicode(name))

                child = self._update_document(child, items)

        elif isinstance(data, list):
            node.set('nodetype', u'list')
            for item in data:
                self._update_document(
                    etree.SubElement(node, u'i'),
                    item)

        elif isinstance(data, set):
            node.set('nodetype', u'unique-list')
            for item in data:
                self._update_document(
                    etree.SubElement(node, u'i'),
                    item)

        elif isinstance(data, tuple):
            node.set('nodetype', u'fixed-list')
            for item in data:
                self._update_document(
                    etree.SubElement(node, u'i'),
                    item)

        elif hasattr(data, 'send'):
            # generator
            node.set('nodetype', u'generated-list')
            for item in data:
                self._update_document(
                    etree.SubElement(node, u'i'),
                    item)

        elif (isinstance(data, object)
              and hasattr(data, '__slots__')):
            children = ((n, getattr(data, n))
                        for n in data.__slots__
                        if n[0] is not '_' and not hasattr(n, '__call__'))

            sub = etree.SubElement(node,
                                   unicode(data.__class__.__name__),
                                   nodetype="container")

            for item, value in children:
                self._update_document(
                    etree.SubElement(sub, unicode(item)),
                    value)

        elif isinstance(data, object):
            try:
                children = ((n, v)
                            for n, v in data.__dict__.iteritems()
                            if n[0] is not '_' and not hasattr(n, '__call__'))

                sub = etree.SubElement(node,
                                       unicode(data.__class__.__name__),
                                       nodetype="container")

                for item, value in children:
                    self._update_document(
                        etree.SubElement(sub, unicode(item)),
                        value)

            except Exception:
                if self.strict_errors:
                    raise TypeError('%s is not XML serializable' % type(data))

                node.set('nodetype', u'unsupported-type')
                node.text = self._to_unicode(type(data))

        else:
            if self.strict_errors:
                raise TypeError('%s is not XML serializable' % type(data))

            node.set('nodetype', u'unsupported-type')
            node.text = self._to_unicode(type(data))

        return node

    def _is_scalar(self, value):
        return isinstance(value, (basestring, float, int, long))

    def _to_unicode(self, string):
        if not string and not self._is_scalar(string):
            return u''

        return unicode(self.__escape(string))

    def _add_processing_instruction(self, node, data):
        raise NotImplementedError(
            'creating processing instructions '
            'has not been implemented'
        )

    def _add_doctype(self, item):
        raise NotImplementedError(
            'creating doctype declarations '
            'has not been implemented'
        )

    def __dict_to_attrs(self, d):
        return ('%s="%s"' % (name, value) for name, value in d.iteritems())

    def __escape(self, data):
        if data is None:
            return None

        if isinstance(data, unicode):
            return data

        if isinstance(data, str):
            try:
                data = unicode(data, 'latin1')
            except Exception:
                pass

        return data

    def __unicodeToHTMLEntities(self, text):
        """Converts unicode to HTML entities.
        For example '&' becomes '&'."""
        return cgi.escape(text).encode('ascii', 'xmlcharrefreplace')


import importlib
import pkgutil
import types
from collections.abc import Iterable
from inspect import isclass

def load_all_modules_in_packages(package_or_set_of_packages):
    """
    Recursively loads all modules from a package object, or set of package objects

    :param package_or_set_of_packages: package object, or iterable of package objects
    :return: list of all unique modules discovered by the function
    """
    if isinstance(package_or_set_of_packages, types.ModuleType):
        packages = [package_or_set_of_packages]
    elif isinstance(package_or_set_of_packages, Iterable) and not isinstance(package_or_set_of_packages, (dict, str)):
        packages = package_or_set_of_packages
    else:
        raise Exception("This function only accepts a module reference, or an iterable of said objects")

    imported = packages.copy()

    for package in packages:
        if not hasattr(package, '__path__'):
            raise Exception(
                'Package object passed in has no __path__ attribute. '
                'Make sure to pass in imported references to the packages in question.'
            )

        for module_finder, name, ispkg in pkgutil.walk_packages(package.__path__):
            module_name = '{}.{}'.format(package.__name__, name)
            current_module = importlib.import_module(module_name)
            imported.append(current_module)
            if ispkg:
                imported += load_all_modules_in_packages(current_module)

    for module in imported:
        # This is to cover cases where simply importing a module doesn't execute all the code/definitions within
        # I don't totally understand the reasons for this, but I do know enumerating a module's context (like with dir)
        # seems to solve things
        dir(module)

    return list(
        {
            module.__name__: module

            for module in imported
        }.values()
    )

def find_subclasses_in_packages(packages, base_classes):
    """
    Find objects in a set of packages that are class definitions that are subclasses of a set of base classes.

    :param packages: List of package references to the packages you're interested in.
    :param base_classes: List of classes that you want to find subclasses for.
    :return: List of objects mapping to the discovered subclasses
    """
    return _discover_objects(packages, base_classes, True)

def find_instances_in_packages(packages, base_classes):
    """
    Find objects in a set of packages that are instances of a set of base classes, or any subclasses.

    :param packages: List of package references to the packages you're interested in.
    :param base_classes: List of classes that you want to find subclasses for.
    :return: List of all discovered instances
    """
    return _discover_objects(packages, base_classes, False)

def _check_item(item, base_classes, class_definitions_only):
    if class_definitions_only:
        return (
                item not in base_classes and
                isclass(item) and
                any([issubclass(item, _class) for _class in base_classes])
        )
    else:
        return (
                isinstance(item, base_classes) and
                item not in base_classes
        )

def _get_name(obj):
    if isclass(obj):
        return obj.__name__
    else:
        return obj.__class__.__name__

def _discover_objects(packages, base_classes, class_definitions_only):
    discovered_objects = []

    modules = load_all_modules_in_packages(packages)

    if not isinstance(base_classes, tuple):
        base_classes = tuple(base_classes)

    for module in modules:
        for name, value in module.__dict__.items():
            if name.startswith("__") and name.endswith("__"):
                continue

            if _check_item(value, base_classes, class_definitions_only):
                discovered_objects.append(value)

    return list(
        {
            _get_name(item): item
            for item in discovered_objects
        }.values()
    )


# encoding: utf-8
"""A dict subclass that supports attribute style access.

Authors:

* Fernando Perez (original)
* Brian Granger (refactoring to a dict subclass)
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from IPython.utils.data import list2dict2

__all__ = ['Struct']

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

class Struct(dict):
    """A dict subclass with attribute style access.

    This dict subclass has a a few extra features:

    * Attribute style access.
    * Protection of class members (like keys, items) when using attribute
      style access.
    * The ability to restrict assignment to only existing keys.
    * Intelligent merging.
    * Overloaded operators.
    """
    _allownew = True
    def __init__(self, *args, **kw):
        """Initialize with a dictionary, another Struct, or data.

        Parameters
        ----------
        args : dict, Struct
            Initialize with one dict or Struct
        kw : dict
            Initialize with key, value pairs.

        Examples
        --------

        >>> s = Struct(a=10,b=30)
        >>> s.a
        10
        >>> s.b
        30
        >>> s2 = Struct(s,c=30)
        >>> sorted(s2.keys())
        ['a', 'b', 'c']
        """
        object.__setattr__(self, '_allownew', True)
        dict.__init__(self, *args, **kw)

    def __setitem__(self, key, value):
        """Set an item with check for allownew.

        Examples
        --------

        >>> s = Struct()
        >>> s['a'] = 10
        >>> s.allow_new_attr(False)
        >>> s['a'] = 10
        >>> s['a']
        10
        >>> try:
        ...     s['b'] = 20
        ... except KeyError:
        ...     print 'this is not allowed'
        ...
        this is not allowed
        """
        if not self._allownew and not self.has_key(key):
            raise KeyError(
                "can't create new attribute %s when allow_new_attr(False)" % key)
        dict.__setitem__(self, key, value)

    def __setattr__(self, key, value):
        """Set an attr with protection of class members.

        This calls :meth:`self.__setitem__` but convert :exc:`KeyError` to
        :exc:`AttributeError`.

        Examples
        --------

        >>> s = Struct()
        >>> s.a = 10
        >>> s.a
        10
        >>> try:
        ...     s.get = 10
        ... except AttributeError:
        ...     print "you can't set a class member"
        ...
        you can't set a class member
        """
        # If key is an str it might be a class member or instance var
        if isinstance(key, str):
            # I can't simply call hasattr here because it calls getattr, which
            # calls self.__getattr__, which returns True for keys in
            # self._data.  But I only want keys in the class and in
            # self.__dict__
            if key in self.__dict__ or hasattr(Struct, key):
                raise AttributeError(
                    'attr %s is a protected member of class Struct.' % key
                )
        try:
            self.__setitem__(key, value)
        except KeyError, e:
            raise AttributeError(e)

    def __getattr__(self, key):
        """Get an attr by calling :meth:`dict.__getitem__`.

        Like :meth:`__setattr__`, this method converts :exc:`KeyError` to
        :exc:`AttributeError`.

        Examples
        --------

        >>> s = Struct(a=10)
        >>> s.a
        10
        >>> type(s.get)
        <... 'builtin_function_or_method'>
        >>> try:
        ...     s.b
        ... except AttributeError:
        ...     print "I don't have that key"
        ...
        I don't have that key
        """
        try:
            result = self[key]
        except KeyError:
            raise AttributeError(key)
        else:
            return result

    def __iadd__(self, other):
        """s += s2 is a shorthand for s.merge(s2).

        Examples
        --------

        >>> s = Struct(a=10,b=30)
        >>> s2 = Struct(a=20,c=40)
        >>> s += s2
        >>> sorted(s.keys())
        ['a', 'b', 'c']
        """
        self.merge(other)
        return self

    def __add__(self,other):
        """s + s2 -> New Struct made from s.merge(s2).

        Examples
        --------

        >>> s1 = Struct(a=10,b=30)
        >>> s2 = Struct(a=20,c=40)
        >>> s = s1 + s2
        >>> sorted(s.keys())
        ['a', 'b', 'c']
        """
        sout = self.copy()
        sout.merge(other)
        return sout

    def __sub__(self,other):
        """s1 - s2 -> remove keys in s2 from s1.

        Examples
        --------

        >>> s1 = Struct(a=10,b=30)
        >>> s2 = Struct(a=40)
        >>> s = s1 - s2
        >>> s
        {'b': 30}
        """
        sout = self.copy()
        sout -= other
        return sout

    def __isub__(self,other):
        """Inplace remove keys from self that are in other.

        Examples
        --------

        >>> s1 = Struct(a=10,b=30)
        >>> s2 = Struct(a=40)
        >>> s1 -= s2
        >>> s1
        {'b': 30}
        """
        for k in other.keys():
            if self.has_key(k):
                del self[k]
        return self

    def __dict_invert(self, data):
        """Helper function for merge.

        Takes a dictionary whose values are lists and returns a dict with
        the elements of each list as keys and the original keys as values.
        """
        outdict = {}
        for k,lst in data.items():
            if isinstance(lst, str):
                lst = lst.split()
            for entry in lst:
                outdict[entry] = k
        return outdict

    def dict(self):
        return self

    def copy(self):
        """Return a copy as a Struct.

        Examples
        --------

        >>> s = Struct(a=10,b=30)
        >>> s2 = s.copy()
        >>> type(s2) is Struct
        True
        """
        return Struct(dict.copy(self))

    def hasattr(self, key):
        """hasattr function available as a method.

        Implemented like has_key.

        Examples
        --------

        >>> s = Struct(a=10)
        >>> s.hasattr('a')
        True
        >>> s.hasattr('b')
        False
        >>> s.hasattr('get')
        False
        """
        return self.has_key(key)

    def allow_new_attr(self, allow = True):
        """Set whether new attributes can be created in this Struct.

        This can be used to catch typos by verifying that the attribute user
        tries to change already exists in this Struct.
        """
        object.__setattr__(self, '_allownew', allow)

    def merge(self, __loc_data__=None, __conflict_solve=None, **kw):
        """Merge two Structs with customizable conflict resolution.

        This is similar to :meth:`update`, but much more flexible. First, a
        dict is made from data+key=value pairs. When merging this dict with
        the Struct S, the optional dictionary 'conflict' is used to decide
        what to do.

        If conflict is not given, the default behavior is to preserve any keys
        with their current value (the opposite of the :meth:`update` method's
        behavior).

        Parameters
        ----------
        __loc_data : dict, Struct
            The data to merge into self
        __conflict_solve : dict
            The conflict policy dict.  The keys are binary functions used to
            resolve the conflict and the values are lists of strings naming
            the keys the conflict resolution function applies to.  Instead of
            a list of strings a space separated string can be used, like
            'a b c'.
        kw : dict
            Additional key, value pairs to merge in

        Notes
        -----

        The `__conflict_solve` dict is a dictionary of binary functions which will be used to
        solve key conflicts.  Here is an example::

            __conflict_solve = dict(
                func1=['a','b','c'],
                func2=['d','e']
            )

        In this case, the function :func:`func1` will be used to resolve
        keys 'a', 'b' and 'c' and the function :func:`func2` will be used for
        keys 'd' and 'e'.  This could also be written as::

            __conflict_solve = dict(func1='a b c',func2='d e')

        These functions will be called for each key they apply to with the
        form::

            func1(self['a'], other['a'])

        The return value is used as the final merged value.

        As a convenience, merge() provides five (the most commonly needed)
        pre-defined policies: preserve, update, add, add_flip and add_s. The
        easiest explanation is their implementation::

            preserve = lambda old,new: old
            update   = lambda old,new: new
            add      = lambda old,new: old + new
            add_flip = lambda old,new: new + old  # note change of order!
            add_s    = lambda old,new: old + ' ' + new  # only for str!

        You can use those four words (as strings) as keys instead
        of defining them as functions, and the merge method will substitute
        the appropriate functions for you.

        For more complicated conflict resolution policies, you still need to
        construct your own functions.

        Examples
        --------

        This show the default policy:

        >>> s = Struct(a=10,b=30)
        >>> s2 = Struct(a=20,c=40)
        >>> s.merge(s2)
        >>> sorted(s.items())
        [('a', 10), ('b', 30), ('c', 40)]

        Now, show how to specify a conflict dict:

        >>> s = Struct(a=10,b=30)
        >>> s2 = Struct(a=20,b=40)
        >>> conflict = {'update':'a','add':'b'}
        >>> s.merge(s2,conflict)
        >>> sorted(s.items())
        [('a', 20), ('b', 70)]
        """

        data_dict = dict(__loc_data__,**kw)

        # policies for conflict resolution: two argument functions which return
        # the value that will go in the new struct
        preserve = lambda old,new: old
        update   = lambda old,new: new
        add      = lambda old,new: old + new
        add_flip = lambda old,new: new + old  # note change of order!
        add_s    = lambda old,new: old + ' ' + new

        # default policy is to keep current keys when there's a conflict
        conflict_solve = list2dict2(self.keys(), default = preserve)

        # the conflict_solve dictionary is given by the user 'inverted': we
        # need a name-function mapping, it comes as a function -> names
        # dict. Make a local copy (b/c we'll make changes), replace user
        # strings for the three builtin policies and invert it.
        if __conflict_solve:
            inv_conflict_solve_user = __conflict_solve.copy()
            for name, func in [('preserve',preserve), ('update',update),
                               ('add',add), ('add_flip',add_flip),
                               ('add_s',add_s)]:
                if name in inv_conflict_solve_user.keys():
                    inv_conflict_solve_user[func] = inv_conflict_solve_user[name]
                    del inv_conflict_solve_user[name]
            conflict_solve.update(self.__dict_invert(inv_conflict_solve_user))
        for key in data_dict:
            if key not in self:
                self[key] = data_dict[key]
            else:
                self[key] = conflict_solve[key](self[key],data_dict[key])


# -*- coding: utf-8 -*-
"""
Class and program to colorize python source code for ANSI terminals.

Based on an HTML code highlighter by Jurgen Hermann found at:
http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52298

Modifications by Fernando Perez (fperez@colorado.edu).

Information on the original HTML highlighter follows:

MoinMoin - Python Source Parser

Title: Colorize Python source using the built-in tokenizer

Submitter: Jurgen Hermann
Last Updated:2001/04/06

Version no:1.2

Description:

This code is part of MoinMoin (http://moin.sourceforge.net/) and converts
Python source code to HTML markup, rendering comments, keywords,
operators, numeric and string literals in different colors.

It shows how to use the built-in keyword, token and tokenize modules to
scan Python source code and re-emit it with no changes to its original
formatting (which is the hard part).
"""

__all__ = ['ANSICodeColors','Parser']

_scheme_default = 'Linux'

# Imports
import StringIO
import keyword
import os
import optparse
import sys
import token
import tokenize

try:
    generate_tokens = tokenize.generate_tokens
except AttributeError:
    # Python 3. Note that we use the undocumented _tokenize because it expects
    # strings, not bytes. See also Python issue #9969.
    generate_tokens = tokenize._tokenize

from IPython.utils.coloransi import *

#############################################################################
### Python Source Parser (does Hilighting)
#############################################################################

_KEYWORD = token.NT_OFFSET + 1
_TEXT    = token.NT_OFFSET + 2

#****************************************************************************
# Builtin color schemes

Colors = TermColors  # just a shorthand

# Build a few color schemes
NoColor = ColorScheme(
    'NoColor',{
    token.NUMBER     : Colors.NoColor,
    token.OP         : Colors.NoColor,
    token.STRING     : Colors.NoColor,
    tokenize.COMMENT : Colors.NoColor,
    token.NAME       : Colors.NoColor,
    token.ERRORTOKEN : Colors.NoColor,

    _KEYWORD         : Colors.NoColor,
    _TEXT            : Colors.NoColor,

    'normal'         : Colors.NoColor  # color off (usu. Colors.Normal)
    }  )

LinuxColors = ColorScheme(
    'Linux',{
    token.NUMBER     : Colors.LightCyan,
    token.OP         : Colors.Yellow,
    token.STRING     : Colors.LightBlue,
    tokenize.COMMENT : Colors.LightRed,
    token.NAME       : Colors.Normal,
    token.ERRORTOKEN : Colors.Red,

    _KEYWORD         : Colors.LightGreen,
    _TEXT            : Colors.Yellow,

    'normal'         : Colors.Normal  # color off (usu. Colors.Normal)
    } )

LightBGColors = ColorScheme(
    'LightBG',{
    token.NUMBER     : Colors.Cyan,
    token.OP         : Colors.Blue,
    token.STRING     : Colors.Blue,
    tokenize.COMMENT : Colors.Red,
    token.NAME       : Colors.Normal,
    token.ERRORTOKEN : Colors.Red,

    _KEYWORD         : Colors.Green,
    _TEXT            : Colors.Blue,

    'normal'         : Colors.Normal  # color off (usu. Colors.Normal)
    }  )

# Build table of color schemes (needed by the parser)
ANSICodeColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors],
                                  _scheme_default)

class Parser:
    """ Format colored Python source.
    """

    def __init__(self, color_table=None,out = sys.stdout):
        """ Create a parser with a specified color table and output channel.

        Call format() to process code.
        """
        self.color_table = color_table and color_table or ANSICodeColors
        self.out = out

    def format(self, raw, out = None, scheme = ''):
        return self.format2(raw, out, scheme)[0]

    def format2(self, raw, out = None, scheme = ''):
        """ Parse and send the colored source.

        If out and scheme are not specified, the defaults (given to
        constructor) are used.

        out should be a file-type object. Optionally, out can be given as the
        string 'str' and the parser will automatically return the output in a
        string."""

        string_output = 0
        if out == 'str' or self.out == 'str' or \
           isinstance(self.out,StringIO.StringIO):
            # XXX - I don't really like this state handling logic, but at this
            # point I don't want to make major changes, so adding the
            # isinstance() check is the simplest I can do to ensure correct
            # behavior.
            out_old = self.out
            self.out = StringIO.StringIO()
            string_output = 1
        elif out is not None:
            self.out = out

        # Fast return of the unmodified input for NoColor scheme
        if scheme == 'NoColor':
            error = False
            self.out.write(raw)
            if string_output:
                return raw,error
            else:
                return None,error

        # local shorthands
        colors = self.color_table[scheme].colors
        self.colors = colors # put in object so __call__ sees it

        # Remove trailing whitespace and normalize tabs
        self.raw = raw.expandtabs().rstrip()

        # store line offsets in self.lines
        self.lines = [0, 0]
        pos = 0
        raw_find = self.raw.find
        lines_append = self.lines.append
        while 1:
            pos = raw_find('\n', pos) + 1
            if not pos: break
            lines_append(pos)
        lines_append(len(self.raw))

        # parse the source and write it
        self.pos = 0
        text = StringIO.StringIO(self.raw)

        error = False
        try:
            for atoken in generate_tokens(text.readline):
                self(*atoken)
        except tokenize.TokenError as ex:
            msg = ex.args[0]
            line = ex.args[1][0]
            self.out.write("%s\n\n*** ERROR: %s%s%s\n" %
                           (colors[token.ERRORTOKEN],
                            msg, self.raw[self.lines[line]:],
                            colors.normal)
                           )
            error = True
        self.out.write(colors.normal+'\n')
        if string_output:
            output = self.out.getvalue()
            self.out = out_old
            return (output, error)
        return (None, error)

    def __call__(self, toktype, toktext, (srow,scol), (erow,ecol), line):
        """ Token handler, with syntax highlighting."""

        # local shorthands
        colors = self.colors
        owrite = self.out.write

        # line separator, so this works across platforms
        linesep = os.linesep

        # calculate new positions
        oldpos = self.pos
        newpos = self.lines[srow] + scol
        self.pos = newpos + len(toktext)

        # send the original whitespace, if needed
        if newpos > oldpos:
            owrite(self.raw[oldpos:newpos])

        # skip indenting tokens
        if toktype in [token.INDENT, token.DEDENT]:
            self.pos = newpos
            return

        # map token type to a color group
        if token.LPAR <= toktype and toktype <= token.OP:
            toktype = token.OP
        elif toktype == token.NAME and keyword.iskeyword(toktext):
            toktype = _KEYWORD
        color = colors.get(toktype, colors[_TEXT])

        #print '<%s>' % toktext,    # dbg

        # Triple quoted strings must be handled carefully so that backtracking
        # in pagers works correctly. We need color terminators on _each_ line.
        if linesep in toktext:
            toktext = toktext.replace(linesep, '%s%s%s' %
                                      (colors.normal,linesep,color))

        # send text
        owrite('%s%s%s' % (color,toktext,colors.normal))

def main(argv=None):
    """Run as a command-line script: colorize a python file or stdin using ANSI
    color escapes and print to stdout.

    Inputs:

      - argv(None): a list of strings like sys.argv[1:] giving the command-line
        arguments. If None, use sys.argv[1:].
    """

    usage_msg = """%prog [options] [filename]

Colorize a python file or stdin using ANSI color escapes and print to stdout.
If no filename is given, or if filename is -, read standard input."""

    parser = optparse.OptionParser(usage=usage_msg)
    newopt = parser.add_option
    newopt('-s','--scheme',metavar='NAME',dest='scheme_name',action='store',
           choices=['Linux','LightBG','NoColor'],default=_scheme_default,
           help="give the color scheme to use. Currently only 'Linux'\
 (default) and 'LightBG' and 'NoColor' are implemented (give without\
 quotes)")

    opts,args = parser.parse_args(argv)

    if len(args) > 1:
        parser.error("you must give at most one filename.")

    if len(args) == 0:
        fname = '-' # no filename given; setup to read from stdin
    else:
        fname = args[0]

    if fname == '-':
        stream = sys.stdin
    else:
        try:
            stream = open(fname)
        except IOError,msg:
            print >> sys.stderr, msg
            sys.exit(1)

    parser = Parser()

    # we need nested try blocks because pre-2.5 python doesn't support unified
    # try-except-finally
    try:
        try:
            # write colorized version to stdout
            parser.format(stream.read(),scheme=opts.scheme_name)
        except IOError,msg:
            # if user reads through a pager and quits, don't print traceback
            if msg.args != (32,'Broken pipe'):
                raise
    finally:
        if stream is not sys.stdin:
            stream.close() # in case a non-handled exception happened above

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""Pylab (matplotlib) support utilities.

Authors
-------

* Fernando Perez.
* Brian Granger
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2009-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import sys
from io import BytesIO

from IPython.utils.decorators import flag_calls

# If user specifies a GUI, that dictates the backend, otherwise we read the
# user's mpl default from the mpl rc structure
backends = {'tk': 'TkAgg',
            'gtk': 'GTKAgg',
            'wx': 'WXAgg',
            'qt': 'Qt4Agg', # qt3 not supported
            'qt4': 'Qt4Agg',
            'osx': 'MacOSX',
            'inline' : 'module://IPython.zmq.pylab.backend_inline'}

# We also need a reverse backends2guis mapping that will properly choose which
# GUI support to activate based on the desired matplotlib backend.  For the
# most part it's just a reverse of the above dict, but we also need to add a
# few others that map to the same GUI manually:
backend2gui = dict(zip(backends.values(), backends.keys()))
# In the reverse mapping, there are a few extra valid matplotlib backends that
# map to the same GUI support
backend2gui['GTK'] = backend2gui['GTKCairo'] = 'gtk'
backend2gui['WX'] = 'wx'
backend2gui['CocoaAgg'] = 'osx'

#-----------------------------------------------------------------------------
# Matplotlib utilities
#-----------------------------------------------------------------------------

def getfigs(*fig_nums):
    """Get a list of matplotlib figures by figure numbers.

    If no arguments are given, all available figures are returned.  If the
    argument list contains references to invalid figures, a warning is printed
    but the function continues pasting further figures.

    Parameters
    ----------
    figs : tuple
        A tuple of ints giving the figure numbers of the figures to return.
    """
    from matplotlib._pylab_helpers import Gcf
    if not fig_nums:
        fig_managers = Gcf.get_all_fig_managers()
        return [fm.canvas.figure for fm in fig_managers]
    else:
        figs = []
        for num in fig_nums:
            f = Gcf.figs.get(num)
            if f is None:
                print('Warning: figure %s not available.' % num)
            else:
                figs.append(f.canvas.figure)
        return figs

def figsize(sizex, sizey):
    """Set the default figure size to be [sizex, sizey].

    This is just an easy to remember, convenience wrapper that sets::

      matplotlib.rcParams['figure.figsize'] = [sizex, sizey]
    """
    import matplotlib
    matplotlib.rcParams['figure.figsize'] = [sizex, sizey]

def print_figure(fig, fmt='png'):
    """Convert a figure to svg or png for inline display."""
    # When there's an empty figure, we shouldn't return anything, otherwise we
    # get big blank areas in the qt console.
    if not fig.axes and not fig.lines:
        return

    fc = fig.get_facecolor()
    ec = fig.get_edgecolor()
    fig.set_facecolor('white')
    fig.set_edgecolor('white')
    try:
        bytes_io = BytesIO()
        fig.canvas.print_figure(bytes_io, format=fmt, bbox_inches='tight')
        data = bytes_io.getvalue()
    finally:
        fig.set_facecolor(fc)
        fig.set_edgecolor(ec)
    return data

# We need a little factory function here to create the closure where
# safe_execfile can live.
def mpl_runner(safe_execfile):
    """Factory to return a matplotlib-enabled runner for %run.

    Parameters
    ----------
    safe_execfile : function
      This must be a function with the same interface as the
      :meth:`safe_execfile` method of IPython.

    Returns
    -------
    A function suitable for use as the ``runner`` argument of the %run magic
    function.
    """

    def mpl_execfile(fname,*where,**kw):
        """matplotlib-aware wrapper around safe_execfile.

        Its interface is identical to that of the :func:`execfile` builtin.

        This is ultimately a call to execfile(), but wrapped in safeties to
        properly handle interactive rendering."""

        import matplotlib
        import matplotlib.pylab as pylab

        #print '*** Matplotlib runner ***' # dbg
        # turn off rendering until end of script
        is_interactive = matplotlib.rcParams['interactive']
        matplotlib.interactive(False)
        safe_execfile(fname,*where,**kw)
        matplotlib.interactive(is_interactive)
        # make rendering call now, if the user tried to do it
        if pylab.draw_if_interactive.called:
            pylab.draw()
            pylab.draw_if_interactive.called = False

    return mpl_execfile

def select_figure_format(shell, fmt):
    """Select figure format for inline backend, either 'png' or 'svg'.

    Using this method ensures only one figure format is active at a time.
    """
    from matplotlib.figure import Figure
    from IPython.zmq.pylab import backend_inline

    svg_formatter = shell.display_formatter.formatters['image/svg+xml']
    png_formatter = shell.display_formatter.formatters['image/png']

    if fmt=='png':
        svg_formatter.type_printers.pop(Figure, None)
        png_formatter.for_type(Figure, lambda fig: print_figure(fig, 'png'))
    elif fmt=='svg':
        png_formatter.type_printers.pop(Figure, None)
        svg_formatter.for_type(Figure, lambda fig: print_figure(fig, 'svg'))
    else:
        raise ValueError("supported formats are: 'png', 'svg', not %r"%fmt)

    # set the format to be used in the backend()
    backend_inline._figure_format = fmt

#-----------------------------------------------------------------------------
# Code for initializing matplotlib and importing pylab
#-----------------------------------------------------------------------------

def find_gui_and_backend(gui=None):
    """Given a gui string return the gui and mpl backend.

    Parameters
    ----------
    gui : str
        Can be one of ('tk','gtk','wx','qt','qt4','inline').

    Returns
    -------
    A tuple of (gui, backend) where backend is one of ('TkAgg','GTKAgg',
    'WXAgg','Qt4Agg','module://IPython.zmq.pylab.backend_inline').
    """

    import matplotlib

    if gui and gui != 'auto':
        # select backend based on requested gui
        backend = backends[gui]
    else:
        backend = matplotlib.rcParams['backend']
        # In this case, we need to find what the appropriate gui selection call
        # should be for IPython, so we can activate inputhook accordingly
        gui = backend2gui.get(backend, None)
    return gui, backend

def activate_matplotlib(backend):
    """Activate the given backend and set interactive to True."""

    import matplotlib
    if backend.startswith('module://'):
        # Work around bug in matplotlib: matplotlib.use converts the
        # backend_id to lowercase even if a module name is specified!
        matplotlib.rcParams['backend'] = backend
    else:
        matplotlib.use(backend)
    matplotlib.interactive(True)

    # This must be imported last in the matplotlib series, after
    # backend/interactivity choices have been made
    import matplotlib.pylab as pylab

    # XXX For now leave this commented out, but depending on discussions with
    # mpl-dev, we may be able to allow interactive switching...
    #import matplotlib.pyplot
    #matplotlib.pyplot.switch_backend(backend)

    pylab.show._needmain = False
    # We need to detect at runtime whether show() is called by the user.
    # For this, we wrap it into a decorator which adds a 'called' flag.
    pylab.draw_if_interactive = flag_calls(pylab.draw_if_interactive)

def import_pylab(user_ns, import_all=True):
    """Import the standard pylab symbols into user_ns."""

    # Import numpy as np/pyplot as plt are conventions we're trying to
    # somewhat standardize on.  Making them available to users by default
    # will greatly help this.
    s = ("import numpy\n"
          "import matplotlib\n"
          "from matplotlib import pylab, mlab, pyplot\n"
          "np = numpy\n"
          "plt = pyplot\n"
          )
    exec s in user_ns

    if import_all:
        s = ("from matplotlib.pylab import *\n"
             "from numpy import *\n")
        exec s in user_ns

def configure_inline_support(shell, backend, user_ns=None):
    """Configure an IPython shell object for matplotlib use.

    Parameters
    ----------
    shell : InteractiveShell instance

    backend : matplotlib backend

    user_ns : dict
      A namespace where all configured variables will be placed.  If not given,
      the `user_ns` attribute of the shell object is used.
    """
    # If using our svg payload backend, register the post-execution
    # function that will pick up the results for display.  This can only be
    # done with access to the real shell object.

    # Note: if we can't load the inline backend, then there's no point
    # continuing (such as in terminal-only shells in environments without
    # zeromq available).
    try:
        from IPython.zmq.pylab.backend_inline import InlineBackend
    except ImportError:
        return

    user_ns = shell.user_ns if user_ns is None else user_ns

    cfg = InlineBackend.instance(config=shell.config)
    cfg.shell = shell
    if cfg not in shell.configurables:
        shell.configurables.append(cfg)

    if backend == backends['inline']:
        from IPython.zmq.pylab.backend_inline import flush_figures
        from matplotlib import pyplot
        shell.register_post_execute(flush_figures)
        # load inline_rc
        pyplot.rcParams.update(cfg.rc)
        # Add 'figsize' to pyplot and to the user's namespace
        user_ns['figsize'] = pyplot.figsize = figsize

    # Setup the default figure format
    fmt = cfg.figure_format
    select_figure_format(shell, fmt)

    # The old pastefig function has been replaced by display
    from IPython.core.display import display
    # Add display and getfigs to the user's namespace
    user_ns['display'] = display
    user_ns['getfigs'] = getfigs

def pylab_activate(user_ns, gui=None, import_all=True, shell=None):
    """Activate pylab mode in the user's namespace.

    Loads and initializes numpy, matplotlib and friends for interactive use.

    Parameters
    ----------
    user_ns : dict
      Namespace where the imports will occur.

    gui : optional, string
      A valid gui name following the conventions of the %gui magic.

    import_all : optional, boolean
      If true, an 'import *' is done from numpy and pylab.

    Returns
    -------
    The actual gui used (if not given as input, it was obtained from matplotlib
    itself, and will be needed next to configure IPython's gui integration.
    """
    gui, backend = find_gui_and_backend(gui)
    activate_matplotlib(backend)
    import_pylab(user_ns, import_all)
    if shell is not None:
        configure_inline_support(shell, backend, user_ns)

    print """
Welcome to pylab, a matplotlib-based Python environment [backend: %s].
For more information, type 'help(pylab)'.""" % backend
    # flush stdout, just to be safe
    sys.stdout.flush()

    return gui