"""A script for watching all traffic on the IOPub channel (stdout/stderr/pyerr) of engines.

This connects to the default cluster, or you can pass the path to your ipcontroller-client.json

Try running this script, and then running a few jobs that print (and call sys.stdout.flush),
and you will see the print statements as they arrive, notably not waiting for the results
to finish.

You can use the zeromq SUBSCRIBE mechanism to only receive information from specific engines,
and easily filter by message type.

Authors
-------
* MinRK
"""

import os
import sys
import json
import zmq

from IPython.zmq.session import Session
from IPython.parallel.util import disambiguate_url
from IPython.utils.py3compat import str_to_bytes
from IPython.utils.path import get_security_file

def main(connection_file):
    """watch iopub channel, and print messages"""

    ctx = zmq.Context.instance()

    with open(connection_file) as f:
        cfg = json.loads(f.read())

    location = cfg['location']
    reg_url = cfg['url']
    session = Session(key=str_to_bytes(cfg['exec_key']))

    query = ctx.socket(zmq.DEALER)
    query.connect(disambiguate_url(cfg['url'], location))
    session.send(query, "connection_request")
    idents,msg = session.recv(query, mode=0)
    c = msg['content']
    iopub_url = disambiguate_url(c['iopub'], location)
    sub = ctx.socket(zmq.SUB)
    # This will subscribe to all messages:
    sub.setsockopt(zmq.SUBSCRIBE, b'')
    # replace with b'' with b'engine.1.stdout' to subscribe only to engine 1's stdout
    # 0MQ subscriptions are simple 'foo*' matches, so 'engine.1.' subscribes
    # to everything from engine 1, but there is no way to subscribe to
    # just stdout from everyone.
    # multiple calls to subscribe will add subscriptions, e.g. to subscribe to
    # engine 1's stderr and engine 2's stdout:
    # sub.setsockopt(zmq.SUBSCRIBE, b'engine.1.stderr')
    # sub.setsockopt(zmq.SUBSCRIBE, b'engine.2.stdout')
    sub.connect(iopub_url)
    while True:
        try:
            idents,msg = session.recv(sub, mode=0)
        except KeyboardInterrupt:
            return
        # ident always length 1 here
        topic = idents[0]
        if msg['msg_type'] == 'stream':
            # stdout/stderr
            # stream names are in msg['content']['name'], if you want to handle
            # them differently
            print("%s: %s" % (topic, msg['content']['data']))
        elif msg['msg_type'] == 'pyerr':
            # Python traceback
            c = msg['content']
            print(topic + ':')
            for line in c['traceback']:
                # indent lines
                print('    ' + line)

if __name__ == '__main__':
    if len(sys.argv) > 1:
        cf = sys.argv[1]
    else:
        # This gets the security file for the default profile:
        cf = get_security_file('ipcontroller-client.json')
    main(cf)


##########################     LICENCE     ###############################

# Copyright (c) 2005-2012, Michele Simionato
# All rights reserved.

# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:

#   Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
#   Redistributions in bytecode form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in
#   the documentation and/or other materials provided with the
#   distribution.

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
# DAMAGE.

"""
Decorator module, see http://pypi.python.org/pypi/decorator
for the documentation.
"""

__version__ = '3.3.3'

__all__ = ["decorator", "FunctionMaker", "partial"]

import sys, re, inspect

try:
    from functools import partial
except ImportError: # for Python version < 2.5
    class partial(object):
        "A simple replacement of functools.partial"
        def __init__(self, func, *args, **kw):
            self.func = func
            self.args = args
            self.keywords = kw
        def __call__(self, *otherargs, **otherkw):
            kw = self.keywords.copy()
            kw.update(otherkw)
            return self.func(*(self.args + otherargs), **kw)

if sys.version >= '3':
    from inspect import getfullargspec
else:
    class getfullargspec(object):
        "A quick and dirty replacement for getfullargspec for Python 2.X"
        def __init__(self, f):
            self.args, self.varargs, self.varkw, self.defaults = \
                inspect.getargspec(f)
            self.kwonlyargs = []
            self.kwonlydefaults = None
        def __iter__(self):
            yield self.args
            yield self.varargs
            yield self.varkw
            yield self.defaults

DEF = re.compile('\s*def\s*([_\w][_\w\d]*)\s*\(')

# basic functionality
class FunctionMaker(object):
    """
    An object with the ability to create functions with a given signature.
    It has attributes name, doc, module, signature, defaults, dict and
    methods update and make.
    """
    def __init__(self, func=None, name=None, signature=None,
                 defaults=None, doc=None, module=None, funcdict=None):
        self.shortsignature = signature
        if func:
            # func can be a class or a callable, but not an instance method
            self.name = func.__name__
            if self.name == '<lambda>': # small hack for lambda functions
                self.name = '_lambda_'
            self.doc = func.__doc__
            self.module = func.__module__
            if inspect.isfunction(func):
                argspec = getfullargspec(func)
                self.annotations = getattr(func, '__annotations__', {})
                for a in ('args', 'varargs', 'varkw', 'defaults', 'kwonlyargs',
                          'kwonlydefaults'):
                    setattr(self, a, getattr(argspec, a))
                for i, arg in enumerate(self.args):
                    setattr(self, 'arg%d' % i, arg)
                if sys.version < '3': # easy way
                    self.shortsignature = self.signature = \
                        inspect.formatargspec(
                        formatvalue=lambda val: "", *argspec)[1:-1]
                else: # Python 3 way
                    self.signature = self.shortsignature = ', '.join(self.args)
                    if self.varargs:
                        self.signature += ', *' + self.varargs
                        self.shortsignature += ', *' + self.varargs
                    if self.kwonlyargs:
                        for a in self.kwonlyargs:
                            self.signature += ', %s=None' % a
                            self.shortsignature += ', %s=%s' % (a, a)
                    if self.varkw:
                        self.signature += ', **' + self.varkw
                        self.shortsignature += ', **' + self.varkw
                self.dict = func.__dict__.copy()
        # func=None happens when decorating a caller
        if name:
            self.name = name
        if signature is not None:
            self.signature = signature
        if defaults:
            self.defaults = defaults
        if doc:
            self.doc = doc
        if module:
            self.module = module
        if funcdict:
            self.dict = funcdict
        # check existence required attributes
        assert hasattr(self, 'name')
        if not hasattr(self, 'signature'):
            raise TypeError('You are decorating a non function: %s' % func)

    def update(self, func, **kw):
        "Update the signature of func with the data in self"
        func.__name__ = self.name
        func.__doc__ = getattr(self, 'doc', None)
        func.__dict__ = getattr(self, 'dict', {})
        func.func_defaults = getattr(self, 'defaults', ())
        func.__kwdefaults__ = getattr(self, 'kwonlydefaults', None)
        func.__annotations__ = getattr(self, 'annotations', None)
        callermodule = sys._getframe(3).f_globals.get('__name__', '?')
        func.__module__ = getattr(self, 'module', callermodule)
        func.__dict__.update(kw)

    def make(self, src_templ, evaldict=None, addsource=False, **attrs):
        "Make a new function from a given template and update the signature"
        src = src_templ % vars(self) # expand name and signature
        evaldict = evaldict or {}
        mo = DEF.match(src)
        if mo is None:
            raise SyntaxError('not a valid function template\n%s' % src)
        name = mo.group(1) # extract the function name
        names = set([name] + [arg.strip(' *') for arg in
                             self.shortsignature.split(',')])
        for n in names:
            if n in ('_func_', '_call_'):
                raise NameError('%s is overridden in\n%s' % (n, src))
        if not src.endswith('\n'): # add a newline just for safety
            src += '\n' # this is needed in old versions of Python
        try:
            code = compile(src, '<string>', 'single')
            # print >> sys.stderr, 'Compiling %s' % src
            exec code in evaldict
        except:
            print >> sys.stderr, 'Error in generated code:'
            print >> sys.stderr, src
            raise
        func = evaldict[name]
        if addsource:
            attrs['__source__'] = src
        self.update(func, **attrs)
        return func

    @classmethod
    def create(cls, obj, body, evaldict, defaults=None,
               doc=None, module=None, addsource=True, **attrs):
        """
        Create a function from the strings name, signature and body.
        evaldict is the evaluation dictionary. If addsource is true an attribute
        __source__ is added to the result. The attributes attrs are added,
        if any.
        """
        if isinstance(obj, str): # "name(signature)"
            name, rest = obj.strip().split('(', 1)
            signature = rest[:-1] #strip a right parens
            func = None
        else: # a function
            name = None
            signature = None
            func = obj
        self = cls(func, name, signature, defaults, doc, module)
        ibody = '\n'.join('    ' + line for line in body.splitlines())
        return self.make('def %(name)s(%(signature)s):\n' + ibody,
                        evaldict, addsource, **attrs)

def decorator(caller, func=None):
    """
    decorator(caller) converts a caller function into a decorator;
    decorator(caller, func) decorates a function using a caller.
    """
    if func is not None: # returns a decorated function
        evaldict = func.func_globals.copy()
        evaldict['_call_'] = caller
        evaldict['_func_'] = func
        return FunctionMaker.create(
            func, "return _call_(_func_, %(shortsignature)s)",
            evaldict, undecorated=func, __wrapped__=func)
    else: # returns a decorator
        if isinstance(caller, partial):
            return partial(decorator, caller)
        # otherwise assume caller is a function
        first = inspect.getargspec(caller)[0][0] # first arg
        evaldict = caller.func_globals.copy()
        evaldict['_call_'] = caller
        evaldict['decorator'] = decorator
        return FunctionMaker.create(
            '%s(%s)' % (caller.__name__, first),
            'return decorator(_call_, %s)' % first,
            evaldict, undecorated=caller, __wrapped__=caller,
            doc=caller.__doc__, module=caller.__module__)


from __future__ import absolute_import

import logging
import operator
import os
import tempfile
import shutil
import warnings

from pip.req import InstallRequirement, RequirementSet, parse_requirements
from pip.locations import build_prefix, virtualenv_no_global, distutils_scheme
from pip.basecommand import Command
from pip.index import PackageFinder
from pip.exceptions import (
    InstallationError, CommandError, PreviousBuildDirError,
)
from pip import cmdoptions
from pip.utils.build import BuildDirectory
from pip.utils.deprecation import RemovedInPip7Warning, RemovedInPip8Warning

logger = logging.getLogger(__name__)

class InstallCommand(Command):
    """
    Install packages from:

    - PyPI (and other indexes) using requirement specifiers.
    - VCS project urls.
    - Local project directories.
    - Local or remote source archives.

    pip also supports installing from "requirements files", which provide
    an easy way to specify a whole environment to be installed.
    """
    name = 'install'

    usage = """
      %prog [options] <requirement specifier> [package-index-options] ...
      %prog [options] -r <requirements file> [package-index-options] ...
      %prog [options] [-e] <vcs project url> ...
      %prog [options] [-e] <local project path> ...
      %prog [options] <archive url/path> ..."""

    summary = 'Install packages.'

    def __init__(self, *args, **kw):
        super(InstallCommand, self).__init__(*args, **kw)

        cmd_opts = self.cmd_opts

        cmd_opts.add_option(cmdoptions.editable.make())
        cmd_opts.add_option(cmdoptions.requirements.make())
        cmd_opts.add_option(cmdoptions.build_dir.make())

        cmd_opts.add_option(
            '-t', '--target',
            dest='target_dir',
            metavar='dir',
            default=None,
            help='Install packages into <dir>. '
                 'By default this will not replace existing files/folders in '
                 '<dir>. Use --upgrade to replace existing packages in <dir> '
                 'with new versions.'
        )

        cmd_opts.add_option(
            '-d', '--download', '--download-dir', '--download-directory',
            dest='download_dir',
            metavar='dir',
            default=None,
            help=("Download packages into <dir> instead of installing them, "
                  "regardless of what's already installed."),
        )

        cmd_opts.add_option(cmdoptions.download_cache.make())
        cmd_opts.add_option(cmdoptions.src.make())

        cmd_opts.add_option(
            '-U', '--upgrade',
            dest='upgrade',
            action='store_true',
            help='Upgrade all specified packages to the newest available '
                 'version. This process is recursive regardless of whether '
                 'a dependency is already satisfied.'
        )

        cmd_opts.add_option(
            '--force-reinstall',
            dest='force_reinstall',
            action='store_true',
            help='When upgrading, reinstall all packages even if they are '
                 'already up-to-date.')

        cmd_opts.add_option(
            '-I', '--ignore-installed',
            dest='ignore_installed',
            action='store_true',
            help='Ignore the installed packages (reinstalling instead).')

        cmd_opts.add_option(cmdoptions.no_deps.make())

        cmd_opts.add_option(
            '--no-install',
            dest='no_install',
            action='store_true',
            help="DEPRECATED. Download and unpack all packages, but don't "
                 "actually install them."
        )

        cmd_opts.add_option(
            '--no-download',
            dest='no_download',
            action="store_true",
            help="DEPRECATED. Don't download any packages, just install the "
                 "ones already downloaded (completes an install run with "
                 "--no-install).")

        cmd_opts.add_option(cmdoptions.install_options.make())
        cmd_opts.add_option(cmdoptions.global_options.make())

        cmd_opts.add_option(
            '--user',
            dest='use_user_site',
            action='store_true',
            help='Install using the user scheme.')

        cmd_opts.add_option(
            '--egg',
            dest='as_egg',
            action='store_true',
            help="Install packages as eggs, not 'flat', like pip normally "
                 "does. This option is not about installing *from* eggs. "
                 "(WARNING: Because this option overrides pip's normal install"
                 " logic, requirements files may not behave as expected.)")

        cmd_opts.add_option(
            '--root',
            dest='root_path',
            metavar='dir',
            default=None,
            help="Install everything relative to this alternate root "
                 "directory.")

        cmd_opts.add_option(
            "--compile",
            action="store_true",
            dest="compile",
            default=True,
            help="Compile py files to pyc",
        )

        cmd_opts.add_option(
            "--no-compile",
            action="store_false",
            dest="compile",
            help="Do not compile py files to pyc",
        )

        cmd_opts.add_option(cmdoptions.use_wheel.make())
        cmd_opts.add_option(cmdoptions.no_use_wheel.make())

        cmd_opts.add_option(
            '--pre',
            action='store_true',
            default=False,
            help="Include pre-release and development versions. By default, "
                 "pip only finds stable versions.")

        cmd_opts.add_option(cmdoptions.no_clean.make())

        index_opts = cmdoptions.make_option_group(
            cmdoptions.index_group,
            self.parser,
        )

        self.parser.insert_option_group(0, index_opts)
        self.parser.insert_option_group(0, cmd_opts)

    def _build_package_finder(self, options, index_urls, session):
        """
        Create a package finder appropriate to this install command.
        This method is meant to be overridden by subclasses, not
        called directly.
        """
        return PackageFinder(
            find_links=options.find_links,
            index_urls=index_urls,
            use_wheel=options.use_wheel,
            allow_external=options.allow_external,
            allow_unverified=options.allow_unverified,
            allow_all_external=options.allow_all_external,
            trusted_hosts=options.trusted_hosts,
            allow_all_prereleases=options.pre,
            process_dependency_links=options.process_dependency_links,
            session=session,
        )

    def run(self, options, args):

        if (
            options.no_install or
            options.no_download
        ):
            warnings.warn(
                "--no-install and --no-download are deprecated. "
                "See https://github.com/pypa/pip/issues/906.",
                RemovedInPip7Warning,
            )

        # If we have --no-install or --no-download and no --build we use the
        # legacy static build dir
        if (options.build_dir is None
                and (options.no_install or options.no_download)):
            options.build_dir = build_prefix

        if options.download_dir:
            options.no_install = True
            options.ignore_installed = True

        if options.build_dir:
            options.build_dir = os.path.abspath(options.build_dir)

        options.src_dir = os.path.abspath(options.src_dir)
        install_options = options.install_options or []
        if options.use_user_site:
            if virtualenv_no_global():
                raise InstallationError(
                    "Can not perform a '--user' install. User site-packages "
                    "are not visible in this virtualenv."
                )
            install_options.append('--user')

        temp_target_dir = None
        if options.target_dir:
            options.ignore_installed = True
            temp_target_dir = tempfile.mkdtemp()
            options.target_dir = os.path.abspath(options.target_dir)
            if (os.path.exists(options.target_dir)
                    and not os.path.isdir(options.target_dir)):
                raise CommandError(
                    "Target path exists but is not a directory, will not "
                    "continue."
                )
            install_options.append('--home=' + temp_target_dir)

        global_options = options.global_options or []
        index_urls = [options.index_url] + options.extra_index_urls
        if options.no_index:
            logger.info('Ignoring indexes: %s', ','.join(index_urls))
            index_urls = []

        if options.use_mirrors:
            warnings.warn(
                "--use-mirrors has been deprecated and will be removed in the "
                "future. Explicit uses of --index-url and/or --extra-index-url"
                " is suggested.",
                RemovedInPip7Warning,
            )

        if options.mirrors:
            warnings.warn(
                "--mirrors has been deprecated and will be removed in the "
                "future. Explicit uses of --index-url and/or --extra-index-url"
                " is suggested.",
                RemovedInPip7Warning,
            )
            index_urls += options.mirrors

        if options.download_cache:
            warnings.warn(
                "--download-cache has been deprecated and will be removed in "
                "the future. Pip now automatically uses and configures its "
                "cache.",
                RemovedInPip8Warning,
            )

        with self._build_session(options) as session:

            finder = self._build_package_finder(options, index_urls, session)

            build_delete = (not (options.no_clean or options.build_dir))
            with BuildDirectory(options.build_dir,
                                delete=build_delete) as build_dir:
                requirement_set = RequirementSet(
                    build_dir=build_dir,
                    src_dir=options.src_dir,
                    download_dir=options.download_dir,
                    upgrade=options.upgrade,
                    as_egg=options.as_egg,
                    ignore_installed=options.ignore_installed,
                    ignore_dependencies=options.ignore_dependencies,
                    force_reinstall=options.force_reinstall,
                    use_user_site=options.use_user_site,
                    target_dir=temp_target_dir,
                    session=session,
                    pycompile=options.compile,
                    isolated=options.isolated_mode,
                )

                for name in args:
                    requirement_set.add_requirement(
                        InstallRequirement.from_line(
                            name, None, isolated=options.isolated_mode,
                        )
                    )

                for name in options.editables:
                    requirement_set.add_requirement(
                        InstallRequirement.from_editable(
                            name,
                            default_vcs=options.default_vcs,
                            isolated=options.isolated_mode,
                        )
                    )

                for filename in options.requirements:
                    for req in parse_requirements(
                            filename,
                            finder=finder, options=options, session=session):
                        requirement_set.add_requirement(req)

                if not requirement_set.has_requirements:
                    opts = {'name': self.name}
                    if options.find_links:
                        msg = ('You must give at least one requirement to '
                               '%(name)s (maybe you meant "pip %(name)s '
                               '%(links)s"?)' %
                               dict(opts, links=' '.join(options.find_links)))
                    else:
                        msg = ('You must give at least one requirement '
                               'to %(name)s (see "pip help %(name)s")' % opts)
                    logger.warning(msg)
                    return

                try:
                    if not options.no_download:
                        requirement_set.prepare_files(finder)
                    else:
                        requirement_set.locate_files()

                    if not options.no_install:
                        requirement_set.install(
                            install_options,
                            global_options,
                            root=options.root_path,
                        )
                        reqs = sorted(
                            requirement_set.successfully_installed,
                            key=operator.attrgetter('name'))
                        items = []
                        for req in reqs:
                            item = req.name
                            try:
                                if hasattr(req, 'installed_version'):
                                    if req.installed_version:
                                        item += '-' + req.installed_version
                            except Exception:
                                pass
                            items.append(item)
                        installed = ' '.join(items)
                        if installed:
                            logger.info('Successfully installed %s', installed)
                    else:
                        downloaded = ' '.join([
                            req.name
                            for req in requirement_set.successfully_downloaded
                        ])
                        if downloaded:
                            logger.info(
                                'Successfully downloaded %s', downloaded
                            )
                except PreviousBuildDirError:
                    options.no_clean = True
                    raise
                finally:
                    # Clean up
                    if ((not options.no_clean)
                            and ((not options.no_install)
                                 or options.download_dir)):
                        requirement_set.cleanup_files()

        if options.target_dir:
            if not os.path.exists(options.target_dir):
                os.makedirs(options.target_dir)

            lib_dir = distutils_scheme('', home=temp_target_dir)['purelib']

            for item in os.listdir(lib_dir):
                target_item_dir = os.path.join(options.target_dir, item)
                if os.path.exists(target_item_dir):
                    if not options.upgrade:
                        logger.warning(
                            'Target directory %s already exists. Specify '
                            '--upgrade to force replacement.',
                            target_item_dir
                        )
                        continue
                    if os.path.islink(target_item_dir):
                        logger.warning(
                            'Target directory %s already exists and is '
                            'a link. Pip will not automatically replace '
                            'links, please remove if replacement is '
                            'desired.',
                            target_item_dir
                        )
                        continue
                    if os.path.isdir(target_item_dir):
                        shutil.rmtree(target_item_dir)
                    else:
                        os.remove(target_item_dir)

                shutil.move(
                    os.path.join(lib_dir, item),
                    target_item_dir
                )
            shutil.rmtree(temp_target_dir)
        return requirement_set


# encoding: utf-8
"""
A base class for a configurable application.

Authors:

* Brian Granger
* Min RK
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import logging
import os
import re
import sys
from copy import deepcopy
from collections import defaultdict

from IPython.external.decorator import decorator

from IPython.config.configurable import SingletonConfigurable
from IPython.config.loader import (
    KVArgParseConfigLoader, PyFileConfigLoader, Config, ArgumentError, ConfigFileNotFound,
)

from IPython.utils.traitlets import (
    Unicode, List, Enum, Dict, Instance, TraitError
)
from IPython.utils.importstring import import_item
from IPython.utils.text import indent, wrap_paragraphs, dedent

#-----------------------------------------------------------------------------
# function for re-wrapping a helpstring
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Descriptions for the various sections
#-----------------------------------------------------------------------------

# merge flags&aliases into options
option_description = """
Arguments that take values are actually convenience aliases to full
Configurables, whose aliases are listed on the help line. For more information
on full configurables, see '--help-all'.
""".strip() # trim newlines of front and back

keyvalue_description = """
Parameters are set from command-line arguments of the form:
`--Class.trait=value`.
This line is evaluated in Python, so simple expressions are allowed, e.g.::
`--C.a='range(3)'` For setting C.a=[0,1,2].
""".strip() # trim newlines of front and back

subcommand_description = """
Subcommands are launched as `{app} cmd [args]`. For information on using
subcommand 'cmd', do: `{app} cmd -h`.
""".strip().format(app=os.path.basename(sys.argv[0]))
# get running program name

#-----------------------------------------------------------------------------
# Application class
#-----------------------------------------------------------------------------

@decorator
def catch_config_error(method, app, *args, **kwargs):
    """Method decorator for catching invalid config (Trait/ArgumentErrors) during init.

    On a TraitError (generally caused by bad config), this will print the trait's
    message, and exit the app.

    For use on init methods, to prevent invoking excepthook on invalid input.
    """
    try:
        return method(app, *args, **kwargs)
    except (TraitError, ArgumentError) as e:
        app.print_description()
        app.print_help()
        app.print_examples()
        app.log.fatal("Bad config encountered during initialization:")
        app.log.fatal(str(e))
        app.log.debug("Config at the time: %s", app.config)
        app.exit(1)

class ApplicationError(Exception):
    pass

class Application(SingletonConfigurable):
    """A singleton application with full configuration support."""

    # The name of the application, will usually match the name of the command
    # line application
    name = Unicode(u'application')

    # The description of the application that is printed at the beginning
    # of the help.
    description = Unicode(u'This is an application.')
    # default section descriptions
    option_description = Unicode(option_description)
    keyvalue_description = Unicode(keyvalue_description)
    subcommand_description = Unicode(subcommand_description)

    # The usage and example string that goes at the end of the help string.
    examples = Unicode()

    # A sequence of Configurable subclasses whose config=True attributes will
    # be exposed at the command line.
    classes = List([])

    # The version string of this application.
    version = Unicode(u'0.0')

    # The log level for the application
    log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
                    default_value=logging.WARN,
                    config=True,
                    help="Set the log level by value or name.")
    def _log_level_changed(self, name, old, new):
        """Adjust the log level when log_level is set."""
        if isinstance(new, basestring):
            new = getattr(logging, new)
            self.log_level = new
        self.log.setLevel(new)

    log_format = Unicode("[%(name)s] %(message)s", config=True,
        help="The Logging format template",
    )
    log = Instance(logging.Logger)
    def _log_default(self):
        """Start logging for this application.

        The default is to log to stdout using a StreaHandler. The log level
        starts at loggin.WARN, but this can be adjusted by setting the
        ``log_level`` attribute.
        """
        log = logging.getLogger(self.__class__.__name__)
        log.setLevel(self.log_level)
        if sys.executable.endswith('pythonw.exe'):
            # this should really go to a file, but file-logging is only
            # hooked up in parallel applications
            _log_handler = logging.StreamHandler(open(os.devnull, 'w'))
        else:
            _log_handler = logging.StreamHandler()
        _log_formatter = logging.Formatter(self.log_format)
        _log_handler.setFormatter(_log_formatter)
        log.addHandler(_log_handler)
        return log

    # the alias map for configurables
    aliases = Dict({'log-level' : 'Application.log_level'})

    # flags for loading Configurables or store_const style flags
    # flags are loaded from this dict by '--key' flags
    # this must be a dict of two-tuples, the first element being the Config/dict
    # and the second being the help string for the flag
    flags = Dict()
    def _flags_changed(self, name, old, new):
        """ensure flags dict is valid"""
        for key,value in new.iteritems():
            assert len(value) == 2, "Bad flag: %r:%s"%(key,value)
            assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value)
            assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value)

    # subcommands for launching other applications
    # if this is not empty, this will be a parent Application
    # this must be a dict of two-tuples,
    # the first element being the application class/import string
    # and the second being the help string for the subcommand
    subcommands = Dict()
    # parse_command_line will initialize a subapp, if requested
    subapp = Instance('IPython.config.application.Application', allow_none=True)

    # extra command-line arguments that don't set config values
    extra_args = List(Unicode)

    def __init__(self, **kwargs):
        SingletonConfigurable.__init__(self, **kwargs)
        # Ensure my class is in self.classes, so my attributes appear in command line
        # options and config files.
        if self.__class__ not in self.classes:
            self.classes.insert(0, self.__class__)

    def _config_changed(self, name, old, new):
        SingletonConfigurable._config_changed(self, name, old, new)
        self.log.debug('Config changed:')
        self.log.debug(repr(new))

    @catch_config_error
    def initialize(self, argv=None):
        """Do the basic steps to configure me.

        Override in subclasses.
        """
        self.parse_command_line(argv)

    def start(self):
        """Start the app mainloop.

        Override in subclasses.
        """
        if self.subapp is not None:
            return self.subapp.start()

    def print_alias_help(self):
        """Print the alias part of the help."""
        if not self.aliases:
            return

        lines = []
        classdict = {}
        for cls in self.classes:
            # include all parents (up to, but excluding Configurable) in available names
            for c in cls.mro()[:-3]:
                classdict[c.__name__] = c

        for alias, longname in self.aliases.iteritems():
            classname, traitname = longname.split('.',1)
            cls = classdict[classname]

            trait = cls.class_traits(config=True)[traitname]
            help = cls.class_get_trait_help(trait).splitlines()
            # reformat first line
            help[0] = help[0].replace(longname, alias) + ' (%s)'%longname
            if len(alias) == 1:
                help[0] = help[0].replace('--%s='%alias, '-%s '%alias)
            lines.extend(help)
        # lines.append('')
        print os.linesep.join(lines)

    def print_flag_help(self):
        """Print the flag part of the help."""
        if not self.flags:
            return

        lines = []
        for m, (cfg,help) in self.flags.iteritems():
            prefix = '--' if len(m) > 1 else '-'
            lines.append(prefix+m)
            lines.append(indent(dedent(help.strip())))
        # lines.append('')
        print os.linesep.join(lines)

    def print_options(self):
        if not self.flags and not self.aliases:
            return
        lines = ['Options']
        lines.append('-'*len(lines[0]))
        lines.append('')
        for p in wrap_paragraphs(self.option_description):
            lines.append(p)
            lines.append('')
        print os.linesep.join(lines)
        self.print_flag_help()
        self.print_alias_help()
        print

    def print_subcommands(self):
        """Print the subcommand part of the help."""
        if not self.subcommands:
            return

        lines = ["Subcommands"]
        lines.append('-'*len(lines[0]))
        lines.append('')
        for p in wrap_paragraphs(self.subcommand_description):
            lines.append(p)
            lines.append('')
        for subc, (cls, help) in self.subcommands.iteritems():
            lines.append(subc)
            if help:
                lines.append(indent(dedent(help.strip())))
        lines.append('')
        print os.linesep.join(lines)

    def print_help(self, classes=False):
        """Print the help for each Configurable class in self.classes.

        If classes=False (the default), only flags and aliases are printed.
        """
        self.print_subcommands()
        self.print_options()

        if classes:
            if self.classes:
                print "Class parameters"
                print "----------------"
                print
                for p in wrap_paragraphs(self.keyvalue_description):
                    print p
                    print

            for cls in self.classes:
                cls.class_print_help()
                print
        else:
            print "To see all available configurables, use `--help-all`"
            print

    def print_description(self):
        """Print the application description."""
        for p in wrap_paragraphs(self.description):
            print p
            print

    def print_examples(self):
        """Print usage and examples.

        This usage string goes at the end of the command line help string
        and should contain examples of the application's usage.
        """
        if self.examples:
            print "Examples"
            print "--------"
            print
            print indent(dedent(self.examples.strip()))
            print

    def print_version(self):
        """Print the version string."""
        print self.version

    def update_config(self, config):
        """Fire the traits events when the config is updated."""
        # Save a copy of the current config.
        newconfig = deepcopy(self.config)
        # Merge the new config into the current one.
        newconfig._merge(config)
        # Save the combined config as self.config, which triggers the traits
        # events.
        self.config = newconfig

    @catch_config_error
    def initialize_subcommand(self, subc, argv=None):
        """Initialize a subcommand with argv."""
        subapp,help = self.subcommands.get(subc)

        if isinstance(subapp, basestring):
            subapp = import_item(subapp)

        # clear existing instances
        self.__class__.clear_instance()
        # instantiate
        self.subapp = subapp.instance()
        # and initialize subapp
        self.subapp.initialize(argv)

    def flatten_flags(self):
        """flatten flags and aliases, so cl-args override as expected.

        This prevents issues such as an alias pointing to InteractiveShell,
        but a config file setting the same trait in TerminalInteraciveShell
        getting inappropriate priority over the command-line arg.

        Only aliases with exactly one descendent in the class list
        will be promoted.

        """
        # build a tree of classes in our list that inherit from a particular
        # it will be a dict by parent classname of classes in our list
        # that are descendents
        mro_tree = defaultdict(list)
        for cls in self.classes:
            clsname = cls.__name__
            for parent in cls.mro()[1:-3]:
                # exclude cls itself and Configurable,HasTraits,object
                mro_tree[parent.__name__].append(clsname)
        # flatten aliases, which have the form:
        # { 'alias' : 'Class.trait' }
        aliases = {}
        for alias, cls_trait in self.aliases.iteritems():
            cls,trait = cls_trait.split('.',1)
            children = mro_tree[cls]
            if len(children) == 1:
                # exactly one descendent, promote alias
                cls = children[0]
            aliases[alias] = '.'.join([cls,trait])

        # flatten flags, which are of the form:
        # { 'key' : ({'Cls' : {'trait' : value}}, 'help')}
        flags = {}
        for key, (flagdict, help) in self.flags.iteritems():
            newflag = {}
            for cls, subdict in flagdict.iteritems():
                children = mro_tree[cls]
                # exactly one descendent, promote flag section
                if len(children) == 1:
                    cls = children[0]
                newflag[cls] = subdict
            flags[key] = (newflag, help)
        return flags, aliases

    @catch_config_error
    def parse_command_line(self, argv=None):
        """Parse the command line arguments."""
        argv = sys.argv[1:] if argv is None else argv

        if argv and argv[0] == 'help':
            # turn `ipython help notebook` into `ipython notebook -h`
            argv = argv[1:] + ['-h']

        if self.subcommands and len(argv) > 0:
            # we have subcommands, and one may have been specified
            subc, subargv = argv[0], argv[1:]
            if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands:
                # it's a subcommand, and *not* a flag or class parameter
                return self.initialize_subcommand(subc, subargv)

        if '-h' in argv or '--help' in argv or '--help-all' in argv:
            self.print_description()
            self.print_help('--help-all' in argv)
            self.print_examples()
            self.exit(0)

        if '--version' in argv or '-V' in argv:
            self.print_version()
            self.exit(0)

        # flatten flags&aliases, so cl-args get appropriate priority:
        flags,aliases = self.flatten_flags()

        loader = KVArgParseConfigLoader(argv=argv, aliases=aliases,
                                        flags=flags)
        config = loader.load_config()
        self.update_config(config)
        # store unparsed args in extra_args
        self.extra_args = loader.extra_args

    @catch_config_error
    def load_config_file(self, filename, path=None):
        """Load a .py based config file by filename and path."""
        loader = PyFileConfigLoader(filename, path=path)
        try:
            config = loader.load_config()
        except ConfigFileNotFound:
            # problem finding the file, raise
            raise
        except Exception:
            # try to get the full filename, but it will be empty in the
            # unlikely event that the error raised before filefind finished
            filename = loader.full_filename or filename
            # problem while running the file
            self.log.error("Exception while loading config file %s",
                            filename, exc_info=True)
        else:
            self.log.debug("Loaded config file: %s", loader.full_filename)
            self.update_config(config)

    def generate_config_file(self):
        """generate default config file from Configurables"""
        lines = ["# Configuration file for %s."%self.name]
        lines.append('')
        lines.append('c = get_config()')
        lines.append('')
        for cls in self.classes:
            lines.append(cls.class_config_section())
        return '\n'.join(lines)

    def exit(self, exit_status=0):
        self.log.debug("Exiting application: %s" % self.name)
        sys.exit(exit_status)

#-----------------------------------------------------------------------------
# utility functions, for convenience
#-----------------------------------------------------------------------------

def boolean_flag(name, configurable, set_help='', unset_help=''):
    """Helper for building basic --trait, --no-trait flags.

    Parameters
    ----------

    name : str
        The name of the flag.
    configurable : str
        The 'Class.trait' string of the trait to be set/unset with the flag
    set_help : unicode
        help string for --name flag
    unset_help : unicode
        help string for --no-name flag

    Returns
    -------

    cfg : dict
        A dict with two keys: 'name', and 'no-name', for setting and unsetting
        the trait, respectively.
    """
    # default helpstrings
    set_help = set_help or "set %s=True"%configurable
    unset_help = unset_help or "set %s=False"%configurable

    cls,trait = configurable.split('.')

    setter = {cls : {trait : True}}
    unsetter = {cls : {trait : False}}
    return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)}


"""An example of how to use IPython for plotting remote parallel data

The two files plotting_frontend.py and plotting_backend.py go together.

This file (plotting_backend.py) performs the actual computation.  For this
example, the computation just generates a set of random numbers that
look like a distribution of particles with 2D position (x,y) and
momentum (px,py).  In a real situation, this file would do some time
consuming and complicated calculation, and could possibly make calls
to MPI.

One important feature is that this script can also be run standalone without
IPython.  This is nice as it allows it to be run in more traditional
settings where IPython isn't being used.

When used with IPython.parallel, this code is run on the engines.  Because this
code doesn't make any plots, the engines don't have to have any plotting
packages installed.
"""
from __future__ import print_function

# Imports
import numpy as N
import time
import random

# Functions
def compute_particles(number):
    x = N.random.standard_normal(number)
    y = N.random.standard_normal(number)
    px = N.random.standard_normal(number)
    py = N.random.standard_normal(number)
    return x, y, px, py

def downsample(array, k):
    """Choose k random elements of array."""
    length = array.shape[0]
    indices = random.sample(xrange(length), k)
    return array[indices]

# Parameters of the run
number = 100000
d_number = 1000

# The actual run

time.sleep(0) # Pretend it took a while
x, y, px, py = compute_particles(number)
# Now downsample the data
downx = downsample(x, d_number)
downy = downsample(x, d_number)
downpx = downsample(px, d_number)
downpy = downsample(py, d_number)

print("downx: ", downx[:10])
print("downy: ", downy[:10])
print("downpx: ", downpx[:10])
print("downpy: ", downpy[:10])


"""Control of and utilities for debugging."""

import os

# When debugging, it can be helpful to force some options, especially when
# debugging the configuration mechanisms you usually use to control debugging!
# This is a list of forced debugging options.
FORCED_DEBUG = []

class DebugControl(object):
    """Control and output for debugging."""

    def __init__(self, options, output):
        """Configure the options and output file for debugging."""
        self.options = options
        self.output = output

    def should(self, option):
        """Decide whether to output debug information in category `option`."""
        return (option in self.options or option in FORCED_DEBUG)

    def write(self, msg):
        """Write a line of debug output."""
        if self.should('pid'):
            msg = "pid %5d: %s" % (os.getpid(), msg)
        self.output.write(msg+"\n")
        self.output.flush()

    def write_formatted_info(self, info):
        """Write a sequence of (label,data) pairs nicely."""
        for line in info_formatter(info):
            self.write(" %s" % line)

def info_formatter(info):
    """Produce a sequence of formatted lines from info.

    `info` is a sequence of pairs (label, data).  The produced lines are
    nicely formatted, ready to print.

    """
    label_len = max([len(l) for l, _d in info])
    for label, data in info:
        if data == []:
            data = "-none-"
        if isinstance(data, (list, tuple)):
            prefix = "%*s:" % (label_len, label)
            for e in data:
                yield "%*s %s" % (label_len+1, prefix, e)
                prefix = ""
        else:
            yield "%*s: %s" % (label_len, label, data)


# encoding: utf-8
"""
A base class for objects that are configurable.

Authors:

* Brian Granger
* Fernando Perez
* Min RK
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import datetime
from copy import deepcopy

from loader import Config
from IPython.utils.traitlets import HasTraits, Instance
from IPython.utils.text import indent, wrap_paragraphs

#-----------------------------------------------------------------------------
# Helper classes for Configurables
#-----------------------------------------------------------------------------

class ConfigurableError(Exception):
    pass

class MultipleInstanceError(ConfigurableError):
    pass

#-----------------------------------------------------------------------------
# Configurable implementation
#-----------------------------------------------------------------------------

class Configurable(HasTraits):

    config = Instance(Config,(),{})
    created = None

    def __init__(self, **kwargs):
        """Create a configurable given a config config.

        Parameters
        ----------
        config : Config
            If this is empty, default values are used. If config is a
            :class:`Config` instance, it will be used to configure the
            instance.

        Notes
        -----
        Subclasses of Configurable must call the :meth:`__init__` method of
        :class:`Configurable` *before* doing anything else and using
        :func:`super`::

            class MyConfigurable(Configurable):
                def __init__(self, config=None):
                    super(MyConfigurable, self).__init__(config)
                    # Then any other code you need to finish initialization.

        This ensures that instances will be configured properly.
        """
        config = kwargs.pop('config', None)
        if config is not None:
            # We used to deepcopy, but for now we are trying to just save
            # by reference.  This *could* have side effects as all components
            # will share config. In fact, I did find such a side effect in
            # _config_changed below. If a config attribute value was a mutable type
            # all instances of a component were getting the same copy, effectively
            # making that a class attribute.
            # self.config = deepcopy(config)
            self.config = config
        # This should go second so individual keyword arguments override
        # the values in config.
        super(Configurable, self).__init__(**kwargs)
        self.created = datetime.datetime.now()

    #-------------------------------------------------------------------------
    # Static trait notifiations
    #-------------------------------------------------------------------------

    def _config_changed(self, name, old, new):
        """Update all the class traits having ``config=True`` as metadata.

        For any class trait with a ``config`` metadata attribute that is
        ``True``, we update the trait with the value of the corresponding
        config entry.
        """
        # Get all traits with a config metadata entry that is True
        traits = self.traits(config=True)

        # We auto-load config section for this class as well as any parent
        # classes that are Configurable subclasses.  This starts with Configurable
        # and works down the mro loading the config for each section.
        section_names = [cls.__name__ for cls in \
            reversed(self.__class__.__mro__) if
            issubclass(cls, Configurable) and issubclass(self.__class__, cls)]

        for sname in section_names:
            # Don't do a blind getattr as that would cause the config to
            # dynamically create the section with name self.__class__.__name__.
            if new._has_section(sname):
                my_config = new[sname]
                for k, v in traits.iteritems():
                    # Don't allow traitlets with config=True to start with
                    # uppercase.  Otherwise, they are confused with Config
                    # subsections.  But, developers shouldn't have uppercase
                    # attributes anyways! (PEP 6)
                    if k[0].upper()==k[0] and not k.startswith('_'):
                        raise ConfigurableError('Configurable traitlets with '
                        'config=True must start with a lowercase so they are '
                        'not confused with Config subsections: %s.%s' % \
                        (self.__class__.__name__, k))
                    try:
                        # Here we grab the value from the config
                        # If k has the naming convention of a config
                        # section, it will be auto created.
                        config_value = my_config[k]
                    except KeyError:
                        pass
                    else:
                        # print "Setting %s.%s from %s.%s=%r" % \
                        #     (self.__class__.__name__,k,sname,k,config_value)
                        # We have to do a deepcopy here if we don't deepcopy the entire
                        # config object. If we don't, a mutable config_value will be
                        # shared by all instances, effectively making it a class attribute.
                        setattr(self, k, deepcopy(config_value))

    def update_config(self, config):
        """Fire the traits events when the config is updated."""
        # Save a copy of the current config.
        newconfig = deepcopy(self.config)
        # Merge the new config into the current one.
        newconfig._merge(config)
        # Save the combined config as self.config, which triggers the traits
        # events.
        self.config = newconfig

    @classmethod
    def class_get_help(cls, inst=None):
        """Get the help string for this class in ReST format.

        If `inst` is given, it's current trait values will be used in place of
        class defaults.
        """
        assert inst is None or isinstance(inst, cls)
        cls_traits = cls.class_traits(config=True)
        final_help = []
        final_help.append(u'%s options' % cls.__name__)
        final_help.append(len(final_help[0])*u'-')
        for k,v in sorted(cls.class_traits(config=True).iteritems()):
            help = cls.class_get_trait_help(v, inst)
            final_help.append(help)
        return '\n'.join(final_help)

    @classmethod
    def class_get_trait_help(cls, trait, inst=None):
        """Get the help string for a single trait.

        If `inst` is given, it's current trait values will be used in place of
        the class default.
        """
        assert inst is None or isinstance(inst, cls)
        lines = []
        header = "--%s.%s=<%s>" % (cls.__name__, trait.name, trait.__class__.__name__)
        lines.append(header)
        if inst is not None:
            lines.append(indent('Current: %r' % getattr(inst, trait.name), 4))
        else:
            try:
                dvr = repr(trait.get_default_value())
            except Exception:
                dvr = None # ignore defaults we can't construct
            if dvr is not None:
                if len(dvr) > 64:
                    dvr = dvr[:61]+'...'
                lines.append(indent('Default: %s' % dvr, 4))
        if 'Enum' in trait.__class__.__name__:
            # include Enum choices
            lines.append(indent('Choices: %r' % (trait.values,)))

        help = trait.get_metadata('help')
        if help is not None:
            help = '\n'.join(wrap_paragraphs(help, 76))
            lines.append(indent(help, 4))
        return '\n'.join(lines)

    @classmethod
    def class_print_help(cls, inst=None):
        """Get the help string for a single trait and print it."""
        print cls.class_get_help(inst)

    @classmethod
    def class_config_section(cls):
        """Get the config class config section"""
        def c(s):
            """return a commented, wrapped block."""
            s = '\n\n'.join(wrap_paragraphs(s, 78))

            return '# ' + s.replace('\n', '\n# ')

        # section header
        breaker = '#' + '-'*78
        s = "# %s configuration"%cls.__name__
        lines = [breaker, s, breaker, '']
        # get the description trait
        desc = cls.class_traits().get('description')
        if desc:
            desc = desc.default_value
        else:
            # no description trait, use __doc__
            desc = getattr(cls, '__doc__', '')
        if desc:
            lines.append(c(desc))
            lines.append('')

        parents = []
        for parent in cls.mro():
            # only include parents that are not base classes
            # and are not the class itself
            # and have some configurable traits to inherit
            if parent is not cls and issubclass(parent, Configurable) and \
                    parent.class_traits(config=True):
                parents.append(parent)

        if parents:
            pstr = ', '.join([ p.__name__ for p in parents ])
            lines.append(c('%s will inherit config from: %s'%(cls.__name__, pstr)))
            lines.append('')

        for name,trait in cls.class_traits(config=True).iteritems():
            help = trait.get_metadata('help') or ''
            lines.append(c(help))
            lines.append('# c.%s.%s = %r'%(cls.__name__, name, trait.get_default_value()))
            lines.append('')
        return '\n'.join(lines)

class SingletonConfigurable(Configurable):
    """A configurable that only allows one instance.

    This class is for classes that should only have one instance of itself
    or *any* subclass. To create and retrieve such a class use the
    :meth:`SingletonConfigurable.instance` method.
    """

    _instance = None

    @classmethod
    def _walk_mro(cls):
        """Walk the cls.mro() for parent classes that are also singletons

        For use in instance()
        """

        for subclass in cls.mro():
            if issubclass(cls, subclass) and \
                    issubclass(subclass, SingletonConfigurable) and \
                    subclass != SingletonConfigurable:
                yield subclass

    @classmethod
    def clear_instance(cls):
        """unset _instance for this class and singleton parents.
        """
        if not cls.initialized():
            return
        for subclass in cls._walk_mro():
            if isinstance(subclass._instance, cls):
                # only clear instances that are instances
                # of the calling class
                subclass._instance = None

    @classmethod
    def instance(cls, *args, **kwargs):
        """Returns a global instance of this class.

        This method create a new instance if none have previously been created
        and returns a previously created instance is one already exists.

        The arguments and keyword arguments passed to this method are passed
        on to the :meth:`__init__` method of the class upon instantiation.

        Examples
        --------

        Create a singleton class using instance, and retrieve it::

            >>> from IPython.config.configurable import SingletonConfigurable
            >>> class Foo(SingletonConfigurable): pass
            >>> foo = Foo.instance()
            >>> foo == Foo.instance()
            True

        Create a subclass that is retrived using the base class instance::

            >>> class Bar(SingletonConfigurable): pass
            >>> class Bam(Bar): pass
            >>> bam = Bam.instance()
            >>> bam == Bar.instance()
            True
        """
        # Create and save the instance
        if cls._instance is None:
            inst = cls(*args, **kwargs)
            # Now make sure that the instance will also be returned by
            # parent classes' _instance attribute.
            for subclass in cls._walk_mro():
                subclass._instance = inst

        if isinstance(cls._instance, cls):
            return cls._instance
        else:
            raise MultipleInstanceError(
                'Multiple incompatible subclass instances of '
                '%s are being created.' % cls.__name__
            )

    @classmethod
    def initialized(cls):
        """Has an instance been created?"""
        return hasattr(cls, "_instance") and cls._instance is not None

class LoggingConfigurable(Configurable):
    """A parent class for Configurables that log.

    Subclasses have a log trait, and the default behavior
    is to get the logger from the currently running Application
    via Application.instance().log.
    """

    log = Instance('logging.Logger')
    def _log_default(self):
        from IPython.config.application import Application
        return Application.instance().log


"""
This plugin provides assert introspection. When the plugin is enabled
and a test failure occurs, the traceback is displayed with extra context
around the line in which the exception was raised. Simple variable
substitution is also performed in the context output to provide more
debugging information.
"""

from nose.plugins import Plugin
from nose.inspector import inspect_traceback

class FailureDetail(Plugin):
    """
    Plugin that provides extra information in tracebacks of test failures.
    """
    score = 600 # before capture

    def options(self, parser, env):
        """Register commmandline options.
        """
        parser.add_option(
            "-d", "--detailed-errors", "--failure-detail",
            action="store_true",
            default=env.get('NOSE_DETAILED_ERRORS'),
            dest="detailedErrors", help="Add detail to error"
            " output by attempting to evaluate failed"
            " asserts [NOSE_DETAILED_ERRORS]")

    def configure(self, options, conf):
        """Configure plugin.
        """
        if not self.can_configure:
            return
        self.enabled = options.detailedErrors
        self.conf = conf

    def formatFailure(self, test, err):
        """Add detail from traceback inspection to error message of a failure.
        """
        ec, ev, tb = err
        tbinfo = inspect_traceback(tb)
        test.tbinfo = tbinfo
        return (ec, '\n'.join([str(ev), tbinfo]), tb)


# encoding: utf-8
"""sys.excepthook for IPython itself, leaves a detailed report on disk.

Authors:

* Fernando Perez
* Brian E. Granger
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2001-2007 Fernando Perez. <fperez@colorado.edu>
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import os
import sys
import traceback
from pprint import pformat

from IPython.core import ultratb
from IPython.core.release import author_email
from IPython.utils.sysinfo import sys_info

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

# Template for the user message.
_default_message_template = """\
Oops, {app_name} crashed. We do our best to make it stable, but...

A crash report was automatically generated with the following information:
  - A verbatim copy of the crash traceback.
  - A copy of your input history during this session.
  - Data on your current {app_name} configuration.

It was left in the file named:
\t'{crash_report_fname}'
If you can email this file to the developers, the information in it will help
them in understanding and correcting the problem.

You can mail it to: {contact_name} at {contact_email}
with the subject '{app_name} Crash Report'.

If you want to do it now, the following command will work (under Unix):
mail -s '{app_name} Crash Report' {contact_email} < {crash_report_fname}

To ensure accurate tracking of this issue, please file a report about it at:
{bug_tracker}
"""

_lite_message_template = """
If you suspect this is an IPython bug, please report it at:
    https://github.com/ipython/ipython/issues
or send an email to the mailing list at {email}

You can print a more detailed traceback right now with "%tb", or use "%debug"
to interactively debug it.

Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
    {config}Application.verbose_crash=True
"""

class CrashHandler(object):
    """Customizable crash handlers for IPython applications.

    Instances of this class provide a :meth:`__call__` method which can be
    used as a ``sys.excepthook``.  The :meth:`__call__` signature is::

        def __call__(self, etype, evalue, etb)
    """

    message_template = _default_message_template
    section_sep = '\n\n'+'*'*75+'\n\n'

    def __init__(self, app, contact_name=None, contact_email=None,
                 bug_tracker=None, show_crash_traceback=True, call_pdb=False):
        """Create a new crash handler

        Parameters
        ----------
        app :  Application
            A running :class:`Application` instance, which will be queried at
            crash time for internal information.

        contact_name : str
            A string with the name of the person to contact.

        contact_email : str
            A string with the email address of the contact.

        bug_tracker : str
            A string with the URL for your project's bug tracker.

        show_crash_traceback : bool
            If false, don't print the crash traceback on stderr, only generate
            the on-disk report

        Non-argument instance attributes:

        These instances contain some non-argument attributes which allow for
        further customization of the crash handler's behavior. Please see the
        source for further details.
        """
        self.crash_report_fname = "Crash_report_%s.txt" % app.name
        self.app = app
        self.call_pdb = call_pdb
        #self.call_pdb = True # dbg
        self.show_crash_traceback = show_crash_traceback
        self.info = dict(app_name = app.name,
                    contact_name = contact_name,
                    contact_email = contact_email,
                    bug_tracker = bug_tracker,
                    crash_report_fname = self.crash_report_fname)

    def __call__(self, etype, evalue, etb):
        """Handle an exception, call for compatible with sys.excepthook"""

        # do not allow the crash handler to be called twice without reinstalling it
        # this prevents unlikely errors in the crash handling from entering an
        # infinite loop.
        sys.excepthook = sys.__excepthook__

        # Report tracebacks shouldn't use color in general (safer for users)
        color_scheme = 'NoColor'

        # Use this ONLY for developer debugging (keep commented out for release)
        #color_scheme = 'Linux'   # dbg
        try:
            rptdir = self.app.ipython_dir
        except:
            rptdir = os.getcwdu()
        if rptdir is None or not os.path.isdir(rptdir):
            rptdir = os.getcwdu()
        report_name = os.path.join(rptdir,self.crash_report_fname)
        # write the report filename into the instance dict so it can get
        # properly expanded out in the user message template
        self.crash_report_fname = report_name
        self.info['crash_report_fname'] = report_name
        TBhandler = ultratb.VerboseTB(
            color_scheme=color_scheme,
            long_header=1,
            call_pdb=self.call_pdb,
        )
        if self.call_pdb:
            TBhandler(etype,evalue,etb)
            return
        else:
            traceback = TBhandler.text(etype,evalue,etb,context=31)

        # print traceback to screen
        if self.show_crash_traceback:
            print >> sys.stderr, traceback

        # and generate a complete report on disk
        try:
            report = open(report_name,'w')
        except:
            print >> sys.stderr, 'Could not create crash report on disk.'
            return

        # Inform user on stderr of what happened
        print >> sys.stderr, '\n'+'*'*70+'\n'
        print >> sys.stderr, self.message_template.format(**self.info)

        # Construct report on disk
        report.write(self.make_report(traceback))
        report.close()
        raw_input("Hit <Enter> to quit (your terminal may close):")

    def make_report(self,traceback):
        """Return a string containing a crash report."""

        sec_sep = self.section_sep

        report = ['*'*75+'\n\n'+'IPython post-mortem report\n\n']
        rpt_add = report.append
        rpt_add(sys_info())

        try:
            config = pformat(self.app.config)
            rpt_add(sec_sep)
            rpt_add('Application name: %s\n\n' % self.app_name)
            rpt_add('Current user configuration structure:\n\n')
            rpt_add(config)
        except:
            pass
        rpt_add(sec_sep+'Crash traceback:\n\n' + traceback)

        return ''.join(report)

def crash_handler_lite(etype, evalue, tb):
    """a light excepthook, adding a small message to the usual traceback"""
    traceback.print_exception(etype, evalue, tb)

    from IPython.core.interactiveshell import InteractiveShell
    if InteractiveShell.initialized():
        # we are in a Shell environment, give %magic example
        config = "%config "
    else:
        # we are not in a shell, show generic config
        config = "c."
    print >> sys.stderr, _lite_message_template.format(email=author_email, config=config)


from django import template

register = template.Library()

class SetVarNode(template.Node):

    def __init__(self, var_name, var_value):
        self.var_name = var_name
        self.var_value = var_value

    def render(self, context):
        try:
            value = template.Variable(self.var_value).resolve(context)
        except template.VariableDoesNotExist:
            value = ""
        context[self.var_name] = value
        return ''

@register.tag
def setvar(parser, token):
    """ {% setvar <var_name> to <var_value> %} """

    try:
        setvar, var_name, to_, var_value = token.split_contents()
    except ValueError:
        raise template.TemplateSyntaxError('Invalid arguments for %r' % token.split_contents()[0])

    return SetVarNode(var_name, var_value)

@register.filter()
def string2var(string):
    """
    {% with user.full_name|truncatechars:16|string2var as truncated_name %}
        {{ truncated_name|highlight:search_kw }} # need to truncated and then highlight
    {% endwith %}
    """
    return string


""" Defines a KernelManager that provides signals and slots.
"""

# System library imports.
from IPython.external.qt import QtCore

# IPython imports.
from IPython.utils.traitlets import Type
from IPython.zmq.kernelmanager import KernelManager, SubSocketChannel, \
    ShellSocketChannel, StdInSocketChannel, HBSocketChannel
from util import MetaQObjectHasTraits, SuperQObject

class SocketChannelQObject(SuperQObject):

    # Emitted when the channel is started.
    started = QtCore.Signal()

    # Emitted when the channel is stopped.
    stopped = QtCore.Signal()

    #---------------------------------------------------------------------------
    # 'ZMQSocketChannel' interface
    #---------------------------------------------------------------------------

    def start(self):
        """ Reimplemented to emit signal.
        """
        super(SocketChannelQObject, self).start()
        self.started.emit()

    def stop(self):
        """ Reimplemented to emit signal.
        """
        super(SocketChannelQObject, self).stop()
        self.stopped.emit()

class QtShellSocketChannel(SocketChannelQObject, ShellSocketChannel):

    # Emitted when any message is received.
    message_received = QtCore.Signal(object)

    # Emitted when a reply has been received for the corresponding request
    # type.
    execute_reply = QtCore.Signal(object)
    complete_reply = QtCore.Signal(object)
    object_info_reply = QtCore.Signal(object)
    history_reply = QtCore.Signal(object)

    # Emitted when the first reply comes back.
    first_reply = QtCore.Signal()

    # Used by the first_reply signal logic to determine if a reply is the
    # first.
    _handlers_called = False

    #---------------------------------------------------------------------------
    # 'ShellSocketChannel' interface
    #---------------------------------------------------------------------------

    def call_handlers(self, msg):
        """ Reimplemented to emit signals instead of making callbacks.
        """
        # Emit the generic signal.
        self.message_received.emit(msg)

        # Emit signals for specialized message types.
        msg_type = msg['header']['msg_type']
        signal = getattr(self, msg_type, None)
        if signal:
            signal.emit(msg)

        if not self._handlers_called:
            self.first_reply.emit()
            self._handlers_called = True

    #---------------------------------------------------------------------------
    # 'QtShellSocketChannel' interface
    #---------------------------------------------------------------------------

    def reset_first_reply(self):
        """ Reset the first_reply signal to fire again on the next reply.
        """
        self._handlers_called = False

class QtSubSocketChannel(SocketChannelQObject, SubSocketChannel):

    # Emitted when any message is received.
    message_received = QtCore.Signal(object)

    # Emitted when a message of type 'stream' is received.
    stream_received = QtCore.Signal(object)

    # Emitted when a message of type 'pyin' is received.
    pyin_received = QtCore.Signal(object)

    # Emitted when a message of type 'pyout' is received.
    pyout_received = QtCore.Signal(object)

    # Emitted when a message of type 'pyerr' is received.
    pyerr_received = QtCore.Signal(object)

    # Emitted when a message of type 'display_data' is received
    display_data_received = QtCore.Signal(object)

    # Emitted when a crash report message is received from the kernel's
    # last-resort sys.excepthook.
    crash_received = QtCore.Signal(object)

    # Emitted when a shutdown is noticed.
    shutdown_reply_received = QtCore.Signal(object)

    #---------------------------------------------------------------------------
    # 'SubSocketChannel' interface
    #---------------------------------------------------------------------------

    def call_handlers(self, msg):
        """ Reimplemented to emit signals instead of making callbacks.
        """
        # Emit the generic signal.
        self.message_received.emit(msg)
        # Emit signals for specialized message types.
        msg_type = msg['header']['msg_type']
        signal = getattr(self, msg_type + '_received', None)
        if signal:
            signal.emit(msg)
        elif msg_type in ('stdout', 'stderr'):
            self.stream_received.emit(msg)

    def flush(self):
        """ Reimplemented to ensure that signals are dispatched immediately.
        """
        super(QtSubSocketChannel, self).flush()
        QtCore.QCoreApplication.instance().processEvents()

class QtStdInSocketChannel(SocketChannelQObject, StdInSocketChannel):

    # Emitted when any message is received.
    message_received = QtCore.Signal(object)

    # Emitted when an input request is received.
    input_requested = QtCore.Signal(object)

    #---------------------------------------------------------------------------
    # 'StdInSocketChannel' interface
    #---------------------------------------------------------------------------

    def call_handlers(self, msg):
        """ Reimplemented to emit signals instead of making callbacks.
        """
        # Emit the generic signal.
        self.message_received.emit(msg)

        # Emit signals for specialized message types.
        msg_type = msg['header']['msg_type']
        if msg_type == 'input_request':
            self.input_requested.emit(msg)

class QtHBSocketChannel(SocketChannelQObject, HBSocketChannel):

    # Emitted when the kernel has died.
    kernel_died = QtCore.Signal(object)

    #---------------------------------------------------------------------------
    # 'HBSocketChannel' interface
    #---------------------------------------------------------------------------

    def call_handlers(self, since_last_heartbeat):
        """ Reimplemented to emit signals instead of making callbacks.
        """
        # Emit the generic signal.
        self.kernel_died.emit(since_last_heartbeat)

class QtKernelManager(KernelManager, SuperQObject):
    """ A KernelManager that provides signals and slots.
    """

    __metaclass__ = MetaQObjectHasTraits

    # Emitted when the kernel manager has started listening.
    started_kernel = QtCore.Signal()

    # Emitted when the kernel manager has started listening.
    started_channels = QtCore.Signal()

    # Emitted when the kernel manager has stopped listening.
    stopped_channels = QtCore.Signal()

    # Use Qt-specific channel classes that emit signals.
    sub_channel_class = Type(QtSubSocketChannel)
    shell_channel_class = Type(QtShellSocketChannel)
    stdin_channel_class = Type(QtStdInSocketChannel)
    hb_channel_class = Type(QtHBSocketChannel)

    #---------------------------------------------------------------------------
    # 'KernelManager' interface
    #---------------------------------------------------------------------------

    #------ Kernel process management ------------------------------------------

    def start_kernel(self, *args, **kw):
        """ Reimplemented for proper heartbeat management.
        """
        if self._shell_channel is not None:
            self._shell_channel.reset_first_reply()
        super(QtKernelManager, self).start_kernel(*args, **kw)
        self.started_kernel.emit()

    #------ Channel management -------------------------------------------------

    def start_channels(self, *args, **kw):
        """ Reimplemented to emit signal.
        """
        super(QtKernelManager, self).start_channels(*args, **kw)
        self.started_channels.emit()

    def stop_channels(self):
        """ Reimplemented to emit signal.
        """
        super(QtKernelManager, self).stop_channels()
        self.stopped_channels.emit()

    @property
    def shell_channel(self):
        """ Reimplemented for proper heartbeat management.
        """
        if self._shell_channel is None:
            self._shell_channel = super(QtKernelManager, self).shell_channel
            self._shell_channel.first_reply.connect(self._first_reply)
        return self._shell_channel

    #---------------------------------------------------------------------------
    # Protected interface
    #---------------------------------------------------------------------------

    def _first_reply(self):
        """ Unpauses the heartbeat channel when the first reply is received on
            the execute channel. Note that this will *not* start the heartbeat
            channel if it is not already running!
        """
        if self._hb_channel is not None:
            self._hb_channel.unpause()


"""Base classes and utilities for readers and writers.

Authors:

* Brian Granger
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from base64 import encodestring, decodestring
import pprint

from IPython.utils import py3compat

str_to_bytes = py3compat.str_to_bytes

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

def restore_bytes(nb):
    """Restore bytes of image data from unicode-only formats.

    Base64 encoding is handled elsewhere.  Bytes objects in the notebook are
    always b64-encoded. We DO NOT encode/decode around file formats.
    """
    for ws in nb.worksheets:
        for cell in ws.cells:
            if cell.cell_type == 'code':
                for output in cell.outputs:
                    if 'png' in output:
                        output.png = str_to_bytes(output.png, 'ascii')
                    if 'jpeg' in output:
                        output.jpeg = str_to_bytes(output.jpeg, 'ascii')
    return nb

# output keys that are likely to have multiline values
_multiline_outputs = ['text', 'html', 'svg', 'latex', 'javascript', 'json']

# FIXME: workaround for old splitlines()
def _join_lines(lines):
    """join lines that have been written by splitlines()

    Has logic to protect against `splitlines()`, which
    should have been `splitlines(True)`
    """
    if lines and lines[0].endswith(('\n', '\r')):
        # created by splitlines(True)
        return u''.join(lines)
    else:
        # created by splitlines()
        return u'\n'.join(lines)

def rejoin_lines(nb):
    """rejoin multiline text into strings

    For reversing effects of ``split_lines(nb)``.

    This only rejoins lines that have been split, so if text objects were not split
    they will pass through unchanged.

    Used when reading JSON files that may have been passed through split_lines.
    """
    for ws in nb.worksheets:
        for cell in ws.cells:
            if cell.cell_type == 'code':
                if 'input' in cell and isinstance(cell.input, list):
                    cell.input = _join_lines(cell.input)
                for output in cell.outputs:
                    for key in _multiline_outputs:
                        item = output.get(key, None)
                        if isinstance(item, list):
                            output[key] = _join_lines(item)
            else: # text, heading cell
                for key in ['source', 'rendered']:
                    item = cell.get(key, None)
                    if isinstance(item, list):
                        cell[key] = _join_lines(item)
    return nb

def split_lines(nb):
    """split likely multiline text into lists of strings

    For file output more friendly to line-based VCS. ``rejoin_lines(nb)`` will
    reverse the effects of ``split_lines(nb)``.

    Used when writing JSON files.
    """
    for ws in nb.worksheets:
        for cell in ws.cells:
            if cell.cell_type == 'code':
                if 'input' in cell and isinstance(cell.input, basestring):
                    cell.input = cell.input.splitlines(True)
                for output in cell.outputs:
                    for key in _multiline_outputs:
                        item = output.get(key, None)
                        if isinstance(item, basestring):
                            output[key] = item.splitlines(True)
            else: # text, heading cell
                for key in ['source', 'rendered']:
                    item = cell.get(key, None)
                    if isinstance(item, basestring):
                        cell[key] = item.splitlines(True)
    return nb

# b64 encode/decode are never actually used, because all bytes objects in
# the notebook are already b64-encoded, and we don't need/want to double-encode

def base64_decode(nb):
    """Restore all bytes objects in the notebook from base64-encoded strings.

    Note: This is never used
    """
    for ws in nb.worksheets:
        for cell in ws.cells:
            if cell.cell_type == 'code':
                for output in cell.outputs:
                    if 'png' in output:
                        if isinstance(output.png, unicode):
                            output.png = output.png.encode('ascii')
                        output.png = decodestring(output.png)
                    if 'jpeg' in output:
                        if isinstance(output.jpeg, unicode):
                            output.jpeg = output.jpeg.encode('ascii')
                        output.jpeg = decodestring(output.jpeg)
    return nb

def base64_encode(nb):
    """Base64 encode all bytes objects in the notebook.

    These will be b64-encoded unicode strings

    Note: This is never used
    """
    for ws in nb.worksheets:
        for cell in ws.cells:
            if cell.cell_type == 'code':
                for output in cell.outputs:
                    if 'png' in output:
                        output.png = encodestring(output.png).decode('ascii')
                    if 'jpeg' in output:
                        output.jpeg = encodestring(output.jpeg).decode('ascii')
    return nb

class NotebookReader(object):
    """A class for reading notebooks."""

    def reads(self, s, **kwargs):
        """Read a notebook from a string."""
        raise NotImplementedError("loads must be implemented in a subclass")

    def read(self, fp, **kwargs):
        """Read a notebook from a file like object"""
        nbs = fp.read()
        if not py3compat.PY3 and not isinstance(nbs, unicode):
            nbs = py3compat.str_to_unicode(nbs)
        return self.reads(nbs, **kwargs)

class NotebookWriter(object):
    """A class for writing notebooks."""

    def writes(self, nb, **kwargs):
        """Write a notebook to a string."""
        raise NotImplementedError("loads must be implemented in a subclass")

    def write(self, nb, fp, **kwargs):
        """Write a notebook to a file like object"""
        nbs = self.writes(nb,**kwargs)
        if not py3compat.PY3 and not isinstance(nbs, unicode):
            # this branch is likely only taken for JSON on Python 2
            nbs = py3compat.str_to_unicode(nbs)
        return fp.write(nbs)