#!/usr/bin/env python
# encoding: utf-8
"""
The IPython controller application.

Authors:

* Brian Granger
* MinRK

"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from __future__ import with_statement

import json
import os
import socket
import stat
import sys

from multiprocessing import Process
from signal import signal, SIGINT, SIGABRT, SIGTERM

import zmq
from zmq.devices import ProcessMonitoredQueue
from zmq.log.handlers import PUBHandler

from IPython.core.profiledir import ProfileDir

from IPython.parallel.apps.baseapp import (
    BaseParallelApplication,
    base_aliases,
    base_flags,
    catch_config_error,
)
from IPython.utils.importstring import import_item
from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict, TraitError

from IPython.zmq.session import (
    Session, session_aliases, session_flags, default_secure
)

from IPython.parallel.controller.heartmonitor import HeartMonitor
from IPython.parallel.controller.hub import HubFactory
from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
from IPython.parallel.controller.sqlitedb import SQLiteDB

from IPython.parallel.util import split_url, disambiguate_url

# conditional import of MongoDB backend class

try:
    from IPython.parallel.controller.mongodb import MongoDB
except ImportError:
    maybe_mongo = []
else:
    maybe_mongo = [MongoDB]

#-----------------------------------------------------------------------------
# Module level variables
#-----------------------------------------------------------------------------

#: The default config file name for this application
default_config_file_name = u'ipcontroller_config.py'

_description = """Start the IPython controller for parallel computing.

The IPython controller provides a gateway between the IPython engines and
clients. The controller needs to be started before the engines and can be
configured using command line options or using a cluster directory. Cluster
directories contain config, log and security files and are usually located in
your ipython directory and named as "profile_name". See the `profile`
and `profile-dir` options for details.
"""

_examples = """
ipcontroller --ip=192.168.0.1 --port=1000  # listen on ip, port for engines
ipcontroller --scheme=pure  # use the pure zeromq scheduler
"""

#-----------------------------------------------------------------------------
# The main application
#-----------------------------------------------------------------------------
flags = {}
flags.update(base_flags)
flags.update({
    'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
                    'Use threads instead of processes for the schedulers'),
    'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
                    'use the SQLiteDB backend'),
    'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
                    'use the MongoDB backend'),
    'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
                    'use the in-memory DictDB backend'),
    'nodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.NoDB'}},
                    """use dummy DB backend, which doesn't store any information.

                    This is the default as of IPython 0.13.

                    To enable delayed or repeated retrieval of results from the Hub,
                    select one of the true db backends.
                    """),
    'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
                    'reuse existing json connection files')
})

flags.update(session_flags)

aliases = dict(
    ssh = 'IPControllerApp.ssh_server',
    enginessh = 'IPControllerApp.engine_ssh_server',
    location = 'IPControllerApp.location',

    url = 'HubFactory.url',
    ip = 'HubFactory.ip',
    transport = 'HubFactory.transport',
    port = 'HubFactory.regport',

    ping = 'HeartMonitor.period',

    scheme = 'TaskScheduler.scheme_name',
    hwm = 'TaskScheduler.hwm',
)
aliases.update(base_aliases)
aliases.update(session_aliases)

class IPControllerApp(BaseParallelApplication):

    name = u'ipcontroller'
    description = _description
    examples = _examples
    config_file_name = Unicode(default_config_file_name)
    classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo

    # change default to True
    auto_create = Bool(True, config=True,
        help="""Whether to create profile dir if it doesn't exist.""")

    reuse_files = Bool(False, config=True,
        help="""Whether to reuse existing json connection files.
        If False, connection files will be removed on a clean exit.
        """
    )
    ssh_server = Unicode(u'', config=True,
        help="""ssh url for clients to use when connecting to the Controller
        processes. It should be of the form: [user@]server[:port]. The
        Controller's listening addresses must be accessible from the ssh server""",
    )
    engine_ssh_server = Unicode(u'', config=True,
        help="""ssh url for engines to use when connecting to the Controller
        processes. It should be of the form: [user@]server[:port]. The
        Controller's listening addresses must be accessible from the ssh server""",
    )
    location = Unicode(u'', config=True,
        help="""The external IP or domain name of the Controller, used for disambiguating
        engine and client connections.""",
    )
    import_statements = List([], config=True,
        help="import statements to be run at startup.  Necessary in some environments"
    )

    use_threads = Bool(False, config=True,
        help='Use threads instead of processes for the schedulers',
    )

    engine_json_file = Unicode('ipcontroller-engine.json', config=True,
        help="JSON filename where engine connection info will be stored.")
    client_json_file = Unicode('ipcontroller-client.json', config=True,
        help="JSON filename where client connection info will be stored.")

    def _cluster_id_changed(self, name, old, new):
        super(IPControllerApp, self)._cluster_id_changed(name, old, new)
        self.engine_json_file = "%s-engine.json" % self.name
        self.client_json_file = "%s-client.json" % self.name

    # internal
    children = List()
    mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')

    def _use_threads_changed(self, name, old, new):
        self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')

    write_connection_files = Bool(True,
        help="""Whether to write connection files to disk.
        True in all cases other than runs with `reuse_files=True` *after the first*
        """
    )

    aliases = Dict(aliases)
    flags = Dict(flags)

    def save_connection_dict(self, fname, cdict):
        """save a connection dict to json file."""
        c = self.config
        url = cdict['url']
        location = cdict['location']
        if not location:
            try:
                proto,ip,port = split_url(url)
            except AssertionError:
                pass
            else:
                try:
                    location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
                except (socket.gaierror, IndexError):
                    self.log.warn("Could not identify this machine's IP, assuming 127.0.0.1."
                    " You may need to specify '--location=<external_ip_address>' to help"
                    " IPython decide when to connect via loopback.")
                    location = '127.0.0.1'
            cdict['location'] = location
        fname = os.path.join(self.profile_dir.security_dir, fname)
        self.log.info("writing connection info to %s", fname)
        with open(fname, 'w') as f:
            f.write(json.dumps(cdict, indent=2))
        os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)

    def load_config_from_json(self):
        """load config from existing json connector files."""
        c = self.config
        self.log.debug("loading config from JSON")
        # load from engine config
        fname = os.path.join(self.profile_dir.security_dir, self.engine_json_file)
        self.log.info("loading connection info from %s", fname)
        with open(fname) as f:
            cfg = json.loads(f.read())
        key = cfg['exec_key']
        # json gives unicode, Session.key wants bytes
        c.Session.key = key.encode('ascii')
        xport,addr = cfg['url'].split('://')
        c.HubFactory.engine_transport = xport
        ip,ports = addr.split(':')
        c.HubFactory.engine_ip = ip
        c.HubFactory.regport = int(ports)
        self.location = cfg['location']
        if not self.engine_ssh_server:
            self.engine_ssh_server = cfg['ssh']
        # load client config
        fname = os.path.join(self.profile_dir.security_dir, self.client_json_file)
        self.log.info("loading connection info from %s", fname)
        with open(fname) as f:
            cfg = json.loads(f.read())
        assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
        xport,addr = cfg['url'].split('://')
        c.HubFactory.client_transport = xport
        ip,ports = addr.split(':')
        c.HubFactory.client_ip = ip
        if not self.ssh_server:
            self.ssh_server = cfg['ssh']
        assert int(ports) == c.HubFactory.regport, "regport mismatch"

    def cleanup_connection_files(self):
        if self.reuse_files:
            self.log.debug("leaving JSON connection files for reuse")
            return
        self.log.debug("cleaning up JSON connection files")
        for f in (self.client_json_file, self.engine_json_file):
            f = os.path.join(self.profile_dir.security_dir, f)
            try:
                os.remove(f)
            except Exception as e:
                self.log.error("Failed to cleanup connection file: %s", e)
            else:
                self.log.debug(u"removed %s", f)

    def load_secondary_config(self):
        """secondary config, loading from JSON and setting defaults"""
        if self.reuse_files:
            try:
                self.load_config_from_json()
            except (AssertionError,IOError) as e:
                self.log.error("Could not load config from JSON: %s" % e)
            else:
                # successfully loaded config from JSON, and reuse=True
                # no need to wite back the same file
                self.write_connection_files = False

        # switch Session.key default to secure
        default_secure(self.config)
        self.log.debug("Config changed")
        self.log.debug(repr(self.config))

    def init_hub(self):
        c = self.config

        self.do_import_statements()

        try:
            self.factory = HubFactory(config=c, log=self.log)
            # self.start_logging()
            self.factory.init_hub()
        except TraitError:
            raise
        except Exception:
            self.log.error("Couldn't construct the Controller", exc_info=True)
            self.exit(1)

        if self.write_connection_files:
            # save to new json config files
            f = self.factory
            cdict = {'exec_key' : f.session.key.decode('ascii'),
                    'ssh' : self.ssh_server,
                    'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
                    'location' : self.location
                    }
            self.save_connection_dict(self.client_json_file, cdict)
            edict = cdict
            edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
            edict['ssh'] = self.engine_ssh_server
            self.save_connection_dict(self.engine_json_file, edict)

    def init_schedulers(self):
        children = self.children
        mq = import_item(str(self.mq_class))

        hub = self.factory
        # disambiguate url, in case of *
        monitor_url = disambiguate_url(hub.monitor_url)
        # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url
        # IOPub relay (in a Process)
        q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub')
        q.bind_in(hub.client_info['iopub'])
        q.bind_out(hub.engine_info['iopub'])
        q.setsockopt_out(zmq.SUBSCRIBE, b'')
        q.connect_mon(monitor_url)
        q.daemon=True
        children.append(q)

        # Multiplexer Queue (in a Process)
        q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out')
        q.bind_in(hub.client_info['mux'])
        q.setsockopt_in(zmq.IDENTITY, b'mux')
        q.bind_out(hub.engine_info['mux'])
        q.connect_mon(monitor_url)
        q.daemon=True
        children.append(q)

        # Control Queue (in a Process)
        q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol')
        q.bind_in(hub.client_info['control'])
        q.setsockopt_in(zmq.IDENTITY, b'control')
        q.bind_out(hub.engine_info['control'])
        q.connect_mon(monitor_url)
        q.daemon=True
        children.append(q)
        try:
            scheme = self.config.TaskScheduler.scheme_name
        except AttributeError:
            scheme = TaskScheduler.scheme_name.get_default_value()
        # Task Queue (in a Process)
        if scheme == 'pure':
            self.log.warn("task::using pure DEALER Task scheduler")
            q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask')
            # q.setsockopt_out(zmq.HWM, hub.hwm)
            q.bind_in(hub.client_info['task'][1])
            q.setsockopt_in(zmq.IDENTITY, b'task')
            q.bind_out(hub.engine_info['task'])
            q.connect_mon(monitor_url)
            q.daemon=True
            children.append(q)
        elif scheme == 'none':
            self.log.warn("task::using no Task scheduler")

        else:
            self.log.info("task::using Python %s Task scheduler"%scheme)
            sargs = (hub.client_info['task'][1], hub.engine_info['task'],
                                monitor_url, disambiguate_url(hub.client_info['notification']))
            kwargs = dict(logname='scheduler', loglevel=self.log_level,
                            log_url = self.log_url, config=dict(self.config))
            if 'Process' in self.mq_class:
                # run the Python scheduler in a Process
                q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
                q.daemon=True
                children.append(q)
            else:
                # single-threaded Controller
                kwargs['in_thread'] = True
                launch_scheduler(*sargs, **kwargs)

    def terminate_children(self):
        child_procs = []
        for child in self.children:
            if isinstance(child, ProcessMonitoredQueue):
                child_procs.append(child.launcher)
            elif isinstance(child, Process):
                child_procs.append(child)
        if child_procs:
            self.log.critical("terminating children...")
            for child in child_procs:
                try:
                    child.terminate()
                except OSError:
                    # already dead
                    pass

    def handle_signal(self, sig, frame):
        self.log.critical("Received signal %i, shutting down", sig)
        self.terminate_children()
        self.loop.stop()

    def init_signal(self):
        for sig in (SIGINT, SIGABRT, SIGTERM):
            signal(sig, self.handle_signal)

    def do_import_statements(self):
        statements = self.import_statements
        for s in statements:
            try:
                self.log.msg("Executing statement: '%s'" % s)
                exec s in globals(), locals()
            except:
                self.log.msg("Error running statement: %s" % s)

    def forward_logging(self):
        if self.log_url:
            self.log.info("Forwarding logging to %s"%self.log_url)
            context = zmq.Context.instance()
            lsock = context.socket(zmq.PUB)
            lsock.connect(self.log_url)
            handler = PUBHandler(lsock)
            handler.root_topic = 'controller'
            handler.setLevel(self.log_level)
            self.log.addHandler(handler)

    @catch_config_error
    def initialize(self, argv=None):
        super(IPControllerApp, self).initialize(argv)
        self.forward_logging()
        self.load_secondary_config()
        self.init_hub()
        self.init_schedulers()

    def start(self):
        # Start the subprocesses:
        self.factory.start()
        # children must be started before signals are setup,
        # otherwise signal-handling will fire multiple times
        for child in self.children:
            child.start()
        self.init_signal()

        self.write_pid_file(overwrite=True)

        try:
            self.factory.loop.start()
        except KeyboardInterrupt:
            self.log.critical("Interrupted, Exiting...\n")
        finally:
            self.cleanup_connection_files()

def launch_new_instance():
    """Create and run the IPython controller"""
    if sys.platform == 'win32':
        # make sure we don't get called from a multiprocessing subprocess
        # this can result in infinite Controllers being started on Windows
        # which doesn't have a proper fork, so multiprocessing is wonky

        # this only comes up when IPython has been installed using vanilla
        # setuptools, and *not* distribute.
        import multiprocessing
        p = multiprocessing.current_process()
        # the main process has name 'MainProcess'
        # subprocesses will have names like 'Process-1'
        if p.name != 'MainProcess':
            # we are a subprocess, don't start another Controller!
            return
    app = IPControllerApp.instance()
    app.initialize()
    app.start()

if __name__ == '__main__':
    launch_new_instance()


"""Magic functions for running cells in various scripts."""
#-----------------------------------------------------------------------------
#  Copyright (c) 2012 The IPython Development Team.
#
#  Distributed under the terms of the Modified BSD License.
#
#  The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

# Stdlib
import os
import re
import sys
import signal
import time
from subprocess import Popen, PIPE
import atexit

# Our own packages
from IPython.config.configurable import Configurable
from IPython.core import magic_arguments
from IPython.core.error import UsageError
from IPython.core.magic import  (
    Magics, magics_class, line_magic, cell_magic
)
from IPython.lib.backgroundjobs import BackgroundJobManager
from IPython.testing.skipdoctest import skip_doctest
from IPython.utils import py3compat
from IPython.utils.process import find_cmd, FindCmdError, arg_split
from IPython.utils.traitlets import List, Dict

#-----------------------------------------------------------------------------
# Magic implementation classes
#-----------------------------------------------------------------------------

def script_args(f):
    """single decorator for adding script args"""
    args = [
        magic_arguments.argument(
            '--out', type=str,
            help="""The variable in which to store stdout from the script.
            If the script is backgrounded, this will be the stdout *pipe*,
            instead of the stderr text itself.
            """
        ),
        magic_arguments.argument(
            '--err', type=str,
            help="""The variable in which to store stderr from the script.
            If the script is backgrounded, this will be the stderr *pipe*,
            instead of the stderr text itself.
            """
        ),
        magic_arguments.argument(
            '--bg', action="store_true",
            help="""Whether to run the script in the background.
            If given, the only way to see the output of the command is
            with --out/err.
            """
        ),
        magic_arguments.argument(
            '--proc', type=str,
            help="""The variable in which to store Popen instance.
            This is used only when --bg option is given.
            """
        ),
    ]
    for arg in args:
        f = arg(f)
    return f

@magics_class
class ScriptMagics(Magics, Configurable):
    """Magics for talking to scripts

    This defines a base `%%script` cell magic for running a cell
    with a program in a subprocess, and registers a few top-level
    magics that call %%script with common interpreters.
    """
    script_magics = List(config=True,
        help="""Extra script cell magics to define

        This generates simple wrappers of `%%script foo` as `%%foo`.

        If you want to add script magics that aren't on your path,
        specify them in script_paths
        """,
    )
    def _script_magics_default(self):
        """default to a common list of programs if we find them"""

        defaults = []
        to_try = []
        if os.name == 'nt':
            defaults.append('cmd')
            to_try.append('powershell')
        to_try.extend([
            'sh',
            'bash',
            'perl',
            'ruby',
            'python3',
            'pypy',
        ])

        for cmd in to_try:
            if cmd in self.script_paths:
                defaults.append(cmd)
            else:
                try:
                    find_cmd(cmd)
                except FindCmdError:
                    # command not found, ignore it
                    pass
                except ImportError:
                    # Windows without pywin32, find_cmd doesn't work
                    pass
                else:
                    defaults.append(cmd)
        return defaults

    script_paths = Dict(config=True,
        help="""Dict mapping short 'ruby' names to full paths, such as '/opt/secret/bin/ruby'

        Only necessary for items in script_magics where the default path will not
        find the right interpreter.
        """
    )

    def __init__(self, shell=None):
        Configurable.__init__(self, config=shell.config)
        self._generate_script_magics()
        Magics.__init__(self, shell=shell)
        self.job_manager = BackgroundJobManager()
        self.bg_processes = []
        atexit.register(self.kill_bg_processes)

    def __del__(self):
        self.kill_bg_processes()

    def _generate_script_magics(self):
        cell_magics = self.magics['cell']
        for name in self.script_magics:
            cell_magics[name] = self._make_script_magic(name)

    def _make_script_magic(self, name):
        """make a named magic, that calls %%script with a particular program"""
        # expand to explicit path if necessary:
        script = self.script_paths.get(name, name)

        @magic_arguments.magic_arguments()
        @script_args
        def named_script_magic(line, cell):
            # if line, add it as cl-flags
            if line:
                 line = "%s %s" % (script, line)
            else:
                line = script
            return self.shebang(line, cell)

        # write a basic docstring:
        named_script_magic.__doc__ = \
        """%%{name} script magic

        Run cells with {script} in a subprocess.

        This is a shortcut for `%%script {script}`
        """.format(**locals())

        return named_script_magic

    @magic_arguments.magic_arguments()
    @script_args
    @cell_magic("script")
    def shebang(self, line, cell):
        """Run a cell via a shell command

        The `%%script` line is like the #! line of script,
        specifying a program (bash, perl, ruby, etc.) with which to run.

        The rest of the cell is run by that program.

        Examples
        --------
        ::

            In [1]: %%script bash
               ...: for i in 1 2 3; do
               ...:   echo $i
               ...: done
            1
            2
            3
        """
        argv = arg_split(line, posix = not sys.platform.startswith('win'))
        args, cmd = self.shebang.parser.parse_known_args(argv)

        p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)

        cell = cell.encode('utf8', 'replace')
        if args.bg:
            self.bg_processes.append(p)
            self._gc_bg_processes()
            if args.out:
                self.shell.user_ns[args.out] = p.stdout
            if args.err:
                self.shell.user_ns[args.err] = p.stderr
            self.job_manager.new(self._run_script, p, cell, daemon=True)
            if args.proc:
                self.shell.user_ns[args.proc] = p
            return

        try:
            out, err = p.communicate(cell)
        except KeyboardInterrupt:
            try:
                p.send_signal(signal.SIGINT)
                time.sleep(0.1)
                if p.poll() is not None:
                    print "Process is interrupted."
                    return
                p.terminate()
                time.sleep(0.1)
                if p.poll() is not None:
                    print "Process is terminated."
                    return
                p.kill()
                print "Process is killed."
            except OSError:
                pass
            except Exception as e:
                print "Error while terminating subprocess (pid=%i): %s" \
                    % (p.pid, e)
            return
        out = py3compat.bytes_to_str(out)
        err = py3compat.bytes_to_str(err)
        if args.out:
            self.shell.user_ns[args.out] = out
        else:
            sys.stdout.write(out)
            sys.stdout.flush()
        if args.err:
            self.shell.user_ns[args.err] = err
        else:
            sys.stderr.write(err)
            sys.stderr.flush()

    def _run_script(self, p, cell):
        """callback for running the script in the background"""
        p.stdin.write(cell)
        p.stdin.close()
        p.wait()

    @line_magic("killbgscripts")
    def killbgscripts(self, _nouse_=''):
        """Kill all BG processes started by %%script and its family."""
        self.kill_bg_processes()
        print "All background processes were killed."

    def kill_bg_processes(self):
        """Kill all BG processes which are still running."""
        for p in self.bg_processes:
            if p.poll() is None:
                try:
                    p.send_signal(signal.SIGINT)
                except:
                    pass
        time.sleep(0.1)
        for p in self.bg_processes:
            if p.poll() is None:
                try:
                    p.terminate()
                except:
                    pass
        time.sleep(0.1)
        for p in self.bg_processes:
            if p.poll() is None:
                try:
                    p.kill()
                except:
                    pass
        self._gc_bg_processes()

    def _gc_bg_processes(self):
        self.bg_processes = [p for p in self.bg_processes if p.poll() is None]


# encoding: utf-8
"""
=============
parallelmagic
=============

Magic command interface for interactive parallel work.

Usage
=====

``%autopx``

{AUTOPX_DOC}

``%px``

{PX_DOC}

``%pxresult``

{RESULT_DOC}

``%pxconfig``

{CONFIG_DOC}

"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008 The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import ast
import re

from IPython.core.error import UsageError
from IPython.core.magic import Magics
from IPython.core import magic_arguments
from IPython.testing.skipdoctest import skip_doctest

#-----------------------------------------------------------------------------
# Definitions of magic functions for use with IPython
#-----------------------------------------------------------------------------

NO_LAST_RESULT = "%pxresult recalls last %px result, which has not yet been used."

def exec_args(f):
    """decorator for adding block/targets args for execution

    applied to %pxconfig and %%px
    """
    args = [
        magic_arguments.argument('-b', '--block', action="store_const",
            const=True, dest='block',
            help="use blocking (sync) execution",
        ),
        magic_arguments.argument('-a', '--noblock', action="store_const",
            const=False, dest='block',
            help="use non-blocking (async) execution",
        ),
        magic_arguments.argument('-t', '--targets', type=str,
            help="specify the targets on which to execute",
        ),
        magic_arguments.argument('--verbose', action="store_const",
            const=True, dest="set_verbose",
            help="print a message at each execution",
        ),
        magic_arguments.argument('--no-verbose', action="store_const",
            const=False, dest="set_verbose",
            help="don't print any messages",
        ),
    ]
    for a in args:
        f = a(f)
    return f

def output_args(f):
    """decorator for output-formatting args

    applied to %pxresult and %%px
    """
    args = [
        magic_arguments.argument('-r', action="store_const", dest='groupby',
            const='order',
            help="collate outputs in order (same as group-outputs=order)"
        ),
        magic_arguments.argument('-e', action="store_const", dest='groupby',
            const='engine',
            help="group outputs by engine (same as group-outputs=engine)"
        ),
        magic_arguments.argument('--group-outputs', dest='groupby', type=str,
            choices=['engine', 'order', 'type'], default='type',
            help="""Group the outputs in a particular way.

            Choices are:

            type: group outputs of all engines by type (stdout, stderr, displaypub, etc.).

            engine: display all output for each engine together.

            order: like type, but individual displaypub output from each engine is collated.
                For example, if multiple plots are generated by each engine, the first
                figure of each engine will be displayed, then the second of each, etc.
            """
        ),
        magic_arguments.argument('-o', '--out', dest='save_name', type=str,
            help="""store the AsyncResult object for this computation
                 in the global namespace under this name.
            """
        ),
    ]
    for a in args:
        f = a(f)
    return f

class ParallelMagics(Magics):
    """A set of magics useful when controlling a parallel IPython cluster.
    """

    # magic-related
    magics = None
    registered = True

    # suffix for magics
    suffix = ''
    # A flag showing if autopx is activated or not
    _autopx = False
    # the current view used by the magics:
    view = None
    # last result cache for %pxresult
    last_result = None
    # verbose flag
    verbose = False

    def __init__(self, shell, view, suffix=''):
        self.view = view
        self.suffix = suffix

        # register magics
        self.magics = dict(cell={},line={})
        line_magics = self.magics['line']

        px = 'px' + suffix
        if not suffix:
            # keep %result for legacy compatibility
            line_magics['result'] = self.result

        line_magics['pxresult' + suffix] = self.result
        line_magics[px] = self.px
        line_magics['pxconfig' + suffix] = self.pxconfig
        line_magics['auto' + px] = self.autopx

        self.magics['cell'][px] = self.cell_px

        super(ParallelMagics, self).__init__(shell=shell)

    def _eval_target_str(self, ts):
        if ':' in ts:
            targets = eval("self.view.client.ids[%s]" % ts)
        elif 'all' in ts:
            targets = 'all'
        else:
            targets = eval(ts)
        return targets

    @magic_arguments.magic_arguments()
    @exec_args
    def pxconfig(self, line):
        """configure default targets/blocking for %px magics"""
        args = magic_arguments.parse_argstring(self.pxconfig, line)
        if args.targets:
            self.view.targets = self._eval_target_str(args.targets)
        if args.block is not None:
            self.view.block = args.block
        if args.set_verbose is not None:
            self.verbose = args.set_verbose

    @magic_arguments.magic_arguments()
    @output_args
    @skip_doctest
    def result(self, line=''):
        """Print the result of the last asynchronous %px command.

        This lets you recall the results of %px computations after
        asynchronous submission (block=False).

        Examples
        --------
        ::

            In [23]: %px os.getpid()
            Async parallel execution on engine(s): all

            In [24]: %pxresult
            Out[8:10]: 60920
            Out[9:10]: 60921
            Out[10:10]: 60922
            Out[11:10]: 60923
        """
        args = magic_arguments.parse_argstring(self.result, line)

        if self.last_result is None:
            raise UsageError(NO_LAST_RESULT)

        self.last_result.get()
        self.last_result.display_outputs(groupby=args.groupby)

    @skip_doctest
    def px(self, line=''):
        """Executes the given python command in parallel.

        Examples
        --------
        ::

            In [24]: %px a = os.getpid()
            Parallel execution on engine(s): all

            In [25]: %px print a
            [stdout:0] 1234
            [stdout:1] 1235
            [stdout:2] 1236
            [stdout:3] 1237
        """
        return self.parallel_execute(line)

    def parallel_execute(self, cell, block=None, groupby='type', save_name=None):
        """implementation used by %px and %%parallel"""

        # defaults:
        block = self.view.block if block is None else block

        base = "Parallel" if block else "Async parallel"

        targets = self.view.targets
        if isinstance(targets, list) and len(targets) > 10:
            str_targets = str(targets[:4])[:-1] + ', ..., ' + str(targets[-4:])[1:]
        else:
            str_targets = str(targets)
        if self.verbose:
            print base + " execution on engine(s): %s" % str_targets

        result = self.view.execute(cell, silent=False, block=False)
        self.last_result = result

        if save_name:
            self.shell.user_ns[save_name] = result

        if block:
            result.get()
            result.display_outputs(groupby)
        else:
            # return AsyncResult only on non-blocking submission
            return result

    @magic_arguments.magic_arguments()
    @exec_args
    @output_args
    @skip_doctest
    def cell_px(self, line='', cell=None):
        """Executes the cell in parallel.

        Examples
        --------
        ::

            In [24]: %%px --noblock
               ....: a = os.getpid()
            Async parallel execution on engine(s): all

            In [25]: %%px
               ....: print a
            [stdout:0] 1234
            [stdout:1] 1235
            [stdout:2] 1236
            [stdout:3] 1237
        """

        args = magic_arguments.parse_argstring(self.cell_px, line)

        if args.targets:
            save_targets = self.view.targets
            self.view.targets = self._eval_target_str(args.targets)
        try:
            return self.parallel_execute(cell, block=args.block,
                                groupby=args.groupby,
                                save_name=args.save_name,
            )
        finally:
            if args.targets:
                self.view.targets = save_targets

    @skip_doctest
    def autopx(self, line=''):
        """Toggles auto parallel mode.

        Once this is called, all commands typed at the command line are send to
        the engines to be executed in parallel. To control which engine are
        used, the ``targets`` attribute of the view before
        entering ``%autopx`` mode.

        Then you can do the following::

            In [25]: %autopx
            %autopx to enabled

            In [26]: a = 10
            Parallel execution on engine(s): [0,1,2,3]
            In [27]: print a
            Parallel execution on engine(s): [0,1,2,3]
            [stdout:0] 10
            [stdout:1] 10
            [stdout:2] 10
            [stdout:3] 10

            In [27]: %autopx
            %autopx disabled
        """
        if self._autopx:
            self._disable_autopx()
        else:
            self._enable_autopx()

    def _enable_autopx(self):
        """Enable %autopx mode by saving the original run_cell and installing
        pxrun_cell.
        """
        # override run_cell
        self._original_run_cell = self.shell.run_cell
        self.shell.run_cell = self.pxrun_cell

        self._autopx = True
        print "%autopx enabled"

    def _disable_autopx(self):
        """Disable %autopx by restoring the original InteractiveShell.run_cell.
        """
        if self._autopx:
            self.shell.run_cell = self._original_run_cell
            self._autopx = False
            print "%autopx disabled"

    def pxrun_cell(self, raw_cell, store_history=False, silent=False):
        """drop-in replacement for InteractiveShell.run_cell.

        This executes code remotely, instead of in the local namespace.

        See InteractiveShell.run_cell for details.
        """

        if (not raw_cell) or raw_cell.isspace():
            return

        ipself = self.shell

        with ipself.builtin_trap:
            cell = ipself.prefilter_manager.prefilter_lines(raw_cell)

            # Store raw and processed history
            if store_history:
                ipself.history_manager.store_inputs(ipself.execution_count,
                                                  cell, raw_cell)

            # ipself.logger.log(cell, raw_cell)

            cell_name = ipself.compile.cache(cell, ipself.execution_count)

            try:
                ast.parse(cell, filename=cell_name)
            except (OverflowError, SyntaxError, ValueError, TypeError,
                    MemoryError):
                # Case 1
                ipself.showsyntaxerror()
                ipself.execution_count += 1
                return None
            except NameError:
                # ignore name errors, because we don't know the remote keys
                pass

        if store_history:
            # Write output to the database. Does nothing unless
            # history output logging is enabled.
            ipself.history_manager.store_output(ipself.execution_count)
            # Each cell is a *single* input, regardless of how many lines it has
            ipself.execution_count += 1
        if re.search(r'get_ipython\(\)\.magic\(u?["\']%?autopx', cell):
            self._disable_autopx()
            return False
        else:
            try:
                result = self.view.execute(cell, silent=False, block=False)
            except:
                ipself.showtraceback()
                return True
            else:
                if self.view.block:
                    try:
                        result.get()
                    except:
                        self.shell.showtraceback()
                        return True
                    else:
                        with ipself.builtin_trap:
                            result.display_outputs()
                return False

__doc__ = __doc__.format(
                AUTOPX_DOC = ' '*8 + ParallelMagics.autopx.__doc__,
                PX_DOC = ' '*8 + ParallelMagics.px.__doc__,
                RESULT_DOC = ' '*8 + ParallelMagics.result.__doc__,
                CONFIG_DOC = ' '*8 + ParallelMagics.pxconfig.__doc__,
)


"""
django-channels consumers for sisy
"""
import arrow
import pytz
from django.conf import settings
from django.utils import timezone

from sisy.models import Task

try:
    TICK_FREQ = settings.SISY_TICK_FREQUENCY
except AttributeError:
    TICK_FREQ = 60 # seconds

TIME_FORMAT = "%Y-%m-%d %H:%M:%S %Z"
TZ = pytz.timezone(settings.TIME_ZONE)

def run_heartbeat(message):
    """Internal ``CLOCK_CHANNEL`` consumer to process task runs"""
    then = arrow.get(message['time'])
    now = arrow.get()

    if (now - then) > timezone.timedelta(seconds=(TICK_FREQ+1)):
        pass # discard old ticks
    else:
        Task.run_tasks()

def run_task(message):
    """Internal ``RUN_TASK`` consumer to run the task's callable"""
    task = Task.objects.get(pk=message['id'])
    if task.allow_overlap:
        task.run(message)
    else:
        if not task.running:
            task.running = True
            task.save()
            try:
                task.run(message)
            finally:
                task.running = False
                task.save()

def remove_task(message):
    """Internal ``KILL_TASK`` consumer to remove retired tasks"""
    task = Task.objects.get(pk=message['id'])
    task.delete()


from __future__ import absolute_import, unicode_literals, print_function

from twisted.internet.defer import Deferred, DeferredQueue
from twisted.internet.error import (
    ConnectionRefusedError, ConnectionDone, ConnectionAborted)
from twisted.internet.interfaces import IStreamClientEndpoint
from twisted.internet.protocol import Protocol, ServerFactory
from twisted.internet.task import deferLater
from twisted.protocols.loopback import loopbackAsync
from twisted.python.failure import Failure
from twisted.web.client import ProxyAgent
from twisted.web.resource import Resource
from twisted.web.server import Site
from zope.interface import implementer

def wait0(r=None):
    """
    Wait zero seconds to give the reactor a chance to work.

    Returns its (optional) argument, so it's useful as a callback.
    """
    from twisted.internet import reactor
    return deferLater(reactor, 0, lambda: r)

class ProtocolDouble(Protocol):
    """
    A stand-in protocol for manually driving one side of a connection.
    """

    def __init__(self):
        self.received = b""
        self.disconnected_reason = None

    def dataReceived(self, data):
        self.received += data

    def connectionLost(self, reason):
        self.connected = False
        self.disconnected_reason = reason

    def write(self, data):
        """
        Write some bytes and allow the reactor to send them.
        """
        self.transport.write(data)
        return wait0()

class FakeServer(object):
    """
    Fake server container for testing client/server interactions.
    """

    def __init__(self, server_factory, auto_accept=True, on_connect=None):
        self.server_factory = server_factory
        self.auto_accept = auto_accept
        self.connection_queue = DeferredQueue()
        self.on_connect = on_connect

    # Public API.

    @classmethod
    def for_protocol(cls, protocol, *args, **kw):
        factory = ServerFactory.forProtocol(protocol)
        return cls(factory, *args, **kw)

    @property
    def endpoint(self):
        """
        Get an endpoint that connects clients to this server.
        """
        return FakeServerEndpoint(self)

    def await_connection(self):
        """
        Wait for a client to start connecting, and then return a
        :class:`FakeConnection` object.
        """
        return self.connection_queue.get()

    # Internal stuff.

    def _handle_connection(self):
        conn = FakeConnection(self)
        if self.on_connect is not None:
            conn._connected_d.addCallback(lambda _: self.on_connect(conn))
        self.connection_queue.put(conn)
        if self.auto_accept:
            conn.accept_connection()
        return conn._accept_d

class FakeConnection(object):
    """
    Fake server connection.
    """

    def __init__(self, server):
        self.server = server
        self.client_protocol = None
        self.server_protocol = None

        self._accept_d = Deferred()
        self._connected_d = Deferred()
        self._finished_d = Deferred()

    @property
    def connected(self):
        return self._connected_d.called and not self._finished_d.called

    @property
    def pending(self):
        return not self._accept_d.called

    def await_connected(self):
        """
        Wait for a client to finish connecting.
        """
        return self._connected_d

    def accept_connection(self):
        """
        Accept a pending connection.
        """
        assert self.pending, "Connection is not pending."
        self.server_protocol = self.server.server_factory.buildProtocol(None)
        self._accept_d.callback(
            FakeServerProtocolWrapper(self, self.server_protocol))
        return self.await_connected()

    def reject_connection(self, reason=None):
        """
        Reject a pending connection.
        """
        assert self.pending, "Connection is not pending."
        if reason is None:
            reason = ConnectionRefusedError()
        self._accept_d.errback(reason)

    def await_finished(self):
        """
        Wait for the both sides of the connection to close.
        """
        return self._finished_d

    # Internal stuff.

    def _finish_connecting(self, client_protocol, finished_d):
        self.client_protocol = client_protocol
        finished_d.chainDeferred(self._finished_d)
        self._connected_d.callback(None)

@implementer(IStreamClientEndpoint)
class FakeServerEndpoint(object):
    """
    This endpoint connects a client directly to a FakeServer.
    """
    def __init__(self, server):
        self.server = server

    def connect(self, protocolFactory):
        d = self.server._handle_connection()
        return d.addCallback(self._make_connection, protocolFactory)

    def _make_connection(self, server, protocolFactory):
        client = protocolFactory.buildProtocol(None)
        patch_protocol_for_agent(client)
        finished_d = loopbackAsync(server, client)
        server.finish_connecting(client, finished_d)
        return client

def patch_protocol_for_agent(protocol):
    """
    Patch the protocol's makeConnection and connectionLost methods to make the
    protocol and its transport behave more like what `Agent` expects.

    While `Agent` is the driving force behind this, other clients and servers
    will no doubt have similar requirements.
    """
    old_makeConnection = protocol.makeConnection
    old_connectionLost = protocol.connectionLost

    def new_makeConnection(transport):
        patch_transport_fake_push_producer(transport)
        patch_transport_abortConnection(transport, protocol)
        return old_makeConnection(transport)

    def new_connectionLost(reason):
        # Replace ConnectionDone with ConnectionAborted if we aborted.
        if protocol._fake_connection_aborted and reason.check(ConnectionDone):
            reason = Failure(ConnectionAborted())
        return old_connectionLost(reason)

    protocol.makeConnection = new_makeConnection
    protocol.connectionLost = new_connectionLost
    protocol._fake_connection_aborted = False

def patch_if_missing(obj, name, method):
    """
    Patch a method onto an object if it isn't already there.
    """
    setattr(obj, name, getattr(obj, name, method))

def patch_transport_fake_push_producer(transport):
    """
    Patch the three methods belonging to IPushProducer onto the transport if it
    doesn't already have them. (`Agent` assumes its transport has these.)
    """
    patch_if_missing(transport, 'pauseProducing', lambda: None)
    patch_if_missing(transport, 'resumeProducing', lambda: None)
    patch_if_missing(transport, 'stopProducing', transport.loseConnection)

def patch_transport_abortConnection(transport, protocol):
    """
    Patch abortConnection() onto the transport if it doesn't already have it.
    (`Agent` assumes its transport has this.)
    """
    def abortConnection():
        protocol._fake_connection_aborted = True
        transport.loseConnection()
    patch_if_missing(transport, 'abortConnection', abortConnection)

class FakeServerProtocolWrapper(Protocol):
    """
    Wrapper around a server protocol to track connection state.
    """

    def __init__(self, connection, protocol):
        self.connection = connection
        patch_protocol_for_agent(protocol)
        self.protocol = protocol

    def makeConnection(self, transport):
        Protocol.makeConnection(self, transport)
        return self.protocol.makeConnection(transport)

    def connectionLost(self, reason):
        return self.protocol.connectionLost(reason)

    def dataReceived(self, data):
        return self.protocol.dataReceived(data)

    def finish_connecting(self, client, finished_d):
        self.connection._finish_connecting(client, finished_d)

class FakeHttpServer(object):
    """
    HTTP server built on top of FakeServer.
    """

    def __init__(self, handler):
        site_factory = Site(HandlerResource(handler))
        self.fake_server = FakeServer(site_factory)

    @property
    def endpoint(self):
        return self.fake_server.endpoint

    def get_agent(self, reactor=None, contextFactory=None):
        """
        Returns an IAgent that makes requests to this fake server.
        """
        return ProxyAgentWithContext(
            self.endpoint, reactor=reactor, contextFactory=contextFactory)

class HandlerResource(Resource):
    isLeaf = True

    def __init__(self, handler):
        Resource.__init__(self)
        self.handler = handler

    def render(self, request):
        return self.handler(request)

class ProxyAgentWithContext(ProxyAgent):
    def __init__(self, endpoint, reactor=None, contextFactory=None):
        self.contextFactory = contextFactory  # To assert on in tests.
        super(ProxyAgentWithContext, self).__init__(endpoint, reactor=reactor)


from __future__ import unicode_literals

import json

from django.shortcuts import render, render_to_response
from django.views.generic import edit
from django.views.generic import detail
from django.views import generic
from django import forms
from django.forms.models import inlineformset_factory, modelformset_factory
from django.utils.decorators import method_decorator
from django.contrib.auth.decorators import login_required
from django.http import HttpResponseRedirect, HttpResponse
from django.contrib.messages.views import SuccessMessageMixin
from django.core.exceptions import PermissionDenied

from .forms import CrispyFormSetHelper

#######################
# Generic view MIXINS #
#######################

class AjaxableResponseMixin(object):
    """
    Edit view (create, update) mixin that will return a json object with the
    errors instead of the rendered content.
    """

    def render_to_json_response(self, context, **response_kwargs):
        data = json.dumps(context)
        response_kwargs['content_type'] = 'application/json'
        return HttpResponse(data, **response_kwargs)

    def form_invalid(self, form):
        response = super(AjaxableResponseMixin, self).form_invalid(form)
        if self.request.is_ajax():
            data = {
                'error': 'form_invalid',
                'errors': form.errors,
            }
            return self.render_to_json_response(data, status=400)
        else:
            return response

    def form_valid(self, form):
        response = super(AjaxableResponseMixin, self).form_valid(form)
        if self.request.is_ajax():
            return self.render_to_json_response({'error': ''})
        else:
            return response

class CrispyFormAjaxResponseMixin(object):
    """
    Edit view (create, update) mixin that will, if it is an AJAX request,
    return just the rendered form(renderd by the crispy_form_raw.html template)
    instead of the full rendered output, and an json object with the new
    primary key on succes.

    Behaves normally for non-ajax requests.
    """

    def render_to_json_response(self, context, **response_kwargs):
        data = json.dumps(context)
        response_kwargs['content_type'] = 'application/json'
        return HttpResponse(data, **response_kwargs)

    def form_invalid(self, form):
        response = super(AjaxableResponseMixin, self).form_invalid(form)
        if self.request.is_ajax():
            output = render_to_response('crispy_form_raw.html', {'form': form})
            return HttpResponse(output, status=400)
        else:
            return response

    def form_valid(self, form):
        response = super(AjaxableResponseMixin, self).form_valid(form)
        if self.request.is_ajax():
            return HttpResponse('')
        else:
            return response

class ExtraContextMixin(object):
    """
    A mixin for Djangos generic views classes that offers an additional
    property extra_context for easily passing additional context vars.
    """

    extra_context = {}

    def get_context_data(self, **kwargs):
        context = super(ExtraContextMixin, self).get_context_data(**kwargs)
        # Add model verbose name to the context if needed in template.
        context['model_name'] = self.model._meta.verbose_name.capitalize()
        context.update(self.extra_context)
        return context

class SaveHookMixin(object):
    """
    A generic edit view mixin that provides pre_save
    post_save, pre_delete and post_delete hooks.
    """

    def pre_save(self, object):
        """
        Hook for altering object before save.
        """

        pass

    def post_save(self, object):
        """
        Hook for altering object after save.
        """

        pass

    def pre_delete(self, object):
        """
        Hook for check or alterations before deleting.
        """

        pass

    def post_delete(self, object):
        """
        Hook for alterations after delete.
        """

    def form_valid(self, form):
        """
        Calls pre and post save hooks.
        """
        self.object = form.save(commit=False)

        # Invoke pre_save hook, and allow it to abort the saving
        # process and do a redirect.
        response = self.pre_save(self.object)
        if response:
            return response

        self.object.save()
        form.save_m2m()
        self.post_save(self.object)

        return HttpResponseRedirect(self.get_success_url())

    def delete(self, request, *args, **kwargs):
        """
        Calls pre and post delete hooks for DelteViews.
        """

        self.object = self.get_object()
        success_url = self.get_success_url()
        self.pre_delete(self.object)
        self.object.delete()
        self.post_delete(self.object)

        return HttpResponseRedirect(success_url)

class AssertUserIsOwnerMixin(object):
    """
    This mixin for edit views asserts that the user updating or deleting the
    model is also the owner of the model.
    The owner is determined by a ForeignKey field on the model.
    The name of the filed is specified by owner_field and defaults to user.

    By default, the check is skipped for superusers.
    If you do not want to do this,
    set assert_user_is_owner_skip_superuser = False.

    TODO: rewrite to prevent duplicate calls to get_object().
    """

    owner_field = "user"
    assert_user_is_owner_skip_superuser = True

    def get(self, request, *args, **kwargs):
        self.object = self.get_object()
        self.assert_user_is_owner(self.object, request.user)

        return super(AssertUserIsOwnerMixin, self).get(request, *args, **kwargs)

    def post(self, request, *args, **kwargs):
        self.object = self.get_object()
        self.assert_user_is_owner(self.object, request.user)

        return super(AssertUserIsOwnerMixin, self).post(
            request, *args, **kwargs)

    def assert_user_is_owner(self, instance, user):
        if user.is_superuser and self.assert_user_is_owner_skip_superuser:
            return

        instance_user = getattr(instance, self.owner_field)
        if instance_user != user:
            raise PermissionDenied()

class UserViewMixin(object):
    """
    IMPORTANT: REQUIRES SaveHookMixin!

    This mixin alters a generic CREATE view that has a "created_by" field by
    automatically setting the user field to the current user when the form is
    submitted.
    The field name of the field to populate is specified by the user_field
    property and defaults to created_by.
    """

    user_field = ['created_by']

    def __init__(self, *args, **kwargs):
        super(UserViewMixin, self).__init__(*args, **kwargs)
        # Ensure that user_field is a list.
        if type(self.user_field) != list:
            self.user_field = [self.user_field]

    def get_initial(self):
        """
        Supply user object as initial data for the specified user_field(s).
        """

        data = super(UserViewMixin, self).get_initial()
        for k in self.user_field:
            data[k] = self.request.user

        return data

    def pre_save(self, instance):
        super(UserViewMixin, self).pre_save(instance)

        """
        Use SaveHookMixin pre_save to set the user.
        """
        if  self.request.user.is_authenticated():
            for field in self.user_field:
                setattr(instance, field, self.request.user)

#############################
# Generic view base classes #
#############################

class ListView(ExtraContextMixin, generic.ListView):
    template_name = "generics/list.html"

class DetailView(detail.DetailView):
    """
    DetailView which tries to show all the fields of a model.
    """

    template_name = "generics/detail.html"
    fields = None

    def get_context_data(self, **kwargs):
        context = super(DetailView, self).get_context_data(**kwargs)

        #fields = self.fields or self.model._meta.get_all_field_names()
        #field_data = {name:getattr(self.object, name) for name in fields}

        #context['fields'] = field_data

        return context

class CreateView(SuccessMessageMixin, ExtraContextMixin, SaveHookMixin, edit.CreateView):
    """
    CreateView that offers extra_context and a default template.
    """

    template_name = 'generics/create.html'

class UpdateView(SuccessMessageMixin, ExtraContextMixin, SaveHookMixin, edit.UpdateView):
    """
    UpdateView that offers extra_context and a default template.
    """

    template_name = 'generics/update.html'

class DeleteView(SuccessMessageMixin, ExtraContextMixin, edit.DeleteView):
    """
    DeleteView that offers extra_context and a default template.
    """

    template_name = 'generics/delete.html'

class FormSetMixin(object):
    """
    A ModelForm mixin that makes creating views with inline
    formsets really easy.
    Used by FormSetCreateView and FormSetUpdateView.
    """

    # The form class used for the inline items.
    inline_form_class = forms.ModelForm

    fields = None
    excluded_fields = ['created_at', 'created_by']

    # m2m fields that should be displayed with the default widget,
    # instead of inline.
    forced_regular_fields = []

    extra = 1
    can_delete = True

    # Extra arguments for the formset factory for a field.
    factory_extra_args = {}

    # Whether the fieldset and the individual items should be  expanded.
    fieldsets_expanded = True
    fieldset_items_expanded = True

    def __init__(self, *args, **kwargs):
        super(FormSetMixin, self).__init__(*args, **kwargs)

        # Auto-set fields to exclude the fieldsets.
        if self.fields == None:
            # All regular fields, except explicitly excluded ones.
            self.fields = [field.name for field in self.model._meta.fields
                if not field.name in self.excluded_fields]
            # Also add m2m fields that are explcitly set to normal widget
            # with self.forced_regular_fields.
            self.fields += [field.name for field in self.model._meta.many_to_many
                if field.name in self.forced_regular_fields]

    def get_fieldset_crispy_helper(self):
        return CrispyFormSetHelper()

    def get_context_data(self, **kwargs):
        context = super(FormSetMixin, self).get_context_data(**kwargs)
        context['fieldsets'] = self.get_fieldsets().items()
        context['helper'] = self.get_fieldset_crispy_helper()
        context['fieldsets_expanded'] = self.fieldsets_expanded
        context['fieldset_items_expanded'] = self.fieldset_items_expanded

        return context

    def pre_save(self, instance):
        """
        Hook for editing the instance.
        """

        pass

    def post_save(self, instance):
        for name, formset in self.formsets.items():
            instances = formset.save()
            for model in instances:
                getattr(instance, name).add(model)

        instance.save()

    def post(self, request, *args, **kwargs):
        form_class = self.get_form_class()
        form = self.get_form(form_class)

        formsets = self.formsets = self.get_fieldsets()

        # Check if both the form and all the fieldsets are valid.
        valid = form.is_valid() and reduce(lambda x, y: x and y, [f.is_valid() for name, f in formsets.items()])

        if valid:
            return self.form_valid(form)
        else:
            return self.form_invalid(form)

class FormSetCreateView(SuccessMessageMixin, ExtraContextMixin, FormSetMixin, SaveHookMixin, edit.CreateView):
    """
    CreateView for a model with inline forms of another model.
    """

    template_name = 'generics/create_fieldsets.html'
    extra = 3
    can_delete = False

    def get_fieldsets(self):
        fieldsets = {}

        for field in self.model._meta.many_to_many:

            target_model = field.related.parent_model

            # For m2m without a through model, use modelformset_factory.
            # For m2m with a thorough model, inlineformset_factory
            # saves a bunch of work.
            #
            # Determine which one to use by this HACKY method:
            # Check for underscores in through model name,
            # since the auto-generated m2m tables have an underscore.

            has_through_model = field.rel.through._meta.object_name.find('_') == -1

            args = [self.request.POST, self.request.FILES] if self.request.method == 'POST' else []
            kwargs = {}

            factory_kwargs = self.factory_extra_args[field.name] if field.name in self.factory_extra_args else {}

            if has_through_model:
                fieldset_cls = inlineformset_factory(self.model, field.rel.through,
                    form=self.inline_form_class, extra=self.extra, **factory_kwargs)
            else:
                fieldset_cls = modelformset_factory(target_model, form=self.inline_form_class,
                    extra=self.extra, can_delete = self.can_delete, **factory_kwargs)

                kwargs['prefix'] = field.name
                kwargs['queryset'] = target_model.objects.none()

            fieldsets[field.name] = fieldset_cls(*args, **kwargs)

        return fieldsets

    def post(self, request, *args, **kwargs):
        self.object = None
        return super(FormSetCreateView, self).post(request, *args, **kwargs)

class FormSetUpdateView(SuccessMessageMixin, ExtraContextMixin, FormSetMixin, SaveHookMixin, edit.UpdateView):
    """
    UpdateView for a model with inline forms of another model.
    """

    template_name = 'generics/update_fieldsets.html'

    def get_fieldsets(self):
        obj = self.get_object()

        fieldsets = {}

        for field in self.model._meta.many_to_many:

            target_model = field.related.parent_model

            # For m2m without a through model, use modelformset_factory.
            # For m2m with a thorough model, inlineformset_factory is
            # saves a bunch of work.
            #
            # Determine which one to use by this HACKY method:
            # Check for underscores in through model name,
            # since the auto-generated m2m tables have an underscore.

            has_through_model = field.rel.through._meta.object_name.find('_') == -1

            args = [self.request.POST, self.request.FILES] if self.request.method == 'POST' else []
            kwargs = {}

            factory_kwargs = self.factory_extra_args[field.name] if field.name in self.factory_extra_args else {}

            if has_through_model:
                fieldset_cls = inlineformset_factory(self.model, field.rel.through,
                    form=self.inline_form_class, extra=self.extra, **factory_kwargs)

                kwargs['instance'] = obj
            else:
                fieldset_cls = modelformset_factory(target_model,  form=self.inline_form_class,
                    extra=self.extra, can_delete = self.can_delete, **factory_kwargs)

                kwargs['prefix'] = field.name
                kwargs['queryset'] = getattr(obj, field.name).all()

            fieldsets[field.name] = fieldset_cls(*args, **kwargs)

        return fieldsets

    def get(self, request, *args, **kwargs):
        self.object = self.get_object()
        return super(FormSetUpdateView, self).get(request, *args, **kwargs)

    def post(self, request, *args, **kwargs):
        self.object = self.get_object()
        return super(FormSetUpdateView, self).post(request, *args, **kwargs)


"""Summary reporting"""

import sys

from coverage.report import Reporter
from coverage.results import Numbers
from coverage.misc import NotPython

class SummaryReporter(Reporter):
    """A reporter for writing the summary report."""

    def __init__(self, coverage, config):
        super(SummaryReporter, self).__init__(coverage, config)
        self.branches = coverage.data.has_arcs()

    def report(self, morfs, outfile=None):
        """Writes a report summarizing coverage statistics per module.

        `outfile` is a file object to write the summary to.

        """
        self.find_code_units(morfs)

        # Prepare the formatting strings
        max_name = max([len(cu.name) for cu in self.code_units] + [5])
        fmt_name = "%%- %ds  " % max_name
        fmt_err = "%s   %s: %s\n"
        header = (fmt_name % "Name") + " Stmts   Miss"
        fmt_coverage = fmt_name + "%6d %6d"
        if self.branches:
            header += " Branch BrMiss"
            fmt_coverage += " %6d %6d"
        width100 = Numbers.pc_str_width()
        header += "%*s" % (width100+4, "Cover")
        fmt_coverage += "%%%ds%%%%" % (width100+3,)
        if self.config.show_missing:
            header += "   Missing"
            fmt_coverage += "   %s"
        rule = "-" * len(header) + "\n"
        header += "\n"
        fmt_coverage += "\n"

        if not outfile:
            outfile = sys.stdout

        # Write the header
        outfile.write(header)
        outfile.write(rule)

        total = Numbers()

        for cu in self.code_units:
            try:
                analysis = self.coverage._analyze(cu)
                nums = analysis.numbers
                args = (cu.name, nums.n_statements, nums.n_missing)
                if self.branches:
                    args += (nums.n_branches, nums.n_missing_branches)
                args += (nums.pc_covered_str,)
                if self.config.show_missing:
                    args += (analysis.missing_formatted(),)
                outfile.write(fmt_coverage % args)
                total += nums
            except KeyboardInterrupt:                   # pragma: not covered
                raise
            except:
                report_it = not self.config.ignore_errors
                if report_it:
                    typ, msg = sys.exc_info()[:2]
                    if typ is NotPython and not cu.should_be_python():
                        report_it = False
                if report_it:
                    outfile.write(fmt_err % (cu.name, typ.__name__, msg))

        if total.n_files > 1:
            outfile.write(rule)
            args = ("TOTAL", total.n_statements, total.n_missing)
            if self.branches:
                args += (total.n_branches, total.n_missing_branches)
            args += (total.pc_covered_str,)
            if self.config.show_missing:
                args += ("",)
            outfile.write(fmt_coverage % args)

        return total.pc_covered


"""IPython extension to reload modules before executing user code.

``autoreload`` reloads modules automatically before entering the execution of
code typed at the IPython prompt.

This makes for example the following workflow possible:

.. sourcecode:: ipython

   In [1]: %load_ext autoreload

   In [2]: %autoreload 2

   In [3]: from foo import some_function

   In [4]: some_function()
   Out[4]: 42

   In [5]: # open foo.py in an editor and change some_function to return 43

   In [6]: some_function()
   Out[6]: 43

The module was reloaded without reloading it explicitly, and the object
imported with ``from foo import ...`` was also updated.

Usage
=====

The following magic commands are provided:

``%autoreload``

    Reload all modules (except those excluded by ``%aimport``)
    automatically now.

``%autoreload 0``

    Disable automatic reloading.

``%autoreload 1``

    Reload all modules imported with ``%aimport`` every time before
    executing the Python code typed.

``%autoreload 2``

    Reload all modules (except those excluded by ``%aimport``) every
    time before executing the Python code typed.

``%aimport``

    List modules which are to be automatically imported or not to be imported.

``%aimport foo``

    Import module 'foo' and mark it to be autoreloaded for ``%autoreload 1``

``%aimport -foo``

    Mark module 'foo' to not be autoreloaded.

Caveats
=======

Reloading Python modules in a reliable way is in general difficult,
and unexpected things may occur. ``%autoreload`` tries to work around
common pitfalls by replacing function code objects and parts of
classes previously in the module with new versions. This makes the
following things to work:

- Functions and classes imported via 'from xxx import foo' are upgraded
  to new versions when 'xxx' is reloaded.

- Methods and properties of classes are upgraded on reload, so that
  calling 'c.foo()' on an object 'c' created before the reload causes
  the new code for 'foo' to be executed.

Some of the known remaining caveats are:

- Replacing code objects does not always succeed: changing a @property
  in a class to an ordinary method or a method to a member variable
  can cause problems (but in old objects only).

- Functions that are removed (eg. via monkey-patching) from a module
  before it is reloaded are not upgraded.

- C extension modules cannot be reloaded, and so cannot be autoreloaded.
"""

skip_doctest = True

#-----------------------------------------------------------------------------
#  Copyright (C) 2000 Thomas Heller
#  Copyright (C) 2008 Pauli Virtanen <pav@iki.fi>
#  Copyright (C) 2012  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------
#
# This IPython module is written by Pauli Virtanen, based on the autoreload
# code by Thomas Heller.

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
import atexit
import imp
import inspect
import os
import sys
import threading
import time
import traceback
import types
import weakref

try:
    # Reload is not defined by default in Python3.
    reload
except NameError:
    from imp import reload

from IPython.utils import pyfile
from IPython.utils.py3compat import PY3

#------------------------------------------------------------------------------
# Autoreload functionality
#------------------------------------------------------------------------------

def _get_compiled_ext():
    """Official way to get the extension of compiled files (.pyc or .pyo)"""
    for ext, mode, typ in imp.get_suffixes():
        if typ == imp.PY_COMPILED:
            return ext

PY_COMPILED_EXT = _get_compiled_ext()

class ModuleReloader(object):
    enabled = False
    """Whether this reloader is enabled"""

    failed = {}
    """Modules that failed to reload: {module: mtime-on-failed-reload, ...}"""

    modules = {}
    """Modules specially marked as autoreloadable."""

    skip_modules = {}
    """Modules specially marked as not autoreloadable."""

    check_all = True
    """Autoreload all modules, not just those listed in 'modules'"""

    old_objects = {}
    """(module-name, name) -> weakref, for replacing old code objects"""

    def mark_module_skipped(self, module_name):
        """Skip reloading the named module in the future"""
        try:
            del self.modules[module_name]
        except KeyError:
            pass
        self.skip_modules[module_name] = True

    def mark_module_reloadable(self, module_name):
        """Reload the named module in the future (if it is imported)"""
        try:
            del self.skip_modules[module_name]
        except KeyError:
            pass
        self.modules[module_name] = True

    def aimport_module(self, module_name):
        """Import a module, and mark it reloadable

        Returns
        -------
        top_module : module
            The imported module if it is top-level, or the top-level
        top_name : module
            Name of top_module

        """
        self.mark_module_reloadable(module_name)

        __import__(module_name)
        top_name = module_name.split('.')[0]
        top_module = sys.modules[top_name]
        return top_module, top_name

    def check(self, check_all=False):
        """Check whether some modules need to be reloaded."""

        if not self.enabled and not check_all:
            return

        if check_all or self.check_all:
            modules = sys.modules.keys()
        else:
            modules = self.modules.keys()

        for modname in modules:
            m = sys.modules.get(modname, None)

            if modname in self.skip_modules:
                continue

            if not hasattr(m, '__file__'):
                continue

            if m.__name__ == '__main__':
                # we cannot reload(__main__)
                continue

            filename = m.__file__
            path, ext = os.path.splitext(filename)

            if ext.lower() == '.py':
                ext = PY_COMPILED_EXT
                pyc_filename = pyfile.cache_from_source(filename)
                py_filename = filename
            else:
                pyc_filename = filename
                try:
                    py_filename = pyfile.source_from_cache(filename)
                except ValueError:
                    continue

            try:
                pymtime = os.stat(py_filename).st_mtime
                if pymtime <= os.stat(pyc_filename).st_mtime:
                    continue
                if self.failed.get(py_filename, None) == pymtime:
                    continue
            except OSError:
                continue

            try:
                superreload(m, reload, self.old_objects)
                if py_filename in self.failed:
                    del self.failed[py_filename]
            except:
                print >> sys.stderr, "[autoreload of %s failed: %s]" % (
                        modname, traceback.format_exc(1))
                self.failed[py_filename] = pymtime

#------------------------------------------------------------------------------
# superreload
#------------------------------------------------------------------------------

if PY3:
    func_attrs = ['__code__', '__defaults__', '__doc__',
                  '__closure__', '__globals__', '__dict__']
else:
    func_attrs = ['func_code', 'func_defaults', 'func_doc',
                  'func_closure', 'func_globals', 'func_dict']

def update_function(old, new):
    """Upgrade the code object of a function"""
    for name in func_attrs:
        try:
            setattr(old, name, getattr(new, name))
        except (AttributeError, TypeError):
            pass

def update_class(old, new):
    """Replace stuff in the __dict__ of a class, and upgrade
    method code objects"""
    for key in old.__dict__.keys():
        old_obj = getattr(old, key)

        try:
            new_obj = getattr(new, key)
        except AttributeError:
            # obsolete attribute: remove it
            try:
                delattr(old, key)
            except (AttributeError, TypeError):
                pass
            continue

        if update_generic(old_obj, new_obj): continue

        try:
            setattr(old, key, getattr(new, key))
        except (AttributeError, TypeError):
            pass # skip non-writable attributes

def update_property(old, new):
    """Replace get/set/del functions of a property"""
    update_generic(old.fdel, new.fdel)
    update_generic(old.fget, new.fget)
    update_generic(old.fset, new.fset)

def isinstance2(a, b, typ):
    return isinstance(a, typ) and isinstance(b, typ)

UPDATE_RULES = [
    (lambda a, b: isinstance2(a, b, type),
     update_class),
    (lambda a, b: isinstance2(a, b, types.FunctionType),
     update_function),
    (lambda a, b: isinstance2(a, b, property),
     update_property),
]

if PY3:
    UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType),
                          lambda a, b: update_function(a.__func__, b.__func__)),
                        ])
else:
    UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.ClassType),
                          update_class),
                         (lambda a, b: isinstance2(a, b, types.MethodType),
                          lambda a, b: update_function(a.im_func, b.im_func)),
                        ])

def update_generic(a, b):
    for type_check, update in UPDATE_RULES:
        if type_check(a, b):
            update(a, b)
            return True
    return False

class StrongRef(object):
    def __init__(self, obj):
        self.obj = obj
    def __call__(self):
        return self.obj

def superreload(module, reload=reload, old_objects={}):
    """Enhanced version of the builtin reload function.

    superreload remembers objects previously in the module, and

    - upgrades the class dictionary of every old class in the module
    - upgrades the code object of every old function and method
    - clears the module's namespace before reloading

    """

    # collect old objects in the module
    for name, obj in module.__dict__.items():
        if not hasattr(obj, '__module__') or obj.__module__ != module.__name__:
            continue
        key = (module.__name__, name)
        try:
            old_objects.setdefault(key, []).append(weakref.ref(obj))
        except TypeError:
            # weakref doesn't work for all types;
            # create strong references for 'important' cases
            if not PY3 and isinstance(obj, types.ClassType):
                old_objects.setdefault(key, []).append(StrongRef(obj))

    # reload module
    try:
        # clear namespace first from old cruft
        old_dict = module.__dict__.copy()
        old_name = module.__name__
        module.__dict__.clear()
        module.__dict__['__name__'] = old_name
        module.__dict__['__loader__'] = old_dict['__loader__']
    except (TypeError, AttributeError, KeyError):
        pass

    try:
        module = reload(module)
    except:
        # restore module dictionary on failed reload
        module.__dict__.update(old_dict)
        raise

    # iterate over all objects and update functions & classes
    for name, new_obj in module.__dict__.items():
        key = (module.__name__, name)
        if key not in old_objects: continue

        new_refs = []
        for old_ref in old_objects[key]:
            old_obj = old_ref()
            if old_obj is None: continue
            new_refs.append(old_ref)
            update_generic(old_obj, new_obj)

        if new_refs:
            old_objects[key] = new_refs
        else:
            del old_objects[key]

    return module

#------------------------------------------------------------------------------
# IPython connectivity
#------------------------------------------------------------------------------

from IPython.core.hooks import TryNext
from IPython.core.magic import Magics, magics_class, line_magic
from IPython.core.plugin import Plugin

@magics_class
class AutoreloadMagics(Magics):
    def __init__(self, *a, **kw):
        super(AutoreloadMagics, self).__init__(*a, **kw)
        self._reloader = ModuleReloader()
        self._reloader.check_all = False

    @line_magic
    def autoreload(self, parameter_s=''):
        r"""%autoreload => Reload modules automatically

        %autoreload
        Reload all modules (except those excluded by %aimport) automatically
        now.

        %autoreload 0
        Disable automatic reloading.

        %autoreload 1
        Reload all modules imported with %aimport every time before executing
        the Python code typed.

        %autoreload 2
        Reload all modules (except those excluded by %aimport) every time
        before executing the Python code typed.

        Reloading Python modules in a reliable way is in general
        difficult, and unexpected things may occur. %autoreload tries to
        work around common pitfalls by replacing function code objects and
        parts of classes previously in the module with new versions. This
        makes the following things to work:

        - Functions and classes imported via 'from xxx import foo' are upgraded
          to new versions when 'xxx' is reloaded.

        - Methods and properties of classes are upgraded on reload, so that
          calling 'c.foo()' on an object 'c' created before the reload causes
          the new code for 'foo' to be executed.

        Some of the known remaining caveats are:

        - Replacing code objects does not always succeed: changing a @property
          in a class to an ordinary method or a method to a member variable
          can cause problems (but in old objects only).

        - Functions that are removed (eg. via monkey-patching) from a module
          before it is reloaded are not upgraded.

        - C extension modules cannot be reloaded, and so cannot be
          autoreloaded.

        """
        if parameter_s == '':
            self._reloader.check(True)
        elif parameter_s == '0':
            self._reloader.enabled = False
        elif parameter_s == '1':
            self._reloader.check_all = False
            self._reloader.enabled = True
        elif parameter_s == '2':
            self._reloader.check_all = True
            self._reloader.enabled = True

    @line_magic
    def aimport(self, parameter_s='', stream=None):
        """%aimport => Import modules for automatic reloading.

        %aimport
        List modules to automatically import and not to import.

        %aimport foo
        Import module 'foo' and mark it to be autoreloaded for %autoreload 1

        %aimport -foo
        Mark module 'foo' to not be autoreloaded for %autoreload 1
        """
        modname = parameter_s
        if not modname:
            to_reload = self._reloader.modules.keys()
            to_reload.sort()
            to_skip = self._reloader.skip_modules.keys()
            to_skip.sort()
            if stream is None:
                stream = sys.stdout
            if self._reloader.check_all:
                stream.write("Modules to reload:\nall-except-skipped\n")
            else:
                stream.write("Modules to reload:\n%s\n" % ' '.join(to_reload))
            stream.write("\nModules to skip:\n%s\n" % ' '.join(to_skip))
        elif modname.startswith('-'):
            modname = modname[1:]
            self._reloader.mark_module_skipped(modname)
        else:
            top_module, top_name = self._reloader.aimport_module(modname)

            # Inject module to user namespace
            self.shell.push({top_name: top_module})

    def pre_run_code_hook(self, ip):
        if not self._reloader.enabled:
            raise TryNext
        try:
            self._reloader.check()
        except:
            pass

class AutoreloadPlugin(Plugin):
    def __init__(self, shell=None, config=None):
        super(AutoreloadPlugin, self).__init__(shell=shell, config=config)
        self.auto_magics = AutoreloadMagics(shell)
        shell.register_magics(self.auto_magics)
        shell.set_hook('pre_run_code_hook', self.auto_magics.pre_run_code_hook)

_loaded = False

def load_ipython_extension(ip):
    """Load the extension in IPython."""
    global _loaded
    if not _loaded:
        plugin = AutoreloadPlugin(shell=ip, config=ip.config)
        ip.plugin_manager.register_plugin('autoreload', plugin)
        _loaded = True


"""hooks for IPython.

In Python, it is possible to overwrite any method of any object if you really
want to.  But IPython exposes a few 'hooks', methods which are _designed_ to
be overwritten by users for customization purposes.  This module defines the
default versions of all such hooks, which get used by IPython if not
overridden by the user.

hooks are simple functions, but they should be declared with 'self' as their
first argument, because when activated they are registered into IPython as
instance methods.  The self argument will be the IPython running instance
itself, so hooks have full access to the entire IPython object.

If you wish to define a new hook and activate it, you need to put the
necessary code into a python file which can be either imported or execfile()'d
from within your profile's ipython_config.py configuration.

For example, suppose that you have a module called 'myiphooks' in your
PYTHONPATH, which contains the following definition:

import os
from IPython.core import ipapi
ip = ipapi.get()

def calljed(self,filename, linenum):
    "My editor hook calls the jed editor directly."
    print "Calling my own editor, jed ..."
    if os.system('jed +%d %s' % (linenum,filename)) != 0:
        raise TryNext()

ip.set_hook('editor', calljed)

You can then enable the functionality by doing 'import myiphooks'
somewhere in your configuration files or ipython command line.
"""

#*****************************************************************************
#       Copyright (C) 2005 Fernando Perez. <fperez@colorado.edu>
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#*****************************************************************************

import os
import subprocess
import sys

from IPython.core.error import TryNext

# List here all the default hooks.  For now it's just the editor functions
# but over time we'll move here all the public API for user-accessible things.

__all__ = ['editor', 'fix_error_editor', 'synchronize_with_editor',
           'input_prefilter', 'shutdown_hook', 'late_startup_hook',
           'show_in_pager','pre_prompt_hook',
           'pre_run_code_hook', 'clipboard_get']

def editor(self, filename, linenum=None, wait=True):
    """Open the default editor at the given filename and linenumber.

    This is IPython's default editor hook, you can use it as an example to
    write your own modified one.  To set your own editor function as the
    new editor hook, call ip.set_hook('editor',yourfunc)."""

    # IPython configures a default editor at startup by reading $EDITOR from
    # the environment, and falling back on vi (unix) or notepad (win32).
    editor = self.editor

    # marker for at which line to open the file (for existing objects)
    if linenum is None or editor=='notepad':
        linemark = ''
    else:
        linemark = '+%d' % int(linenum)

    # Enclose in quotes if necessary and legal
    if ' ' in editor and os.path.isfile(editor) and editor[0] != '"':
        editor = '"%s"' % editor

    # Call the actual editor
    proc = subprocess.Popen('%s %s %s' % (editor, linemark, filename),
                            shell=True)
    if wait and proc.wait() != 0:
        raise TryNext()

import tempfile
def fix_error_editor(self,filename,linenum,column,msg):
    """Open the editor at the given filename, linenumber, column and
    show an error message. This is used for correcting syntax errors.
    The current implementation only has special support for the VIM editor,
    and falls back on the 'editor' hook if VIM is not used.

    Call ip.set_hook('fix_error_editor',youfunc) to use your own function,
    """
    def vim_quickfix_file():
        t = tempfile.NamedTemporaryFile()
        t.write('%s:%d:%d:%s\n' % (filename,linenum,column,msg))
        t.flush()
        return t
    if os.path.basename(self.editor) != 'vim':
        self.hooks.editor(filename,linenum)
        return
    t = vim_quickfix_file()
    try:
        if os.system('vim --cmd "set errorformat=%f:%l:%c:%m" -q ' + t.name):
            raise TryNext()
    finally:
        t.close()

def synchronize_with_editor(self, filename, linenum, column):
        pass

class CommandChainDispatcher:
    """ Dispatch calls to a chain of commands until some func can handle it

    Usage: instantiate, execute "add" to add commands (with optional
    priority), execute normally via f() calling mechanism.

    """
    def __init__(self,commands=None):
        if commands is None:
            self.chain = []
        else:
            self.chain = commands

    def __call__(self,*args, **kw):
        """ Command chain is called just like normal func.

        This will call all funcs in chain with the same args as were given to
        this function, and return the result of first func that didn't raise
        TryNext"""
        last_exc = TryNext()
        for prio,cmd in self.chain:
            #print "prio",prio,"cmd",cmd #dbg
            try:
                return cmd(*args, **kw)
            except TryNext as exc:
                last_exc = exc
        # if no function will accept it, raise TryNext up to the caller
        raise last_exc

    def __str__(self):
        return str(self.chain)

    def add(self, func, priority=0):
        """ Add a func to the cmd chain with given priority """
        self.chain.append((priority, func))
        self.chain.sort(key=lambda x: x[0])

    def __iter__(self):
        """ Return all objects in chain.

        Handy if the objects are not callable.
        """
        return iter(self.chain)

def input_prefilter(self,line):
    """ Default input prefilter

    This returns the line as unchanged, so that the interpreter
    knows that nothing was done and proceeds with "classic" prefiltering
    (%magics, !shell commands etc.).

    Note that leading whitespace is not passed to this hook. Prefilter
    can't alter indentation.

    """
    #print "attempt to rewrite",line #dbg
    return line

def shutdown_hook(self):
    """ default shutdown hook

    Typically, shotdown hooks should raise TryNext so all shutdown ops are done
    """

    #print "default shutdown hook ok" # dbg
    return

def late_startup_hook(self):
    """ Executed after ipython has been constructed and configured

    """
    #print "default startup hook ok" # dbg

def show_in_pager(self,s):
    """ Run a string through pager """
    # raising TryNext here will use the default paging functionality
    raise TryNext

def pre_prompt_hook(self):
    """ Run before displaying the next prompt

    Use this e.g. to display output from asynchronous operations (in order
    to not mess up text entry)
    """

    return None

def pre_run_code_hook(self):
    """ Executed before running the (prefiltered) code in IPython """
    return None

def clipboard_get(self):
    """ Get text from the clipboard.
    """
    from IPython.lib.clipboard import (
        osx_clipboard_get, tkinter_clipboard_get,
        win32_clipboard_get
    )
    if sys.platform == 'win32':
        chain = [win32_clipboard_get, tkinter_clipboard_get]
    elif sys.platform == 'darwin':
        chain = [osx_clipboard_get, tkinter_clipboard_get]
    else:
        chain = [tkinter_clipboard_get]
    dispatcher = CommandChainDispatcher()
    for func in chain:
        dispatcher.add(func)
    text = dispatcher()
    return text


import os
from types import ModuleType

from pkginfo.bdist import BDist
from pkginfo.develop import Develop
from pkginfo.installed import Installed
from pkginfo.sdist import SDist
from pkginfo.wheel import Wheel

def get_metadata(path_or_module, metadata_version=None):
    """ Try to create a Distribution 'path_or_module'.

    o 'path_or_module' may be a module object.

    o If a string, 'path_or_module' may point to an sdist file, a bdist
      file, an installed package, or a working checkout (if it contains
      PKG-INFO).

    o Return None if 'path_or_module' can't be parsed.
    """
    if isinstance(path_or_module, ModuleType):
        try:
            return Installed(path_or_module, metadata_version)
        except (ValueError, IOError): #pragma NO COVER
            pass

    try:
        __import__(path_or_module)
    except ImportError:
        pass
    else:
        try:
            return Installed(path_or_module, metadata_version)
        except (ValueError, IOError): #pragma NO COVER
            pass

    if os.path.isfile(path_or_module):
        try:
            return SDist(path_or_module, metadata_version)
        except (ValueError, IOError):
            pass

        try:
            return BDist(path_or_module, metadata_version)
        except (ValueError, IOError): #pragma NO COVER
            pass

        try:
            return Wheel(path_or_module, metadata_version)
        except (ValueError, IOError): #pragma NO COVER
            pass

    if os.path.isdir(path_or_module):
        try:
            return Develop(path_or_module, metadata_version)
        except (ValueError, IOError): #pragma NO COVER
            pass


"""
This plugin provides ``--pdb`` and ``--pdb-failures`` options. The ``--pdb``
option will drop the test runner into pdb when it encounters an error. To
drop into pdb on failure, use ``--pdb-failures``.
"""

import pdb
from nose.plugins.base import Plugin

class Pdb(Plugin):
    """
    Provides --pdb and --pdb-failures options that cause the test runner to
    drop into pdb if it encounters an error or failure, respectively.
    """
    enabled_for_errors = False
    enabled_for_failures = False
    score = 5 # run last, among builtins

    def options(self, parser, env):
        """Register commandline options.
        """
        parser.add_option(
            "--pdb", action="store_true", dest="debugErrors",
            default=env.get('NOSE_PDB', False),
            help="Drop into debugger on errors")
        parser.add_option(
            "--pdb-failures", action="store_true",
            dest="debugFailures",
            default=env.get('NOSE_PDB_FAILURES', False),
            help="Drop into debugger on failures")

    def configure(self, options, conf):
        """Configure which kinds of exceptions trigger plugin.
        """
        self.conf = conf
        self.enabled = options.debugErrors or options.debugFailures
        self.enabled_for_errors = options.debugErrors
        self.enabled_for_failures = options.debugFailures

    def addError(self, test, err):
        """Enter pdb if configured to debug errors.
        """
        if not self.enabled_for_errors:
            return
        self.debug(err)

    def addFailure(self, test, err):
        """Enter pdb if configured to debug failures.
        """
        if not self.enabled_for_failures:
            return
        self.debug(err)

    def debug(self, err):
        import sys # FIXME why is this import here?
        ec, ev, tb = err
        stdout = sys.stdout
        sys.stdout = sys.__stdout__
        try:
            pdb.post_mortem(tb)
        finally:
            sys.stdout = stdout


import re
from django.utils.safestring import mark_safe
from django.utils.html import conditional_escape
from django.template import Library

register = Library()

@register.filter()
def obfuscate(email, linktext=None, autoescape=None):
    """
    Given a string representing an email address,
    returns a mailto link with rot13 JavaScript obfuscation.

    Accepts an optional argument to use as the link text;
    otherwise uses the email address itself.
    """
    if autoescape:
        esc = conditional_escape
    else:
        esc = lambda x: x

    email = re.sub('@', '\\\\100', re.sub('\.', '\\\\056', esc(email))).encode('rot13')

    if linktext:
        linktext = esc(linktext).encode('rot13')
    else:
        linktext = email

    rotten_link = """<script type="text/javascript">document.write \
        ("<n uers=\\\"znvygb:%s\\\">%s<\\057n>".replace(/[a-zA-Z]/g, \
        function(c){return String.fromCharCode((c<="Z"?90:122)>=\
        (c=c.charCodeAt(0)+13)?c:c-26);}));</script>""" % (email, linktext)
    return mark_safe(rotten_link)

obfuscate.needs_autoescape = True


import re
from django import template
from django.template import Node, TemplateSyntaxError
from django.utils.safestring import mark_safe

register = template.Library()

@register.filter
def roundplus(number):
    """
    given an number, this fuction rounds the number as the following examples:
    87 -> 87, 100 -> 100+, 188 -> 100+, 999 -> 900+, 1001 -> 1000+, ...etc
    """
    num = str(number)
    if not num.isdigit():
        return num

    num = str(number)
    digits = len(num)
    rounded = '100+'

    if digits < 3:
        rounded = num
    elif digits == 3:
        rounded = num[0] + '00+'
    elif digits == 4:
        rounded = num[0] + 'K+'
    elif digits == 5:
        rounded = num[:1] + 'K+'
    else:
        rounded = '100K+'

    return rounded