# encoding: utf-8
"""
Facilities for launching IPython processes asynchronously.

Authors:

* Brian Granger
* MinRK
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import copy
import logging
import os
import pipes
import stat
import sys
import time

# signal imports, handling various platforms, versions

from signal import SIGINT, SIGTERM
try:
    from signal import SIGKILL
except ImportError:
    # Windows
    SIGKILL=SIGTERM

try:
    # Windows >= 2.7, 3.2
    from signal import CTRL_C_EVENT as SIGINT
except ImportError:
    pass

from subprocess import Popen, PIPE, STDOUT
try:
    from subprocess import check_output
except ImportError:
    # pre-2.7, define check_output with Popen
    def check_output(*args, **kwargs):
        kwargs.update(dict(stdout=PIPE))
        p = Popen(*args, **kwargs)
        out,err = p.communicate()
        return out

from zmq.eventloop import ioloop

from IPython.config.application import Application
from IPython.config.configurable import LoggingConfigurable
from IPython.utils.text import EvalFormatter
from IPython.utils.traitlets import (
    Any, Integer, CFloat, List, Unicode, Dict, Instance, HasTraits, CRegExp
)
from IPython.utils.path import get_home_dir
from IPython.utils.process import find_cmd, FindCmdError

from .win32support import forward_read_events

from .winhpcjob import IPControllerTask, IPEngineTask, IPControllerJob, IPEngineSetJob

WINDOWS = os.name == 'nt'

#-----------------------------------------------------------------------------
# Paths to the kernel apps
#-----------------------------------------------------------------------------

cmd = "from IPython.parallel.apps.%s import launch_new_instance; launch_new_instance()"

ipcluster_cmd_argv = [sys.executable, "-c", cmd % "ipclusterapp"]

ipengine_cmd_argv = [sys.executable, "-c", cmd % "ipengineapp"]

ipcontroller_cmd_argv = [sys.executable, "-c", cmd % "ipcontrollerapp"]

#-----------------------------------------------------------------------------
# Base launchers and errors
#-----------------------------------------------------------------------------

class LauncherError(Exception):
    pass

class ProcessStateError(LauncherError):
    pass

class UnknownStatus(LauncherError):
    pass

class BaseLauncher(LoggingConfigurable):
    """An asbtraction for starting, stopping and signaling a process."""

    # In all of the launchers, the work_dir is where child processes will be
    # run. This will usually be the profile_dir, but may not be. any work_dir
    # passed into the __init__ method will override the config value.
    # This should not be used to set the work_dir for the actual engine
    # and controller. Instead, use their own config files or the
    # controller_args, engine_args attributes of the launchers to add
    # the work_dir option.
    work_dir = Unicode(u'.')
    loop = Instance('zmq.eventloop.ioloop.IOLoop')

    start_data = Any()
    stop_data = Any()

    def _loop_default(self):
        return ioloop.IOLoop.instance()

    def __init__(self, work_dir=u'.', config=None, **kwargs):
        super(BaseLauncher, self).__init__(work_dir=work_dir, config=config, **kwargs)
        self.state = 'before' # can be before, running, after
        self.stop_callbacks = []
        self.start_data = None
        self.stop_data = None

    @property
    def args(self):
        """A list of cmd and args that will be used to start the process.

        This is what is passed to :func:`spawnProcess` and the first element
        will be the process name.
        """
        return self.find_args()

    def find_args(self):
        """The ``.args`` property calls this to find the args list.

        Subcommand should implement this to construct the cmd and args.
        """
        raise NotImplementedError('find_args must be implemented in a subclass')

    @property
    def arg_str(self):
        """The string form of the program arguments."""
        return ' '.join(self.args)

    @property
    def running(self):
        """Am I running."""
        if self.state == 'running':
            return True
        else:
            return False

    def start(self):
        """Start the process."""
        raise NotImplementedError('start must be implemented in a subclass')

    def stop(self):
        """Stop the process and notify observers of stopping.

        This method will return None immediately.
        To observe the actual process stopping, see :meth:`on_stop`.
        """
        raise NotImplementedError('stop must be implemented in a subclass')

    def on_stop(self, f):
        """Register a callback to be called with this Launcher's stop_data
        when the process actually finishes.
        """
        if self.state=='after':
            return f(self.stop_data)
        else:
            self.stop_callbacks.append(f)

    def notify_start(self, data):
        """Call this to trigger startup actions.

        This logs the process startup and sets the state to 'running'.  It is
        a pass-through so it can be used as a callback.
        """

        self.log.debug('Process %r started: %r', self.args[0], data)
        self.start_data = data
        self.state = 'running'
        return data

    def notify_stop(self, data):
        """Call this to trigger process stop actions.

        This logs the process stopping and sets the state to 'after'. Call
        this to trigger callbacks registered via :meth:`on_stop`."""

        self.log.debug('Process %r stopped: %r', self.args[0], data)
        self.stop_data = data
        self.state = 'after'
        for i in range(len(self.stop_callbacks)):
            d = self.stop_callbacks.pop()
            d(data)
        return data

    def signal(self, sig):
        """Signal the process.

        Parameters
        ----------
        sig : str or int
            'KILL', 'INT', etc., or any signal number
        """
        raise NotImplementedError('signal must be implemented in a subclass')

class ClusterAppMixin(HasTraits):
    """MixIn for cluster args as traits"""
    profile_dir=Unicode('')
    cluster_id=Unicode('')

    @property
    def cluster_args(self):
        return ['--profile-dir', self.profile_dir, '--cluster-id', self.cluster_id]

class ControllerMixin(ClusterAppMixin):
    controller_cmd = List(ipcontroller_cmd_argv, config=True,
        help="""Popen command to launch ipcontroller.""")
    # Command line arguments to ipcontroller.
    controller_args = List(['--log-to-file','--log-level=%i' % logging.INFO], config=True,
        help="""command-line args to pass to ipcontroller""")

class EngineMixin(ClusterAppMixin):
    engine_cmd = List(ipengine_cmd_argv, config=True,
        help="""command to launch the Engine.""")
    # Command line arguments for ipengine.
    engine_args = List(['--log-to-file','--log-level=%i' % logging.INFO], config=True,
        help="command-line arguments to pass to ipengine"
    )

#-----------------------------------------------------------------------------
# Local process launchers
#-----------------------------------------------------------------------------

class LocalProcessLauncher(BaseLauncher):
    """Start and stop an external process in an asynchronous manner.

    This will launch the external process with a working directory of
    ``self.work_dir``.
    """

    # This is used to to construct self.args, which is passed to
    # spawnProcess.
    cmd_and_args = List([])
    poll_frequency = Integer(100) # in ms

    def __init__(self, work_dir=u'.', config=None, **kwargs):
        super(LocalProcessLauncher, self).__init__(
            work_dir=work_dir, config=config, **kwargs
        )
        self.process = None
        self.poller = None

    def find_args(self):
        return self.cmd_and_args

    def start(self):
        self.log.debug("Starting %s: %r", self.__class__.__name__, self.args)
        if self.state == 'before':
            self.process = Popen(self.args,
                stdout=PIPE,stderr=PIPE,stdin=PIPE,
                env=os.environ,
                cwd=self.work_dir
            )
            if WINDOWS:
                self.stdout = forward_read_events(self.process.stdout)
                self.stderr = forward_read_events(self.process.stderr)
            else:
                self.stdout = self.process.stdout.fileno()
                self.stderr = self.process.stderr.fileno()
            self.loop.add_handler(self.stdout, self.handle_stdout, self.loop.READ)
            self.loop.add_handler(self.stderr, self.handle_stderr, self.loop.READ)
            self.poller = ioloop.PeriodicCallback(self.poll, self.poll_frequency, self.loop)
            self.poller.start()
            self.notify_start(self.process.pid)
        else:
            s = 'The process was already started and has state: %r' % self.state
            raise ProcessStateError(s)

    def stop(self):
        return self.interrupt_then_kill()

    def signal(self, sig):
        if self.state == 'running':
            if WINDOWS and sig != SIGINT:
                # use Windows tree-kill for better child cleanup
                check_output(['taskkill', '-pid', str(self.process.pid), '-t', '-f'])
            else:
                self.process.send_signal(sig)

    def interrupt_then_kill(self, delay=2.0):
        """Send INT, wait a delay and then send KILL."""
        try:
            self.signal(SIGINT)
        except Exception:
            self.log.debug("interrupt failed")
            pass
        self.killer  = ioloop.DelayedCallback(lambda : self.signal(SIGKILL), delay*1000, self.loop)
        self.killer.start()

    # callbacks, etc:

    def handle_stdout(self, fd, events):
        if WINDOWS:
            line = self.stdout.recv()
        else:
            line = self.process.stdout.readline()
        # a stopped process will be readable but return empty strings
        if line:
            self.log.debug(line[:-1])
        else:
            self.poll()

    def handle_stderr(self, fd, events):
        if WINDOWS:
            line = self.stderr.recv()
        else:
            line = self.process.stderr.readline()
        # a stopped process will be readable but return empty strings
        if line:
            self.log.debug(line[:-1])
        else:
            self.poll()

    def poll(self):
        status = self.process.poll()
        if status is not None:
            self.poller.stop()
            self.loop.remove_handler(self.stdout)
            self.loop.remove_handler(self.stderr)
            self.notify_stop(dict(exit_code=status, pid=self.process.pid))
        return status

class LocalControllerLauncher(LocalProcessLauncher, ControllerMixin):
    """Launch a controller as a regular external process."""

    def find_args(self):
        return self.controller_cmd + self.cluster_args + self.controller_args

    def start(self):
        """Start the controller by profile_dir."""
        return super(LocalControllerLauncher, self).start()

class LocalEngineLauncher(LocalProcessLauncher, EngineMixin):
    """Launch a single engine as a regular externall process."""

    def find_args(self):
        return self.engine_cmd + self.cluster_args + self.engine_args

class LocalEngineSetLauncher(LocalEngineLauncher):
    """Launch a set of engines as regular external processes."""

    delay = CFloat(0.1, config=True,
        help="""delay (in seconds) between starting each engine after the first.
        This can help force the engines to get their ids in order, or limit
        process flood when starting many engines."""
    )

    # launcher class
    launcher_class = LocalEngineLauncher

    launchers = Dict()
    stop_data = Dict()

    def __init__(self, work_dir=u'.', config=None, **kwargs):
        super(LocalEngineSetLauncher, self).__init__(
            work_dir=work_dir, config=config, **kwargs
        )
        self.stop_data = {}

    def start(self, n):
        """Start n engines by profile or profile_dir."""
        dlist = []
        for i in range(n):
            if i > 0:
                time.sleep(self.delay)
            el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log,
                                    profile_dir=self.profile_dir, cluster_id=self.cluster_id,
            )

            # Copy the engine args over to each engine launcher.
            el.engine_cmd = copy.deepcopy(self.engine_cmd)
            el.engine_args = copy.deepcopy(self.engine_args)
            el.on_stop(self._notice_engine_stopped)
            d = el.start()
            self.launchers[i] = el
            dlist.append(d)
        self.notify_start(dlist)
        return dlist

    def find_args(self):
        return ['engine set']

    def signal(self, sig):
        dlist = []
        for el in self.launchers.itervalues():
            d = el.signal(sig)
            dlist.append(d)
        return dlist

    def interrupt_then_kill(self, delay=1.0):
        dlist = []
        for el in self.launchers.itervalues():
            d = el.interrupt_then_kill(delay)
            dlist.append(d)
        return dlist

    def stop(self):
        return self.interrupt_then_kill()

    def _notice_engine_stopped(self, data):
        pid = data['pid']
        for idx,el in self.launchers.iteritems():
            if el.process.pid == pid:
                break
        self.launchers.pop(idx)
        self.stop_data[idx] = data
        if not self.launchers:
            self.notify_stop(self.stop_data)

#-----------------------------------------------------------------------------
# MPI launchers
#-----------------------------------------------------------------------------

class MPILauncher(LocalProcessLauncher):
    """Launch an external process using mpiexec."""

    mpi_cmd = List(['mpiexec'], config=True,
        help="The mpiexec command to use in starting the process."
    )
    mpi_args = List([], config=True,
        help="The command line arguments to pass to mpiexec."
    )
    program = List(['date'],
        help="The program to start via mpiexec.")
    program_args = List([],
        help="The command line argument to the program."
    )
    n = Integer(1)

    def __init__(self, *args, **kwargs):
        # deprecation for old MPIExec names:
        config = kwargs.get('config', {})
        for oldname in ('MPIExecLauncher', 'MPIExecControllerLauncher', 'MPIExecEngineSetLauncher'):
            deprecated = config.get(oldname)
            if deprecated:
                newname = oldname.replace('MPIExec', 'MPI')
                config[newname].update(deprecated)
                self.log.warn("WARNING: %s name has been deprecated, use %s", oldname, newname)

        super(MPILauncher, self).__init__(*args, **kwargs)

    def find_args(self):
        """Build self.args using all the fields."""
        return self.mpi_cmd + ['-n', str(self.n)] + self.mpi_args + \
               self.program + self.program_args

    def start(self, n):
        """Start n instances of the program using mpiexec."""
        self.n = n
        return super(MPILauncher, self).start()

class MPIControllerLauncher(MPILauncher, ControllerMixin):
    """Launch a controller using mpiexec."""

    # alias back to *non-configurable* program[_args] for use in find_args()
    # this way all Controller/EngineSetLaunchers have the same form, rather
    # than *some* having `program_args` and others `controller_args`
    @property
    def program(self):
        return self.controller_cmd

    @property
    def program_args(self):
        return self.cluster_args + self.controller_args

    def start(self):
        """Start the controller by profile_dir."""
        return super(MPIControllerLauncher, self).start(1)

class MPIEngineSetLauncher(MPILauncher, EngineMixin):
    """Launch engines using mpiexec"""

    # alias back to *non-configurable* program[_args] for use in find_args()
    # this way all Controller/EngineSetLaunchers have the same form, rather
    # than *some* having `program_args` and others `controller_args`
    @property
    def program(self):
        return self.engine_cmd

    @property
    def program_args(self):
        return self.cluster_args + self.engine_args

    def start(self, n):
        """Start n engines by profile or profile_dir."""
        self.n = n
        return super(MPIEngineSetLauncher, self).start(n)

# deprecated MPIExec names
class DeprecatedMPILauncher(object):
    def warn(self):
        oldname = self.__class__.__name__
        newname = oldname.replace('MPIExec', 'MPI')
        self.log.warn("WARNING: %s name is deprecated, use %s", oldname, newname)

class MPIExecLauncher(MPILauncher, DeprecatedMPILauncher):
    """Deprecated, use MPILauncher"""
    def __init__(self, *args, **kwargs):
        super(MPIExecLauncher, self).__init__(*args, **kwargs)
        self.warn()

class MPIExecControllerLauncher(MPIControllerLauncher, DeprecatedMPILauncher):
    """Deprecated, use MPIControllerLauncher"""
    def __init__(self, *args, **kwargs):
        super(MPIExecControllerLauncher, self).__init__(*args, **kwargs)
        self.warn()

class MPIExecEngineSetLauncher(MPIEngineSetLauncher, DeprecatedMPILauncher):
    """Deprecated, use MPIEngineSetLauncher"""
    def __init__(self, *args, **kwargs):
        super(MPIExecEngineSetLauncher, self).__init__(*args, **kwargs)
        self.warn()

#-----------------------------------------------------------------------------
# SSH launchers
#-----------------------------------------------------------------------------

# TODO: Get SSH Launcher back to level of sshx in 0.10.2

class SSHLauncher(LocalProcessLauncher):
    """A minimal launcher for ssh.

    To be useful this will probably have to be extended to use the ``sshx``
    idea for environment variables.  There could be other things this needs
    as well.
    """

    ssh_cmd = List(['ssh'], config=True,
        help="command for starting ssh")
    ssh_args = List(['-tt'], config=True,
        help="args to pass to ssh")
    scp_cmd = List(['scp'], config=True,
        help="command for sending files")
    program = List(['date'],
        help="Program to launch via ssh")
    program_args = List([],
        help="args to pass to remote program")
    hostname = Unicode('', config=True,
        help="hostname on which to launch the program")
    user = Unicode('', config=True,
        help="username for ssh")
    location = Unicode('', config=True,
        help="user@hostname location for ssh in one setting")
    to_fetch = List([], config=True,
        help="List of (remote, local) files to fetch after starting")
    to_send = List([], config=True,
        help="List of (local, remote) files to send before starting")

    def _hostname_changed(self, name, old, new):
        if self.user:
            self.location = u'%s@%s' % (self.user, new)
        else:
            self.location = new

    def _user_changed(self, name, old, new):
        self.location = u'%s@%s' % (new, self.hostname)

    def find_args(self):
        return self.ssh_cmd + self.ssh_args + [self.location] + \
               list(map(pipes.quote, self.program + self.program_args))

    def _send_file(self, local, remote):
        """send a single file"""
        remote = "%s:%s" % (self.location, remote)
        for i in range(10):
            if not os.path.exists(local):
                self.log.debug("waiting for %s" % local)
                time.sleep(1)
            else:
                break
        self.log.info("sending %s to %s", local, remote)
        check_output(self.scp_cmd + [local, remote])

    def send_files(self):
        """send our files (called before start)"""
        if not self.to_send:
            return
        for local_file, remote_file in self.to_send:
            self._send_file(local_file, remote_file)

    def _fetch_file(self, remote, local):
        """fetch a single file"""
        full_remote = "%s:%s" % (self.location, remote)
        self.log.info("fetching %s from %s", local, full_remote)
        for i in range(10):
            # wait up to 10s for remote file to exist
            check = check_output(self.ssh_cmd + self.ssh_args + \
                [self.location, 'test -e', remote, "&& echo 'yes' || echo 'no'"])
            check = check.strip()
            if check == 'no':
                time.sleep(1)
            elif check == 'yes':
                break
        check_output(self.scp_cmd + [full_remote, local])

    def fetch_files(self):
        """fetch remote files (called after start)"""
        if not self.to_fetch:
            return
        for remote_file, local_file in self.to_fetch:
            self._fetch_file(remote_file, local_file)

    def start(self, hostname=None, user=None):
        if hostname is not None:
            self.hostname = hostname
        if user is not None:
            self.user = user

        self.send_files()
        super(SSHLauncher, self).start()
        self.fetch_files()

    def signal(self, sig):
        if self.state == 'running':
            # send escaped ssh connection-closer
            self.process.stdin.write('~.')
            self.process.stdin.flush()

class SSHClusterLauncher(SSHLauncher):

    remote_profile_dir = Unicode('', config=True,
        help="""The remote profile_dir to use.

        If not specified, use calling profile, stripping out possible leading homedir.
        """)

    def _remote_profile_dir_default(self):
        """turns /home/you/.ipython/profile_foo into .ipython/profile_foo
        """
        home = get_home_dir()
        if not home.endswith('/'):
            home = home+'/'

        if self.profile_dir.startswith(home):
            return self.profile_dir[len(home):]
        else:
            return self.profile_dir

    def _cluster_id_changed(self, name, old, new):
        if new:
            raise ValueError("cluster id not supported by SSH launchers")

    @property
    def cluster_args(self):
        return ['--profile-dir', self.remote_profile_dir]

class SSHControllerLauncher(SSHClusterLauncher, ControllerMixin):

    # alias back to *non-configurable* program[_args] for use in find_args()
    # this way all Controller/EngineSetLaunchers have the same form, rather
    # than *some* having `program_args` and others `controller_args`

    def _controller_cmd_default(self):
        return ['ipcontroller']

    @property
    def program(self):
        return self.controller_cmd

    @property
    def program_args(self):
        return self.cluster_args + self.controller_args

    def _to_fetch_default(self):
        return [
            (os.path.join(self.remote_profile_dir, 'security', cf),
             os.path.join(self.profile_dir, 'security', cf),)
            for cf in ('ipcontroller-client.json', 'ipcontroller-engine.json')
        ]

class SSHEngineLauncher(SSHClusterLauncher, EngineMixin):

    # alias back to *non-configurable* program[_args] for use in find_args()
    # this way all Controller/EngineSetLaunchers have the same form, rather
    # than *some* having `program_args` and others `controller_args`

    def _engine_cmd_default(self):
        return ['ipengine']

    @property
    def program(self):
        return self.engine_cmd

    @property
    def program_args(self):
        return self.cluster_args + self.engine_args

    def _to_send_default(self):
        return [
            (os.path.join(self.profile_dir, 'security', cf),
             os.path.join(self.remote_profile_dir, 'security', cf))
            for cf in ('ipcontroller-client.json', 'ipcontroller-engine.json')
        ]

class SSHEngineSetLauncher(LocalEngineSetLauncher):
    launcher_class = SSHEngineLauncher
    engines = Dict(config=True,
        help="""dict of engines to launch.  This is a dict by hostname of ints,
        corresponding to the number of engines to start on that host.""")

    def _engine_cmd_default(self):
        return ['ipengine']

    @property
    def engine_count(self):
        """determine engine count from `engines` dict"""
        count = 0
        for n in self.engines.itervalues():
            if isinstance(n, (tuple,list)):
                n,args = n
            count += n
        return count

    def start(self, n):
        """Start engines by profile or profile_dir.
        `n` is ignored, and the `engines` config property is used instead.
        """

        dlist = []
        for host, n in self.engines.iteritems():
            if isinstance(n, (tuple, list)):
                n, args = n
            else:
                args = copy.deepcopy(self.engine_args)

            if '@' in host:
                user,host = host.split('@',1)
            else:
                user=None
            for i in range(n):
                if i > 0:
                    time.sleep(self.delay)
                el = self.launcher_class(work_dir=self.work_dir, config=self.config, log=self.log,
                                        profile_dir=self.profile_dir, cluster_id=self.cluster_id,
                )
                if i > 0:
                    # only send files for the first engine on each host
                    el.to_send = []

                # Copy the engine args over to each engine launcher.
                el.engine_cmd = self.engine_cmd
                el.engine_args = args
                el.on_stop(self._notice_engine_stopped)
                d = el.start(user=user, hostname=host)
                self.launchers[ "%s/%i" % (host,i) ] = el
                dlist.append(d)
        self.notify_start(dlist)
        return dlist

class SSHProxyEngineSetLauncher(SSHClusterLauncher):
    """Launcher for calling
    `ipcluster engines` on a remote machine.

    Requires that remote profile is already configured.
    """

    n = Integer()
    ipcluster_cmd = List(['ipcluster'], config=True)

    @property
    def program(self):
        return self.ipcluster_cmd + ['engines']

    @property
    def program_args(self):
        return ['-n', str(self.n), '--profile-dir', self.remote_profile_dir]

    def _to_send_default(self):
        return [
            (os.path.join(self.profile_dir, 'security', cf),
             os.path.join(self.remote_profile_dir, 'security', cf))
            for cf in ('ipcontroller-client.json', 'ipcontroller-engine.json')
    ]

    def start(self, n):
        self.n = n
        super(SSHProxyEngineSetLauncher, self).start()

#-----------------------------------------------------------------------------
# Windows HPC Server 2008 scheduler launchers
#-----------------------------------------------------------------------------

# This is only used on Windows.
def find_job_cmd():
    if WINDOWS:
        try:
            return find_cmd('job')
        except (FindCmdError, ImportError):
            # ImportError will be raised if win32api is not installed
            return 'job'
    else:
        return 'job'

class WindowsHPCLauncher(BaseLauncher):

    job_id_regexp = CRegExp(r'\d+', config=True,
        help="""A regular expression used to get the job id from the output of the
        submit_command. """
        )
    job_file_name = Unicode(u'ipython_job.xml', config=True,
        help="The filename of the instantiated job script.")
    # The full path to the instantiated job script. This gets made dynamically
    # by combining the work_dir with the job_file_name.
    job_file = Unicode(u'')
    scheduler = Unicode('', config=True,
        help="The hostname of the scheduler to submit the job to.")
    job_cmd = Unicode(find_job_cmd(), config=True,
        help="The command for submitting jobs.")

    def __init__(self, work_dir=u'.', config=None, **kwargs):
        super(WindowsHPCLauncher, self).__init__(
            work_dir=work_dir, config=config, **kwargs
        )

    @property
    def job_file(self):
        return os.path.join(self.work_dir, self.job_file_name)

    def write_job_file(self, n):
        raise NotImplementedError("Implement write_job_file in a subclass.")

    def find_args(self):
        return [u'job.exe']

    def parse_job_id(self, output):
        """Take the output of the submit command and return the job id."""
        m = self.job_id_regexp.search(output)
        if m is not None:
            job_id = m.group()
        else:
            raise LauncherError("Job id couldn't be determined: %s" % output)
        self.job_id = job_id
        self.log.info('Job started with id: %r', job_id)
        return job_id

    def start(self, n):
        """Start n copies of the process using the Win HPC job scheduler."""
        self.write_job_file(n)
        args = [
            'submit',
            '/jobfile:%s' % self.job_file,
            '/scheduler:%s' % self.scheduler
        ]
        self.log.debug("Starting Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))

        output = check_output([self.job_cmd]+args,
            env=os.environ,
            cwd=self.work_dir,
            stderr=STDOUT
        )
        job_id = self.parse_job_id(output)
        self.notify_start(job_id)
        return job_id

    def stop(self):
        args = [
            'cancel',
            self.job_id,
            '/scheduler:%s' % self.scheduler
        ]
        self.log.info("Stopping Win HPC Job: %s" % (self.job_cmd + ' ' + ' '.join(args),))
        try:
            output = check_output([self.job_cmd]+args,
                env=os.environ,
                cwd=self.work_dir,
                stderr=STDOUT
            )
        except:
            output = 'The job already appears to be stoppped: %r' % self.job_id
        self.notify_stop(dict(job_id=self.job_id, output=output))  # Pass the output of the kill cmd
        return output

class WindowsHPCControllerLauncher(WindowsHPCLauncher, ClusterAppMixin):

    job_file_name = Unicode(u'ipcontroller_job.xml', config=True,
        help="WinHPC xml job file.")
    controller_args = List([], config=False,
        help="extra args to pass to ipcontroller")

    def write_job_file(self, n):
        job = IPControllerJob(config=self.config)

        t = IPControllerTask(config=self.config)
        # The tasks work directory is *not* the actual work directory of
        # the controller. It is used as the base path for the stdout/stderr
        # files that the scheduler redirects to.
        t.work_directory = self.profile_dir
        # Add the profile_dir and from self.start().
        t.controller_args.extend(self.cluster_args)
        t.controller_args.extend(self.controller_args)
        job.add_task(t)

        self.log.debug("Writing job description file: %s", self.job_file)
        job.write(self.job_file)

    @property
    def job_file(self):
        return os.path.join(self.profile_dir, self.job_file_name)

    def start(self):
        """Start the controller by profile_dir."""
        return super(WindowsHPCControllerLauncher, self).start(1)

class WindowsHPCEngineSetLauncher(WindowsHPCLauncher, ClusterAppMixin):

    job_file_name = Unicode(u'ipengineset_job.xml', config=True,
        help="jobfile for ipengines job")
    engine_args = List([], config=False,
        help="extra args to pas to ipengine")

    def write_job_file(self, n):
        job = IPEngineSetJob(config=self.config)

        for i in range(n):
            t = IPEngineTask(config=self.config)
            # The tasks work directory is *not* the actual work directory of
            # the engine. It is used as the base path for the stdout/stderr
            # files that the scheduler redirects to.
            t.work_directory = self.profile_dir
            # Add the profile_dir and from self.start().
            t.engine_args.extend(self.cluster_args)
            t.engine_args.extend(self.engine_args)
            job.add_task(t)

        self.log.debug("Writing job description file: %s", self.job_file)
        job.write(self.job_file)

    @property
    def job_file(self):
        return os.path.join(self.profile_dir, self.job_file_name)

    def start(self, n):
        """Start the controller by profile_dir."""
        return super(WindowsHPCEngineSetLauncher, self).start(n)

#-----------------------------------------------------------------------------
# Batch (PBS) system launchers
#-----------------------------------------------------------------------------

class BatchClusterAppMixin(ClusterAppMixin):
    """ClusterApp mixin that updates the self.context dict, rather than cl-args."""
    def _profile_dir_changed(self, name, old, new):
        self.context[name] = new
    _cluster_id_changed = _profile_dir_changed

    def _profile_dir_default(self):
        self.context['profile_dir'] = ''
        return ''
    def _cluster_id_default(self):
        self.context['cluster_id'] = ''
        return ''

class BatchSystemLauncher(BaseLauncher):
    """Launch an external process using a batch system.

    This class is designed to work with UNIX batch systems like PBS, LSF,
    GridEngine, etc.  The overall model is that there are different commands
    like qsub, qdel, etc. that handle the starting and stopping of the process.

    This class also has the notion of a batch script. The ``batch_template``
    attribute can be set to a string that is a template for the batch script.
    This template is instantiated using string formatting. Thus the template can
    use {n} fot the number of instances. Subclasses can add additional variables
    to the template dict.
    """

    # Subclasses must fill these in.  See PBSEngineSet
    submit_command = List([''], config=True,
        help="The name of the command line program used to submit jobs.")
    delete_command = List([''], config=True,
        help="The name of the command line program used to delete jobs.")
    job_id_regexp = CRegExp('', config=True,
        help="""A regular expression used to get the job id from the output of the
        submit_command.""")
    batch_template = Unicode('', config=True,
        help="The string that is the batch script template itself.")
    batch_template_file = Unicode(u'', config=True,
        help="The file that contains the batch template.")
    batch_file_name = Unicode(u'batch_script', config=True,
        help="The filename of the instantiated batch script.")
    queue = Unicode(u'', config=True,
        help="The PBS Queue.")

    def _queue_changed(self, name, old, new):
        self.context[name] = new

    n = Integer(1)
    _n_changed = _queue_changed

    # not configurable, override in subclasses
    # PBS Job Array regex
    job_array_regexp = CRegExp('')
    job_array_template = Unicode('')
    # PBS Queue regex
    queue_regexp = CRegExp('')
    queue_template = Unicode('')
    # The default batch template, override in subclasses
    default_template = Unicode('')
    # The full path to the instantiated batch script.
    batch_file = Unicode(u'')
    # the format dict used with batch_template:
    context = Dict()
    def _context_default(self):
        """load the default context with the default values for the basic keys

        because the _trait_changed methods only load the context if they
        are set to something other than the default value.
        """
        return dict(n=1, queue=u'', profile_dir=u'', cluster_id=u'')

    # the Formatter instance for rendering the templates:
    formatter = Instance(EvalFormatter, (), {})

    def find_args(self):
        return self.submit_command + [self.batch_file]

    def __init__(self, work_dir=u'.', config=None, **kwargs):
        super(BatchSystemLauncher, self).__init__(
            work_dir=work_dir, config=config, **kwargs
        )
        self.batch_file = os.path.join(self.work_dir, self.batch_file_name)

    def parse_job_id(self, output):
        """Take the output of the submit command and return the job id."""
        m = self.job_id_regexp.search(output)
        if m is not None:
            job_id = m.group()
        else:
            raise LauncherError("Job id couldn't be determined: %s" % output)
        self.job_id = job_id
        self.log.info('Job submitted with job id: %r', job_id)
        return job_id

    def write_batch_script(self, n):
        """Instantiate and write the batch script to the work_dir."""
        self.n = n
        # first priority is batch_template if set
        if self.batch_template_file and not self.batch_template:
            # second priority is batch_template_file
            with open(self.batch_template_file) as f:
                self.batch_template = f.read()
        if not self.batch_template:
            # third (last) priority is default_template
            self.batch_template = self.default_template

            # add jobarray or queue lines to user-specified template
            # note that this is *only* when user did not specify a template.
            # print self.job_array_regexp.search(self.batch_template)
            if not self.job_array_regexp.search(self.batch_template):
                self.log.debug("adding job array settings to batch script")
                firstline, rest = self.batch_template.split('\n',1)
                self.batch_template = u'\n'.join([firstline, self.job_array_template, rest])

            # print self.queue_regexp.search(self.batch_template)
            if self.queue and not self.queue_regexp.search(self.batch_template):
                self.log.debug("adding PBS queue settings to batch script")
                firstline, rest = self.batch_template.split('\n',1)
                self.batch_template = u'\n'.join([firstline, self.queue_template, rest])

        script_as_string = self.formatter.format(self.batch_template, **self.context)
        self.log.debug('Writing batch script: %s', self.batch_file)

        with open(self.batch_file, 'w') as f:
            f.write(script_as_string)
        os.chmod(self.batch_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)

    def start(self, n):
        """Start n copies of the process using a batch system."""
        self.log.debug("Starting %s: %r", self.__class__.__name__, self.args)
        # Here we save profile_dir in the context so they
        # can be used in the batch script template as {profile_dir}
        self.write_batch_script(n)
        output = check_output(self.args, env=os.environ)

        job_id = self.parse_job_id(output)
        self.notify_start(job_id)
        return job_id

    def stop(self):
        output = check_output(self.delete_command+[self.job_id], env=os.environ)
        self.notify_stop(dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd
        return output

class PBSLauncher(BatchSystemLauncher):
    """A BatchSystemLauncher subclass for PBS."""

    submit_command = List(['qsub'], config=True,
        help="The PBS submit command ['qsub']")
    delete_command = List(['qdel'], config=True,
        help="The PBS delete command ['qsub']")
    job_id_regexp = CRegExp(r'\d+', config=True,
        help="Regular expresion for identifying the job ID [r'\d+']")

    batch_file = Unicode(u'')
    job_array_regexp = CRegExp('#PBS\W+-t\W+[\w\d\-\$]+')
    job_array_template = Unicode('#PBS -t 1-{n}')
    queue_regexp = CRegExp('#PBS\W+-q\W+\$?\w+')
    queue_template = Unicode('#PBS -q {queue}')

class PBSControllerLauncher(PBSLauncher, BatchClusterAppMixin):
    """Launch a controller using PBS."""

    batch_file_name = Unicode(u'pbs_controller', config=True,
        help="batch file name for the controller job.")
    default_template= Unicode("""#!/bin/sh
#PBS -V
#PBS -N ipcontroller
%s --log-to-file --profile-dir="{profile_dir}" --cluster-id="{cluster_id}"
"""%(' '.join(map(pipes.quote, ipcontroller_cmd_argv))))

    def start(self):
        """Start the controller by profile or profile_dir."""
        return super(PBSControllerLauncher, self).start(1)

class PBSEngineSetLauncher(PBSLauncher, BatchClusterAppMixin):
    """Launch Engines using PBS"""
    batch_file_name = Unicode(u'pbs_engines', config=True,
        help="batch file name for the engine(s) job.")
    default_template= Unicode(u"""#!/bin/sh
#PBS -V
#PBS -N ipengine
%s --profile-dir="{profile_dir}" --cluster-id="{cluster_id}"
"""%(' '.join(map(pipes.quote,ipengine_cmd_argv))))

    def start(self, n):
        """Start n engines by profile or profile_dir."""
        return super(PBSEngineSetLauncher, self).start(n)

#SGE is very similar to PBS

class SGELauncher(PBSLauncher):
    """Sun GridEngine is a PBS clone with slightly different syntax"""
    job_array_regexp = CRegExp('#\$\W+\-t')
    job_array_template = Unicode('#$ -t 1-{n}')
    queue_regexp = CRegExp('#\$\W+-q\W+\$?\w+')
    queue_template = Unicode('#$ -q {queue}')

class SGEControllerLauncher(SGELauncher, BatchClusterAppMixin):
    """Launch a controller using SGE."""

    batch_file_name = Unicode(u'sge_controller', config=True,
        help="batch file name for the ipontroller job.")
    default_template= Unicode(u"""#$ -V
#$ -S /bin/sh
#$ -N ipcontroller
%s --log-to-file --profile-dir="{profile_dir}" --cluster-id="{cluster_id}"
"""%(' '.join(map(pipes.quote, ipcontroller_cmd_argv))))

    def start(self):
        """Start the controller by profile or profile_dir."""
        return super(SGEControllerLauncher, self).start(1)

class SGEEngineSetLauncher(SGELauncher, BatchClusterAppMixin):
    """Launch Engines with SGE"""
    batch_file_name = Unicode(u'sge_engines', config=True,
        help="batch file name for the engine(s) job.")
    default_template = Unicode("""#$ -V
#$ -S /bin/sh
#$ -N ipengine
%s --profile-dir="{profile_dir}" --cluster-id="{cluster_id}"
"""%(' '.join(map(pipes.quote, ipengine_cmd_argv))))

    def start(self, n):
        """Start n engines by profile or profile_dir."""
        return super(SGEEngineSetLauncher, self).start(n)

# LSF launchers

class LSFLauncher(BatchSystemLauncher):
    """A BatchSystemLauncher subclass for LSF."""

    submit_command = List(['bsub'], config=True,
                          help="The PBS submit command ['bsub']")
    delete_command = List(['bkill'], config=True,
                          help="The PBS delete command ['bkill']")
    job_id_regexp = CRegExp(r'\d+', config=True,
                            help="Regular expresion for identifying the job ID [r'\d+']")

    batch_file = Unicode(u'')
    job_array_regexp = CRegExp('#BSUB[ \t]-J+\w+\[\d+-\d+\]')
    job_array_template = Unicode('#BSUB -J ipengine[1-{n}]')
    queue_regexp = CRegExp('#BSUB[ \t]+-q[ \t]+\w+')
    queue_template = Unicode('#BSUB -q {queue}')

    def start(self, n):
        """Start n copies of the process using LSF batch system.
        This cant inherit from the base class because bsub expects
        to be piped a shell script in order to honor the #BSUB directives :
        bsub < script
        """
        # Here we save profile_dir in the context so they
        # can be used in the batch script template as {profile_dir}
        self.write_batch_script(n)
        #output = check_output(self.args, env=os.environ)
        piped_cmd = self.args[0]+'<\"'+self.args[1]+'\"'
        self.log.debug("Starting %s: %s", self.__class__.__name__, piped_cmd)
        p = Popen(piped_cmd, shell=True,env=os.environ,stdout=PIPE)
        output,err = p.communicate()
        job_id = self.parse_job_id(output)
        self.notify_start(job_id)
        return job_id

class LSFControllerLauncher(LSFLauncher, BatchClusterAppMixin):
    """Launch a controller using LSF."""

    batch_file_name = Unicode(u'lsf_controller', config=True,
                              help="batch file name for the controller job.")
    default_template= Unicode("""#!/bin/sh
    #BSUB -J ipcontroller
    #BSUB -oo ipcontroller.o.%%J
    #BSUB -eo ipcontroller.e.%%J
    %s --log-to-file --profile-dir="{profile_dir}" --cluster-id="{cluster_id}"
    """%(' '.join(map(pipes.quote,ipcontroller_cmd_argv))))

    def start(self):
        """Start the controller by profile or profile_dir."""
        return super(LSFControllerLauncher, self).start(1)

class LSFEngineSetLauncher(LSFLauncher, BatchClusterAppMixin):
    """Launch Engines using LSF"""
    batch_file_name = Unicode(u'lsf_engines', config=True,
                              help="batch file name for the engine(s) job.")
    default_template= Unicode(u"""#!/bin/sh
    #BSUB -oo ipengine.o.%%J
    #BSUB -eo ipengine.e.%%J
    %s --profile-dir="{profile_dir}" --cluster-id="{cluster_id}"
    """%(' '.join(map(pipes.quote, ipengine_cmd_argv))))

    def start(self, n):
        """Start n engines by profile or profile_dir."""
        return super(LSFEngineSetLauncher, self).start(n)

#-----------------------------------------------------------------------------
# A launcher for ipcluster itself!
#-----------------------------------------------------------------------------

class IPClusterLauncher(LocalProcessLauncher):
    """Launch the ipcluster program in an external process."""

    ipcluster_cmd = List(ipcluster_cmd_argv, config=True,
        help="Popen command for ipcluster")
    ipcluster_args = List(
        ['--clean-logs=True', '--log-to-file', '--log-level=%i'%logging.INFO], config=True,
        help="Command line arguments to pass to ipcluster.")
    ipcluster_subcommand = Unicode('start')
    profile = Unicode('default')
    n = Integer(2)

    def find_args(self):
        return self.ipcluster_cmd + [self.ipcluster_subcommand] + \
            ['--n=%i'%self.n, '--profile=%s'%self.profile] + \
            self.ipcluster_args

    def start(self):
        return super(IPClusterLauncher, self).start()

#-----------------------------------------------------------------------------
# Collections of launchers
#-----------------------------------------------------------------------------

local_launchers = [
    LocalControllerLauncher,
    LocalEngineLauncher,
    LocalEngineSetLauncher,
]
mpi_launchers = [
    MPILauncher,
    MPIControllerLauncher,
    MPIEngineSetLauncher,
]
ssh_launchers = [
    SSHLauncher,
    SSHControllerLauncher,
    SSHEngineLauncher,
    SSHEngineSetLauncher,
]
winhpc_launchers = [
    WindowsHPCLauncher,
    WindowsHPCControllerLauncher,
    WindowsHPCEngineSetLauncher,
]
pbs_launchers = [
    PBSLauncher,
    PBSControllerLauncher,
    PBSEngineSetLauncher,
]
sge_launchers = [
    SGELauncher,
    SGEControllerLauncher,
    SGEEngineSetLauncher,
]
lsf_launchers = [
    LSFLauncher,
    LSFControllerLauncher,
    LSFEngineSetLauncher,
]
all_launchers = local_launchers + mpi_launchers + ssh_launchers + winhpc_launchers\
                + pbs_launchers + sge_launchers + lsf_launchers


import os
import stat
import pwd
import ConfigParser
import re
import time
from gurumate.base import fail
from stat import *
from pwd import getpwuid

def check_isfile(filepath):
    try:
        st = os.stat(filepath)
    except OSError:
        fail("no file named (%s) in the system" % filepath)

    if not stat.S_ISREG(st.st_mode):
        fail("%s is not a regular file." % filepath)

def check_isdir(dirpath):
    try:
        st = os.stat(dirpath)
    except OSError:
        fail("no directory named (%s) in the system" % dirpath)

    if not stat.S_ISDIR(st.st_mode):
        fail("%s is not a directory." % dirpath)

def check_readaccess(filepath, user_name):
    try:
        st = os.stat(filepath)
    except OSError:
        fail("no file named (%s) in the system" % filepath)

    mode = st.st_mode
    # Others
    if mode & stat.S_IROTH:
        return

    try:
        passdb_entry = pwd.getpwnam(user_name)
    except KeyError:
        fail("no user called (%s) on the system" % user_name)
    user_id = passdb_entry.pw_uid
    group_id = passdb_entry.pw_gid

    owner_uid = st.st_uid
    owner_gid = st.st_gid

    # Group
    if owner_gid == group_id:
        if mode & stat.S_IRGRP:
            return

    # User
    if owner_uid == user_id:
        if mode & stat.S_IRUSR:
            return

    fail("user (%s) has no read access on file (%s)" % (user_name, filepath))

def check_writeaccess(filepath, user_name):
    try:
        st = os.stat(filepath)
    except OSError:
        fail("no file name (%s) in the system" % filepath)

    mode = st.st_mode

    # Others
    if mode & stat.S_IWOTH:
        return

    try:
        passdb_entry = pwd.getpwnam(user_name)
    except KeyError:
        fail("no user called (%s) on the system" % user_name)
    user_id = passdb_entry.pw_uid
    group_id = passdb_entry.pw_gid

    owner_uid = st.st_uid
    owner_gid = st.st_gid

    if owner_gid == group_id:
        if mode & stat.S_IWGRP:
            return

    if owner_uid == user_id:
        if mode & stat.S_IWUSR:
            return

    fail("user (%s) has no write access on file (%s)" % (user_name, filepath))

def get_ini_config_value(file_path, section, key): # returns a value for an attribute in a file "used with configuration files"
    parser = ConfigParser.ConfigParser()
    parser.read(file_path)
    value = parser.get(section, key)
    return value

def get_apache_config_value(file_path, section, module_name, key):
    with open(file_path) as file:
        try:
            text = file.read()
            section_start = "<"+section + " "+module_name + ">"
            section_end  = "</"+ section+">"
            all_sections = re.findall(r'\</?[^\>]+\>([\s\S]+)\<//?[^\>]+\>', text, re.MULTILINE)[0]
            print "all sections : ", all_sections

            split_start = all_sections.index(section_start)
            split_end = all_sections.index(section_end)
            wanted_section = all_sections[split_start +1: split_end]
            section_lines = wanted_section.split('\n')
            value = ""
            for line in section_lines:
                cleaned = line.strip()
                if cleaned.startswith(key):
                    value = cleaned[len(key)+1:]
                    break
            return value
        except:
            return ""

def get_file_configvalue(filepath,key, delimeter):
    #needs to be fixed or replaced
    line=os.popen("grep ^"+key+" "+filepath).readline()
    try:
        if line.startswith("#"):
            return None
        tuple_line=line.partition(delimeter)
        return tuple_line[2].strip()
    except:
        return None

def get_keys_value(filepath, key, delimeter):
    lines = open(filepath).readlines()
    for line in lines:
        line = line.strip()
        if line.startswith("#"):
            continue
        elif line.startswith(key):
            value = line.split(delimeter)[1]
            return value
    return ""

def check_filemode(filepath, mode):
    """Return True if 'file' matches ('permission') which should be entered in octal.
    """

    filemode = stat.S_IMODE(os.stat(filepath).st_mode)
    return (oct(filemode) == mode)

def get_modification_datetime(filepath):
      latest_modified = os.path.getmtime(filepath)
      return latest_modified

def is_recently_modified(filepath, threshold):
    mod_datetime = get_modification_datetime(filepath)
    delta = float(time.strftime("%s", time.localtime())) - mod_datetime
    if delta < threshold:
        return True
    return False

def check_exists(filepath):
    return os.path.exists(filepath)

def get_source_of_symbolic_link(symbolic):
    state = os.lstat(symbolic)
    if stat.S_ISLNK(state.st_mode):
        dest = os.popen("ls -l %s" % symbolic).readline().strip().split()[-1]
        return dest
    else:
        fail("File is not a symbolic link")

def get_permissions(filepath):
    permissions = oct(os.stat(filepath)[ST_MODE])[-3:]
    return permissions

def get_owner(filepath):
    uid = os.stat(filepath).st_uid
    owner = getpwuid(uid).pw_name
    return owner
def all_lines_commented(filepath):
    all_lines = [line.strip() for line in open(filepath).readlines()]
    for line in all_lines:
        if line:
            if not line.startswith("#"):
                return False
    return True

def check_hidden(file):
    return file.startswith('.') or file.endswith("~")

def list_dir(directorypath, show_hidden=False):
    hidden_cond = (lambda _: False) if show_hidden else check_hidden
    files_list = [cont for cont in os.listdir(directorypath) if os.path.isfile(os.path.join(directorypath, cont)) if not hidden_cond(cont)]
    directory_list = [cont for cont in os.listdir(directorypath) if os.path.isdir(os.path.join(directorypath, cont)) if not hidden_cond(cont)]
    return files_list, directory_list

def search_all_files(path, search):
    dirs_list = [os.path.join(dirname, subdirname) for dirname, dirnames, _ in os.walk(path) for subdirname in dirnames if subdirname == search]
    files_list = [os.path.join(dirname, filename) for dirname, _, filenames in os.walk(path) for filename in filenames if filename == search]
    return files_list, dirs_list

def search_by_extension(path, extension):
    return [os.path.join(dirname,filename) for dirname, _, filesnames in os.walk(path) for filename in filesnames if filename.endswith(extension)]

def is_empty(filepath):
    return os.stat(filepath)[6]==0

def user_access(path, username):
    return True if username == get_owner(path) else False

def extract_correct(extractpath, extractfiles = [], extractdirs = [] ):
    files, dirs = list_dir(extractpath)
    if extractdirs:
        for dir in dirs:
            if not (dir in extractdirs):
                return False
    if extractfiles:
        for file in files:
            if not (file in extractfiles):
                return False
    return True


#-----------------------------------------------------------------------------
# Copyright (c) 2010, IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

# Standard libary imports.
from base64 import decodestring
import os
import re

# System libary imports.
from IPython.external.qt import QtCore, QtGui

# Local imports
from IPython.utils.traitlets import Bool
from IPython.frontend.qt.svg import save_svg, svg_to_clipboard, svg_to_image
from ipython_widget import IPythonWidget

class RichIPythonWidget(IPythonWidget):
    """ An IPythonWidget that supports rich text, including lists, images, and
        tables. Note that raw performance will be reduced compared to the plain
        text version.
    """

    # RichIPythonWidget protected class variables.
    _payload_source_plot = 'IPython.zmq.pylab.backend_payload.add_plot_payload'
    _jpg_supported = Bool(False)

    # Used to determine whether a given html export attempt has already
    # displayed a warning about being unable to convert a png to svg.
    _svg_warning_displayed = False

    #---------------------------------------------------------------------------
    # 'object' interface
    #---------------------------------------------------------------------------

    def __init__(self, *args, **kw):
        """ Create a RichIPythonWidget.
        """
        kw['kind'] = 'rich'
        super(RichIPythonWidget, self).__init__(*args, **kw)

        # Configure the ConsoleWidget HTML exporter for our formats.
        self._html_exporter.image_tag = self._get_image_tag

        # Dictionary for resolving document resource names to SVG data.
        self._name_to_svg_map = {}

        # Do we support jpg ?
        # it seems that sometime jpg support is a plugin of QT, so try to assume
        # it is not always supported.
        _supported_format = map(str, QtGui.QImageReader.supportedImageFormats())
        self._jpg_supported = 'jpeg' in _supported_format

    #---------------------------------------------------------------------------
    # 'ConsoleWidget' public interface overides
    #---------------------------------------------------------------------------

    def export_html(self):
        """ Shows a dialog to export HTML/XML in various formats.

        Overridden in order to reset the _svg_warning_displayed flag prior
        to the export running.
        """
        self._svg_warning_displayed = False
        super(RichIPythonWidget, self).export_html()

    #---------------------------------------------------------------------------
    # 'ConsoleWidget' protected interface
    #---------------------------------------------------------------------------

    def _context_menu_make(self, pos):
        """ Reimplemented to return a custom context menu for images.
        """
        format = self._control.cursorForPosition(pos).charFormat()
        name = format.stringProperty(QtGui.QTextFormat.ImageName)
        if name:
            menu = QtGui.QMenu()

            menu.addAction('Copy Image', lambda: self._copy_image(name))
            menu.addAction('Save Image As...', lambda: self._save_image(name))
            menu.addSeparator()

            svg = self._name_to_svg_map.get(name, None)
            if svg is not None:
                menu.addSeparator()
                menu.addAction('Copy SVG', lambda: svg_to_clipboard(svg))
                menu.addAction('Save SVG As...',
                               lambda: save_svg(svg, self._control))
        else:
            menu = super(RichIPythonWidget, self)._context_menu_make(pos)
        return menu

    #---------------------------------------------------------------------------
    # 'BaseFrontendMixin' abstract interface
    #---------------------------------------------------------------------------
    def _pre_image_append(self, msg, prompt_number):
        """ Append the Out[] prompt  and make the output nicer

        Shared code for some the following if statement
        """
        self.log.debug("pyout: %s", msg.get('content', ''))
        self._append_plain_text(self.output_sep, True)
        self._append_html(self._make_out_prompt(prompt_number), True)
        self._append_plain_text('\n', True)

    def _handle_pyout(self, msg):
        """ Overridden to handle rich data types, like SVG.
        """
        if not self._hidden and self._is_from_this_session(msg):
            content = msg['content']
            prompt_number = content.get('execution_count', 0)
            data = content['data']
            if data.has_key('image/svg+xml'):
                self._pre_image_append(msg, prompt_number)
                self._append_svg(data['image/svg+xml'], True)
                self._append_html(self.output_sep2, True)
            elif data.has_key('image/png'):
                self._pre_image_append(msg, prompt_number)
                self._append_png(decodestring(data['image/png'].encode('ascii')), True)
                self._append_html(self.output_sep2, True)
            elif data.has_key('image/jpeg') and self._jpg_supported:
                self._pre_image_append(msg, prompt_number)
                self._append_jpg(decodestring(data['image/jpeg'].encode('ascii')), True)
                self._append_html(self.output_sep2, True)
            else:
                # Default back to the plain text representation.
                return super(RichIPythonWidget, self)._handle_pyout(msg)

    def _handle_display_data(self, msg):
        """ Overridden to handle rich data types, like SVG.
        """
        if not self._hidden and self._is_from_this_session(msg):
            source = msg['content']['source']
            data = msg['content']['data']
            metadata = msg['content']['metadata']
            # Try to use the svg or html representations.
            # FIXME: Is this the right ordering of things to try?
            if data.has_key('image/svg+xml'):
                self.log.debug("display: %s", msg.get('content', ''))
                svg = data['image/svg+xml']
                self._append_svg(svg, True)
            elif data.has_key('image/png'):
                self.log.debug("display: %s", msg.get('content', ''))
                # PNG data is base64 encoded as it passes over the network
                # in a JSON structure so we decode it.
                png = decodestring(data['image/png'].encode('ascii'))
                self._append_png(png, True)
            elif data.has_key('image/jpeg') and self._jpg_supported:
                self.log.debug("display: %s", msg.get('content', ''))
                jpg = decodestring(data['image/jpeg'].encode('ascii'))
                self._append_jpg(jpg, True)
            else:
                # Default back to the plain text representation.
                return super(RichIPythonWidget, self)._handle_display_data(msg)

    #---------------------------------------------------------------------------
    # 'RichIPythonWidget' protected interface
    #---------------------------------------------------------------------------

    def _append_jpg(self, jpg, before_prompt=False):
        """ Append raw JPG data to the widget."""
        self._append_custom(self._insert_jpg, jpg, before_prompt)

    def _append_png(self, png, before_prompt=False):
        """ Append raw PNG data to the widget.
        """
        self._append_custom(self._insert_png, png, before_prompt)

    def _append_svg(self, svg, before_prompt=False):
        """ Append raw SVG data to the widget.
        """
        self._append_custom(self._insert_svg, svg, before_prompt)

    def _add_image(self, image):
        """ Adds the specified QImage to the document and returns a
            QTextImageFormat that references it.
        """
        document = self._control.document()
        name = str(image.cacheKey())
        document.addResource(QtGui.QTextDocument.ImageResource,
                             QtCore.QUrl(name), image)
        format = QtGui.QTextImageFormat()
        format.setName(name)
        return format

    def _copy_image(self, name):
        """ Copies the ImageResource with 'name' to the clipboard.
        """
        image = self._get_image(name)
        QtGui.QApplication.clipboard().setImage(image)

    def _get_image(self, name):
        """ Returns the QImage stored as the ImageResource with 'name'.
        """
        document = self._control.document()
        image = document.resource(QtGui.QTextDocument.ImageResource,
                                  QtCore.QUrl(name))
        return image

    def _get_image_tag(self, match, path = None, format = "png"):
        """ Return (X)HTML mark-up for the image-tag given by match.

        Parameters
        ----------
        match : re.SRE_Match
            A match to an HTML image tag as exported by Qt, with
            match.group("Name") containing the matched image ID.

        path : string|None, optional [default None]
            If not None, specifies a path to which supporting files may be
            written (e.g., for linked images).  If None, all images are to be
            included inline.

        format : "png"|"svg"|"jpg", optional [default "png"]
            Format for returned or referenced images.
        """
        if format in ("png","jpg"):
            try:
                image = self._get_image(match.group("name"))
            except KeyError:
                return "Couldn't find image %s" % match.group("name")

            if path is not None:
                if not os.path.exists(path):
                    os.mkdir(path)
                relpath = os.path.basename(path)
                if image.save("%s/qt_img%s.%s" % (path, match.group("name"), format),
                              "PNG"):
                    return '' % (relpath,
                                                            match.group("name"),format)
                else:
                    return "Couldn't save image!"
            else:
                ba = QtCore.QByteArray()
                buffer_ = QtCore.QBuffer(ba)
                buffer_.open(QtCore.QIODevice.WriteOnly)
                image.save(buffer_, format.upper())
                buffer_.close()
                return '' % (
                    format,re.sub(r'(.{60})',r'\1\n',str(ba.toBase64())))

        elif format == "svg":
            try:
                svg = str(self._name_to_svg_map[match.group("name")])
            except KeyError:
                if not self._svg_warning_displayed:
                    QtGui.QMessageBox.warning(self, 'Error converting PNG to SVG.',
                        'Cannot convert a PNG to SVG.  To fix this, add this '
                        'to your ipython config:\n\n'
                        '\tc.InlineBackendConfig.figure_format = \'svg\'\n\n'
                        'And regenerate the figures.',
                                              QtGui.QMessageBox.Ok)
                    self._svg_warning_displayed = True
                return ("Cannot convert a PNG to SVG.  "
                        "To fix this, add this to your config: "
                        "c.InlineBackendConfig.figure_format = 'svg' "
                        "and regenerate the figures.")

            # Not currently checking path, because it's tricky to find a
            # cross-browser way to embed external SVG images (e.g., via
            # object or embed tags).

            # Chop stand-alone header from matplotlib SVG
            offset = svg.find("<svg")
            assert(offset > -1)

            return svg[offset:]

        else:
            return 'Unrecognized image format'

    def _insert_jpg(self, cursor, jpg):
        """ Insert raw PNG data into the widget."""
        self._insert_img(cursor, jpg, 'jpg')

    def _insert_png(self, cursor, png):
        """ Insert raw PNG data into the widget.
        """
        self._insert_img(cursor, png, 'png')

    def _insert_img(self, cursor, img, fmt):
        """ insert a raw image, jpg or png """
        try:
            image = QtGui.QImage()
            image.loadFromData(img, fmt.upper())
        except ValueError:
            self._insert_plain_text(cursor, 'Received invalid %s data.'%fmt)
        else:
            format = self._add_image(image)
            cursor.insertBlock()
            cursor.insertImage(format)
            cursor.insertBlock()

    def _insert_svg(self, cursor, svg):
        """ Insert raw SVG data into the widet.
        """
        try:
            image = svg_to_image(svg)
        except ValueError:
            self._insert_plain_text(cursor, 'Received invalid SVG data.')
        else:
            format = self._add_image(image)
            self._name_to_svg_map[format.name()] = svg
            cursor.insertBlock()
            cursor.insertImage(format)
            cursor.insertBlock()

    def _save_image(self, name, format='PNG'):
        """ Shows a save dialog for the ImageResource with 'name'.
        """
        dialog = QtGui.QFileDialog(self._control, 'Save Image')
        dialog.setAcceptMode(QtGui.QFileDialog.AcceptSave)
        dialog.setDefaultSuffix(format.lower())
        dialog.setNameFilter('%s file (*.%s)' % (format, format.lower()))
        if dialog.exec_():
            filename = dialog.selectedFiles()[0]
            image = self._get_image(name)
            image.save(filename, format)


"""A ZMQ-based subclass of InteractiveShell.

This code is meant to ease the refactoring of the base InteractiveShell into
something with a cleaner architecture for 2-process use, without actually
breaking InteractiveShell itself.  So we're doing something a bit ugly, where
we subclass and override what we want to fix.  Once this is working well, we
can go back to the base class and refactor the code for a cleaner inheritance
implementation that doesn't rely on so much monkeypatching.

But this lets us maintain a fully working IPython as we develop the new
machinery.  This should thus be thought of as scaffolding.
"""
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
from __future__ import print_function

# Stdlib
import os
import sys
import time

# System library imports
from zmq.eventloop import ioloop

# Our own
from IPython.core.interactiveshell import (
    InteractiveShell, InteractiveShellABC
)
from IPython.core import page
from IPython.core.autocall import ZMQExitAutocall
from IPython.core.displaypub import DisplayPublisher
from IPython.core.magics import MacroToEdit, CodeMagics
from IPython.core.magic import magics_class, line_magic, Magics
from IPython.core.payloadpage import install_payload_page
from IPython.lib.kernel import (
    get_connection_file, get_connection_info, connect_qtconsole
)
from IPython.testing.skipdoctest import skip_doctest
from IPython.utils import io
from IPython.utils.jsonutil import json_clean, encode_images
from IPython.utils.process import arg_split
from IPython.utils import py3compat
from IPython.utils.traitlets import Instance, Type, Dict, CBool, CBytes
from IPython.utils.warn import warn, error
from IPython.zmq.displayhook import ZMQShellDisplayHook
from IPython.zmq.session import extract_header
from session import Session

#-----------------------------------------------------------------------------
# Functions and classes
#-----------------------------------------------------------------------------

class ZMQDisplayPublisher(DisplayPublisher):
    """A display publisher that publishes data using a ZeroMQ PUB socket."""

    session = Instance(Session)
    pub_socket = Instance('zmq.Socket')
    parent_header = Dict({})
    topic = CBytes(b'displaypub')

    def set_parent(self, parent):
        """Set the parent for outbound messages."""
        self.parent_header = extract_header(parent)

    def _flush_streams(self):
        """flush IO Streams prior to display"""
        sys.stdout.flush()
        sys.stderr.flush()

    def publish(self, source, data, metadata=None):
        self._flush_streams()
        if metadata is None:
            metadata = {}
        self._validate_data(source, data, metadata)
        content = {}
        content['source'] = source
        content['data'] = encode_images(data)
        content['metadata'] = metadata
        self.session.send(
            self.pub_socket, u'display_data', json_clean(content),
            parent=self.parent_header, ident=self.topic,
        )

    def clear_output(self, stdout=True, stderr=True, other=True):
        content = dict(stdout=stdout, stderr=stderr, other=other)

        if stdout:
            print('\r', file=sys.stdout, end='')
        if stderr:
            print('\r', file=sys.stderr, end='')

        self._flush_streams()

        self.session.send(
            self.pub_socket, u'clear_output', content,
            parent=self.parent_header, ident=self.topic,
        )

@magics_class
class KernelMagics(Magics):
    #------------------------------------------------------------------------
    # Magic overrides
    #------------------------------------------------------------------------
    # Once the base class stops inheriting from magic, this code needs to be
    # moved into a separate machinery as well.  For now, at least isolate here
    # the magics which this class needs to implement differently from the base
    # class, or that are unique to it.

    @line_magic
    def doctest_mode(self, parameter_s=''):
        """Toggle doctest mode on and off.

        This mode is intended to make IPython behave as much as possible like a
        plain Python shell, from the perspective of how its prompts, exceptions
        and output look.  This makes it easy to copy and paste parts of a
        session into doctests.  It does so by:

        - Changing the prompts to the classic ``>>>`` ones.
        - Changing the exception reporting mode to 'Plain'.
        - Disabling pretty-printing of output.

        Note that IPython also supports the pasting of code snippets that have
        leading '>>>' and '...' prompts in them.  This means that you can paste
        doctests from files or docstrings (even if they have leading
        whitespace), and the code will execute correctly.  You can then use
        '%history -t' to see the translated history; this will give you the
        input after removal of all the leading prompts and whitespace, which
        can be pasted back into an editor.

        With these features, you can switch into this mode easily whenever you
        need to do testing and changes to doctests, without having to leave
        your existing IPython session.
        """

        from IPython.utils.ipstruct import Struct

        # Shorthands
        shell = self.shell
        disp_formatter = self.shell.display_formatter
        ptformatter = disp_formatter.formatters['text/plain']
        # dstore is a data store kept in the instance metadata bag to track any
        # changes we make, so we can undo them later.
        dstore = shell.meta.setdefault('doctest_mode', Struct())
        save_dstore = dstore.setdefault

        # save a few values we'll need to recover later
        mode = save_dstore('mode', False)
        save_dstore('rc_pprint', ptformatter.pprint)
        save_dstore('rc_plain_text_only',disp_formatter.plain_text_only)
        save_dstore('xmode', shell.InteractiveTB.mode)

        if mode == False:
            # turn on
            ptformatter.pprint = False
            disp_formatter.plain_text_only = True
            shell.magic('xmode Plain')
        else:
            # turn off
            ptformatter.pprint = dstore.rc_pprint
            disp_formatter.plain_text_only = dstore.rc_plain_text_only
            shell.magic("xmode " + dstore.xmode)

        # Store new mode and inform on console
        dstore.mode = bool(1-int(mode))
        mode_label = ['OFF','ON'][dstore.mode]
        print('Doctest mode is:', mode_label)

        # Send the payload back so that clients can modify their prompt display
        payload = dict(
            source='IPython.zmq.zmqshell.ZMQInteractiveShell.doctest_mode',
            mode=dstore.mode)
        shell.payload_manager.write_payload(payload)

    _find_edit_target = CodeMagics._find_edit_target

    @skip_doctest
    @line_magic
    def edit(self, parameter_s='', last_call=['','']):
        """Bring up an editor and execute the resulting code.

        Usage:
          %edit [options] [args]

        %edit runs an external text editor. You will need to set the command for
        this editor via the ``TerminalInteractiveShell.editor`` option in your
        configuration file before it will work.

        This command allows you to conveniently edit multi-line code right in
        your IPython session.

        If called without arguments, %edit opens up an empty editor with a
        temporary file and will execute the contents of this file when you
        close it (don't forget to save it!).

        Options:

        -n <number>: open the editor at a specified line number.  By default,
        the IPython editor hook uses the unix syntax 'editor +N filename', but
        you can configure this by providing your own modified hook if your
        favorite editor supports line-number specifications with a different
        syntax.

        -p: this will call the editor with the same data as the previous time
        it was used, regardless of how long ago (in your current session) it
        was.

        -r: use 'raw' input.  This option only applies to input taken from the
        user's history.  By default, the 'processed' history is used, so that
        magics are loaded in their transformed version to valid Python.  If
        this option is given, the raw input as typed as the command line is
        used instead.  When you exit the editor, it will be executed by
        IPython's own processor.

        -x: do not execute the edited code immediately upon exit. This is
        mainly useful if you are editing programs which need to be called with
        command line arguments, which you can then do using %run.

        Arguments:

        If arguments are given, the following possibilites exist:

        - The arguments are numbers or pairs of colon-separated numbers (like
        1 4:8 9). These are interpreted as lines of previous input to be
        loaded into the editor. The syntax is the same of the %macro command.

        - If the argument doesn't start with a number, it is evaluated as a
        variable and its contents loaded into the editor. You can thus edit
        any string which contains python code (including the result of
        previous edits).

        - If the argument is the name of an object (other than a string),
        IPython will try to locate the file where it was defined and open the
        editor at the point where it is defined. You can use `%edit function`
        to load an editor exactly at the point where 'function' is defined,
        edit it and have the file be executed automatically.

        If the object is a macro (see %macro for details), this opens up your
        specified editor with a temporary file containing the macro's data.
        Upon exit, the macro is reloaded with the contents of the file.

        Note: opening at an exact line is only supported under Unix, and some
        editors (like kedit and gedit up to Gnome 2.8) do not understand the
        '+NUMBER' parameter necessary for this feature. Good editors like
        (X)Emacs, vi, jed, pico and joe all do.

        - If the argument is not found as a variable, IPython will look for a
        file with that name (adding .py if necessary) and load it into the
        editor. It will execute its contents with execfile() when you exit,
        loading any code in the file into your interactive namespace.

        After executing your code, %edit will return as output the code you
        typed in the editor (except when it was an existing file). This way
        you can reload the code in further invocations of %edit as a variable,
        via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
        the output.

        Note that %edit is also available through the alias %ed.

        This is an example of creating a simple function inside the editor and
        then modifying it. First, start up the editor:

        In [1]: ed
        Editing... done. Executing edited code...
        Out[1]: 'def foo():n    print "foo() was defined in an editing session"n'

        We can then call the function foo():

        In [2]: foo()
        foo() was defined in an editing session

        Now we edit foo.  IPython automatically loads the editor with the
        (temporary) file where foo() was previously defined:

        In [3]: ed foo
        Editing... done. Executing edited code...

        And if we call foo() again we get the modified version:

        In [4]: foo()
        foo() has now been changed!

        Here is an example of how to edit a code snippet successive
        times. First we call the editor:

        In [5]: ed
        Editing... done. Executing edited code...
        hello
        Out[5]: "print 'hello'n"

        Now we call it again with the previous output (stored in _):

        In [6]: ed _
        Editing... done. Executing edited code...
        hello world
        Out[6]: "print 'hello world'n"

        Now we call it with the output #8 (stored in _8, also as Out[8]):

        In [7]: ed _8
        Editing... done. Executing edited code...
        hello again
        Out[7]: "print 'hello again'n"
        """

        opts,args = self.parse_options(parameter_s,'prn:')

        try:
            filename, lineno, _ = CodeMagics._find_edit_target(self.shell, args, opts, last_call)
        except MacroToEdit as e:
            # TODO: Implement macro editing over 2 processes.
            print("Macro editing not yet implemented in 2-process model.")
            return

        # Make sure we send to the client an absolute path, in case the working
        # directory of client and kernel don't match
        filename = os.path.abspath(filename)

        payload = {
            'source' : 'IPython.zmq.zmqshell.ZMQInteractiveShell.edit_magic',
            'filename' : filename,
            'line_number' : lineno
        }
        self.shell.payload_manager.write_payload(payload)

    # A few magics that are adapted to the specifics of using pexpect and a
    # remote terminal

    @line_magic
    def clear(self, arg_s):
        """Clear the terminal."""
        if os.name == 'posix':
            self.shell.system("clear")
        else:
            self.shell.system("cls")

    if os.name == 'nt':
        # This is the usual name in windows
        cls = line_magic('cls')(clear)

    # Terminal pagers won't work over pexpect, but we do have our own pager

    @line_magic
    def less(self, arg_s):
        """Show a file through the pager.

        Files ending in .py are syntax-highlighted."""
        cont = open(arg_s).read()
        if arg_s.endswith('.py'):
            cont = self.shell.pycolorize(cont)
        page.page(cont)

    more = line_magic('more')(less)

    # Man calls a pager, so we also need to redefine it
    if os.name == 'posix':
        @line_magic
        def man(self, arg_s):
            """Find the man page for the given command and display in pager."""
            page.page(self.shell.getoutput('man %s | col -b' % arg_s,
                                           split=False))

    @line_magic
    def connect_info(self, arg_s):
        """Print information for connecting other clients to this kernel

        It will print the contents of this session's connection file, as well as
        shortcuts for local clients.

        In the simplest case, when called from the most recently launched kernel,
        secondary clients can be connected, simply with:

        $> ipython <app> --existing

        """

        from IPython.core.application import BaseIPythonApplication as BaseIPApp

        if BaseIPApp.initialized():
            app = BaseIPApp.instance()
            security_dir = app.profile_dir.security_dir
            profile = app.profile
        else:
            profile = 'default'
            security_dir = ''

        try:
            connection_file = get_connection_file()
            info = get_connection_info(unpack=False)
        except Exception as e:
            error("Could not get connection info: %r" % e)
            return

        # add profile flag for non-default profile
        profile_flag = "--profile %s" % profile if profile != 'default' else ""

        # if it's in the security dir, truncate to basename
        if security_dir == os.path.dirname(connection_file):
            connection_file = os.path.basename(connection_file)

        print (info + '\n')
        print ("Paste the above JSON into a file, and connect with:\n"
            "    $> ipython <app> --existing <file>\n"
            "or, if you are local, you can connect with just:\n"
            "    $> ipython <app> --existing {0} {1}\n"
            "or even just:\n"
            "    $> ipython <app> --existing {1}\n"
            "if this is the most recent IPython session you have started.".format(
            connection_file, profile_flag
            )
        )

    @line_magic
    def qtconsole(self, arg_s):
        """Open a qtconsole connected to this kernel.

        Useful for connecting a qtconsole to running notebooks, for better
        debugging.
        """

        # %qtconsole should imply bind_kernel for engines:
        try:
            from IPython.parallel import bind_kernel
        except ImportError:
            # technically possible, because parallel has higher pyzmq min-version
            pass
        else:
            bind_kernel()

        try:
            p = connect_qtconsole(argv=arg_split(arg_s, os.name=='posix'))
        except Exception as e:
            error("Could not start qtconsole: %r" % e)
            return

def safe_unicode(e):
    """unicode(e) with various fallbacks. Used for exceptions, which may not be
    safe to call unicode() on.
    """
    try:
        return unicode(e)
    except UnicodeError:
        pass

    try:
        return py3compat.str_to_unicode(str(e))
    except UnicodeError:
        pass

    try:
        return py3compat.str_to_unicode(repr(e))
    except UnicodeError:
        pass

    return u'Unrecoverably corrupt evalue'

class ZMQInteractiveShell(InteractiveShell):
    """A subclass of InteractiveShell for ZMQ."""

    displayhook_class = Type(ZMQShellDisplayHook)
    display_pub_class = Type(ZMQDisplayPublisher)

    # Override the traitlet in the parent class, because there's no point using
    # readline for the kernel. Can be removed when the readline code is moved
    # to the terminal frontend.
    colors_force = CBool(True)
    readline_use = CBool(False)
    # autoindent has no meaning in a zmqshell, and attempting to enable it
    # will print a warning in the absence of readline.
    autoindent = CBool(False)

    exiter = Instance(ZMQExitAutocall)
    def _exiter_default(self):
        return ZMQExitAutocall(self)

    def _exit_now_changed(self, name, old, new):
        """stop eventloop when exit_now fires"""
        if new:
            loop = ioloop.IOLoop.instance()
            loop.add_timeout(time.time()+0.1, loop.stop)

    keepkernel_on_exit = None

    # Over ZeroMQ, GUI control isn't done with PyOS_InputHook as there is no
    # interactive input being read; we provide event loop support in ipkernel
    from .eventloops import enable_gui
    enable_gui = staticmethod(enable_gui)

    def init_environment(self):
        """Configure the user's environment.

        """
        env = os.environ
        # These two ensure 'ls' produces nice coloring on BSD-derived systems
        env['TERM'] = 'xterm-color'
        env['CLICOLOR'] = '1'
        # Since normal pagers don't work at all (over pexpect we don't have
        # single-key control of the subprocess), try to disable paging in
        # subprocesses as much as possible.
        env['PAGER'] = 'cat'
        env['GIT_PAGER'] = 'cat'

        # And install the payload version of page.
        install_payload_page()

    def auto_rewrite_input(self, cmd):
        """Called to show the auto-rewritten input for autocall and friends.

        FIXME: this payload is currently not correctly processed by the
        frontend.
        """
        new = self.prompt_manager.render('rewrite') + cmd
        payload = dict(
            source='IPython.zmq.zmqshell.ZMQInteractiveShell.auto_rewrite_input',
            transformed_input=new,
            )
        self.payload_manager.write_payload(payload)

    def ask_exit(self):
        """Engage the exit actions."""
        self.exit_now = True
        payload = dict(
            source='IPython.zmq.zmqshell.ZMQInteractiveShell.ask_exit',
            exit=True,
            keepkernel=self.keepkernel_on_exit,
            )
        self.payload_manager.write_payload(payload)

    def _showtraceback(self, etype, evalue, stb):

        exc_content = {
            u'traceback' : stb,
            u'ename' : unicode(etype.__name__),
            u'evalue' : safe_unicode(evalue)
        }

        dh = self.displayhook
        # Send exception info over pub socket for other clients than the caller
        # to pick up
        topic = None
        if dh.topic:
            topic = dh.topic.replace(b'pyout', b'pyerr')

        exc_msg = dh.session.send(dh.pub_socket, u'pyerr', json_clean(exc_content), dh.parent_header, ident=topic)

        # FIXME - Hack: store exception info in shell object.  Right now, the
        # caller is reading this info after the fact, we need to fix this logic
        # to remove this hack.  Even uglier, we need to store the error status
        # here, because in the main loop, the logic that sets it is being
        # skipped because runlines swallows the exceptions.
        exc_content[u'status'] = u'error'
        self._reply_content = exc_content
        # /FIXME

        return exc_content

    def set_next_input(self, text):
        """Send the specified text to the frontend to be presented at the next
        input cell."""
        payload = dict(
            source='IPython.zmq.zmqshell.ZMQInteractiveShell.set_next_input',
            text=text
        )
        self.payload_manager.write_payload(payload)

    #-------------------------------------------------------------------------
    # Things related to magics
    #-------------------------------------------------------------------------

    def init_magics(self):
        super(ZMQInteractiveShell, self).init_magics()
        self.register_magics(KernelMagics)
        self.run_line_magic('alias_magic', 'ed edit')

InteractiveShellABC.register(ZMQInteractiveShell)


'''
    Apache Specific Utilities

'''
import gurumate.base
from gurumate.base import fail

def check_running(process_name="apache2"):
    '''
        CHECK if process (default=apache2) is not running
    '''
    if not gurumate.base.get_pid_list(process_name):
        fail("Apache process '%s' doesn't seem to be working" % process_name)
        return False #unreachable
    return True

def check_listening_port(port=80, process_name="apache2"):
    '''
        CHECK if process (default=apache2) is running and listening on port 80
    '''
    if port not in get_listening_ports(process_name):
        fail("Apache is not listening on port %s, \
        maybe you forgot to configure the port? \
        or did you forget to start the service?" % port)

def get_listening_ports(process_name="apache2"):
    '''
        returns a list of listening ports for running process (default=apache2)
    '''
    ports = set()
    for _, address_info in gurumate.base.get_listening_ports(process_name):
        ports.add(address_info[1])
    return list(ports)


"""Config file for coverage.py"""

import os, re, sys
from coverage.backward import string_class, iitems

# In py3, # ConfigParser was renamed to the more-standard configparser
try:
    import configparser                             # pylint: disable=F0401
except ImportError:
    import ConfigParser as configparser

class HandyConfigParser(configparser.RawConfigParser):
    """Our specialization of ConfigParser."""

    def read(self, filename):
        """Read a filename as UTF-8 configuration data."""
        kwargs = {}
        if sys.version_info >= (3, 2):
            kwargs['encoding'] = "utf-8"
        return configparser.RawConfigParser.read(self, filename, **kwargs)

    def get(self, *args, **kwargs):
        v = configparser.RawConfigParser.get(self, *args, **kwargs)
        def dollar_replace(m):
            """Called for each $replacement."""
            # Only one of the groups will have matched, just get its text.
            word = [w for w in m.groups() if w is not None][0]
            if word == "$":
                return "$"
            else:
                return os.environ.get(word, '')

        dollar_pattern = r"""(?x)   # Use extended regex syntax
            \$(?:                   # A dollar sign, then
            (?P<v1>\w+) |           #   a plain word,
            {(?P<v2>\w+)} |         #   or a {-wrapped word,
            (?P<char>[$])           #   or a dollar sign.
            )
            """
        v = re.sub(dollar_pattern, dollar_replace, v)
        return v

    def getlist(self, section, option):
        """Read a list of strings.

        The value of `section` and `option` is treated as a comma- and newline-
        separated list of strings.  Each value is stripped of whitespace.

        Returns the list of strings.

        """
        value_list = self.get(section, option)
        values = []
        for value_line in value_list.split('\n'):
            for value in value_line.split(','):
                value = value.strip()
                if value:
                    values.append(value)
        return values

    def getlinelist(self, section, option):
        """Read a list of full-line strings.

        The value of `section` and `option` is treated as a newline-separated
        list of strings.  Each value is stripped of whitespace.

        Returns the list of strings.

        """
        value_list = self.get(section, option)
        return list(filter(None, value_list.split('\n')))

# The default line exclusion regexes
DEFAULT_EXCLUDE = [
    '(?i)# *pragma[: ]*no *cover',
    ]

# The default partial branch regexes, to be modified by the user.
DEFAULT_PARTIAL = [
    '(?i)# *pragma[: ]*no *branch',
    ]

# The default partial branch regexes, based on Python semantics.
# These are any Python branching constructs that can't actually execute all
# their branches.
DEFAULT_PARTIAL_ALWAYS = [
    'while (True|1|False|0):',
    'if (True|1|False|0):',
    ]

class CoverageConfig(object):
    """Coverage.py configuration.

    The attributes of this class are the various settings that control the
    operation of coverage.py.

    """
    def __init__(self):
        """Initialize the configuration attributes to their defaults."""
        # Metadata about the config.
        self.attempted_config_files = []
        self.config_files = []

        # Defaults for [run]
        self.branch = False
        self.cover_pylib = False
        self.data_file = ".coverage"
        self.parallel = False
        self.timid = False
        self.source = None
        self.debug = []

        # Defaults for [report]
        self.exclude_list = DEFAULT_EXCLUDE[:]
        self.ignore_errors = False
        self.include = None
        self.omit = None
        self.partial_list = DEFAULT_PARTIAL[:]
        self.partial_always_list = DEFAULT_PARTIAL_ALWAYS[:]
        self.precision = 0
        self.show_missing = False

        # Defaults for [html]
        self.html_dir = "htmlcov"
        self.extra_css = None
        self.html_title = "Coverage report"

        # Defaults for [xml]
        self.xml_output = "coverage.xml"

        # Defaults for [paths]
        self.paths = {}

    def from_environment(self, env_var):
        """Read configuration from the `env_var` environment variable."""
        # Timidity: for nose users, read an environment variable.  This is a
        # cheap hack, since the rest of the command line arguments aren't
        # recognized, but it solves some users' problems.
        env = os.environ.get(env_var, '')
        if env:
            self.timid = ('--timid' in env)

    MUST_BE_LIST = ["omit", "include", "debug"]

    def from_args(self, **kwargs):
        """Read config values from `kwargs`."""
        for k, v in iitems(kwargs):
            if v is not None:
                if k in self.MUST_BE_LIST and isinstance(v, string_class):
                    v = [v]
                setattr(self, k, v)

    def from_file(self, filename):
        """Read configuration from a .rc file.

        `filename` is a file name to read.

        """
        self.attempted_config_files.append(filename)

        cp = HandyConfigParser()
        files_read = cp.read(filename)
        if files_read is not None:  # return value changed in 2.4
            self.config_files.extend(files_read)

        for option_spec in self.CONFIG_FILE_OPTIONS:
            self.set_attr_from_config_option(cp, *option_spec)

        # [paths] is special
        if cp.has_section('paths'):
            for option in cp.options('paths'):
                self.paths[option] = cp.getlist('paths', option)

    CONFIG_FILE_OPTIONS = [
        # [run]
        ('branch', 'run:branch', 'boolean'),
        ('cover_pylib', 'run:cover_pylib', 'boolean'),
        ('data_file', 'run:data_file'),
        ('debug', 'run:debug', 'list'),
        ('include', 'run:include', 'list'),
        ('omit', 'run:omit', 'list'),
        ('parallel', 'run:parallel', 'boolean'),
        ('source', 'run:source', 'list'),
        ('timid', 'run:timid', 'boolean'),

        # [report]
        ('exclude_list', 'report:exclude_lines', 'linelist'),
        ('ignore_errors', 'report:ignore_errors', 'boolean'),
        ('include', 'report:include', 'list'),
        ('omit', 'report:omit', 'list'),
        ('partial_list', 'report:partial_branches', 'linelist'),
        ('partial_always_list', 'report:partial_branches_always', 'linelist'),
        ('precision', 'report:precision', 'int'),
        ('show_missing', 'report:show_missing', 'boolean'),

        # [html]
        ('html_dir', 'html:directory'),
        ('extra_css', 'html:extra_css'),
        ('html_title', 'html:title'),

        # [xml]
        ('xml_output', 'xml:output'),
        ]

    def set_attr_from_config_option(self, cp, attr, where, type_=''):
        """Set an attribute on self if it exists in the ConfigParser."""
        section, option = where.split(":")
        if cp.has_option(section, option):
            method = getattr(cp, 'get'+type_)
            setattr(self, attr, method(section, option))