#!/usr/bin/env python
# encoding: utf-8
"""
The ipcluster application.

Authors:

* Brian Granger
* MinRK

"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import errno
import logging
import os
import re
import signal

from subprocess import check_call, CalledProcessError, PIPE
import zmq
from zmq.eventloop import ioloop

from IPython.config.application import Application, boolean_flag, catch_config_error
from IPython.config.loader import Config
from IPython.core.application import BaseIPythonApplication
from IPython.core.profiledir import ProfileDir
from IPython.utils.daemonize import daemonize
from IPython.utils.importstring import import_item
from IPython.utils.sysinfo import num_cpus
from IPython.utils.traitlets import (Integer, Unicode, Bool, CFloat, Dict, List, Any,
                                        DottedObjectName)

from IPython.parallel.apps.baseapp import (
    BaseParallelApplication,
    PIDFileError,
    base_flags, base_aliases
)

#-----------------------------------------------------------------------------
# Module level variables
#-----------------------------------------------------------------------------

default_config_file_name = u'ipcluster_config.py'

_description = """Start an IPython cluster for parallel computing.

An IPython cluster consists of 1 controller and 1 or more engines.
This command automates the startup of these processes using a wide
range of startup methods (SSH, local processes, PBS, mpiexec,
Windows HPC Server 2008). To start a cluster with 4 engines on your
local host simply do 'ipcluster start --n=4'. For more complex usage
you will typically do 'ipython profile create mycluster --parallel', then edit
configuration files, followed by 'ipcluster start --profile=mycluster --n=4'.
"""

_main_examples = """
ipcluster start --n=4 # start a 4 node cluster on localhost
ipcluster start -h    # show the help string for the start subcmd

ipcluster stop -h     # show the help string for the stop subcmd
ipcluster engines -h  # show the help string for the engines subcmd
"""

_start_examples = """
ipython profile create mycluster --parallel # create mycluster profile
ipcluster start --profile=mycluster --n=4   # start mycluster with 4 nodes
"""

_stop_examples = """
ipcluster stop --profile=mycluster  # stop a running cluster by profile name
"""

_engines_examples = """
ipcluster engines --profile=mycluster --n=4  # start 4 engines only
"""

# Exit codes for ipcluster

# This will be the exit code if the ipcluster appears to be running because
# a .pid file exists
ALREADY_STARTED = 10

# This will be the exit code if ipcluster stop is run, but there is not .pid
# file to be found.
ALREADY_STOPPED = 11

# This will be the exit code if ipcluster engines is run, but there is not .pid
# file to be found.
NO_CLUSTER = 12

#-----------------------------------------------------------------------------
# Utilities
#-----------------------------------------------------------------------------

def find_launcher_class(clsname, kind):
    """Return a launcher for a given clsname and kind.

    Parameters
    ==========
    clsname : str
        The full name of the launcher class, either with or without the
        module path, or an abbreviation (MPI, SSH, SGE, PBS, LSF,
        WindowsHPC).
    kind : str
        Either 'EngineSet' or 'Controller'.
    """
    if '.' not in clsname:
        # not a module, presume it's the raw name in apps.launcher
        if kind and kind not in clsname:
            # doesn't match necessary full class name, assume it's
            # just 'PBS' or 'MPI' prefix:
            clsname = clsname + kind + 'Launcher'
        clsname = 'IPython.parallel.apps.launcher.'+clsname
    klass = import_item(clsname)
    return klass

#-----------------------------------------------------------------------------
# Main application
#-----------------------------------------------------------------------------

start_help = """Start an IPython cluster for parallel computing

Start an ipython cluster by its profile name or cluster
directory. Cluster directories contain configuration, log and
security related files and are named using the convention
'profile_<name>' and should be creating using the 'start'
subcommand of 'ipcluster'. If your cluster directory is in
the cwd or the ipython directory, you can simply refer to it
using its profile name, 'ipcluster start --n=4 --profile=<profile>`,
otherwise use the 'profile-dir' option.
"""
stop_help = """Stop a running IPython cluster

Stop a running ipython cluster by its profile name or cluster
directory. Cluster directories are named using the convention
'profile_<name>'. If your cluster directory is in
the cwd or the ipython directory, you can simply refer to it
using its profile name, 'ipcluster stop --profile=<profile>`, otherwise
use the '--profile-dir' option.
"""
engines_help = """Start engines connected to an existing IPython cluster

Start one or more engines to connect to an existing Cluster
by profile name or cluster directory.
Cluster directories contain configuration, log and
security related files and are named using the convention
'profile_<name>' and should be creating using the 'start'
subcommand of 'ipcluster'. If your cluster directory is in
the cwd or the ipython directory, you can simply refer to it
using its profile name, 'ipcluster engines --n=4 --profile=<profile>`,
otherwise use the 'profile-dir' option.
"""
stop_aliases = dict(
    signal='IPClusterStop.signal',
)
stop_aliases.update(base_aliases)

class IPClusterStop(BaseParallelApplication):
    name = u'ipcluster'
    description = stop_help
    examples = _stop_examples
    config_file_name = Unicode(default_config_file_name)

    signal = Integer(signal.SIGINT, config=True,
        help="signal to use for stopping processes.")

    aliases = Dict(stop_aliases)

    def start(self):
        """Start the app for the stop subcommand."""
        try:
            pid = self.get_pid_from_file()
        except PIDFileError:
            self.log.critical(
                'Could not read pid file, cluster is probably not running.'
            )
            # Here I exit with a unusual exit status that other processes
            # can watch for to learn how I existed.
            self.remove_pid_file()
            self.exit(ALREADY_STOPPED)

        if not self.check_pid(pid):
            self.log.critical(
                'Cluster [pid=%r] is not running.' % pid
            )
            self.remove_pid_file()
            # Here I exit with a unusual exit status that other processes
            # can watch for to learn how I existed.
            self.exit(ALREADY_STOPPED)

        elif os.name=='posix':
            sig = self.signal
            self.log.info(
                "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
            )
            try:
                os.kill(pid, sig)
            except OSError:
                self.log.error("Stopping cluster failed, assuming already dead.",
                    exc_info=True)
                self.remove_pid_file()
        elif os.name=='nt':
            try:
                # kill the whole tree
                p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE)
            except (CalledProcessError, OSError):
                self.log.error("Stopping cluster failed, assuming already dead.",
                    exc_info=True)
            self.remove_pid_file()

engine_aliases = {}
engine_aliases.update(base_aliases)
engine_aliases.update(dict(
    n='IPClusterEngines.n',
    engines = 'IPClusterEngines.engine_launcher_class',
    daemonize = 'IPClusterEngines.daemonize',
))
engine_flags = {}
engine_flags.update(base_flags)

engine_flags.update(dict(
    daemonize=(
        {'IPClusterEngines' : {'daemonize' : True}},
        """run the cluster into the background (not available on Windows)""",
    )
))
class IPClusterEngines(BaseParallelApplication):

    name = u'ipcluster'
    description = engines_help
    examples = _engines_examples
    usage = None
    config_file_name = Unicode(default_config_file_name)
    default_log_level = logging.INFO
    classes = List()
    def _classes_default(self):
        from IPython.parallel.apps import launcher
        launchers = launcher.all_launchers
        eslaunchers = [ l for l in launchers if 'EngineSet' in l.__name__]
        return [ProfileDir]+eslaunchers

    n = Integer(num_cpus(), config=True,
        help="""The number of engines to start. The default is to use one for each
        CPU on your machine""")

    engine_launcher = Any(config=True, help="Deprecated, use engine_launcher_class")
    def _engine_launcher_changed(self, name, old, new):
        if isinstance(new, basestring):
            self.log.warn("WARNING: %s.engine_launcher is deprecated as of 0.12,"
                    " use engine_launcher_class" % self.__class__.__name__)
            self.engine_launcher_class = new
    engine_launcher_class = DottedObjectName('LocalEngineSetLauncher',
        config=True,
        help="""The class for launching a set of Engines. Change this value
        to use various batch systems to launch your engines, such as PBS,SGE,MPI,etc.
        Each launcher class has its own set of configuration options, for making sure
        it will work in your environment.

        You can also write your own launcher, and specify it's absolute import path,
        as in 'mymodule.launcher.FTLEnginesLauncher`.

        IPython's bundled examples include:

            Local : start engines locally as subprocesses [default]
            MPI : use mpiexec to launch engines in an MPI environment
            PBS : use PBS (qsub) to submit engines to a batch queue
            SGE : use SGE (qsub) to submit engines to a batch queue
            LSF : use LSF (bsub) to submit engines to a batch queue
            SSH : use SSH to start the controller
                        Note that SSH does *not* move the connection files
                        around, so you will likely have to do this manually
                        unless the machines are on a shared file system.
            WindowsHPC : use Windows HPC

        If you are using one of IPython's builtin launchers, you can specify just the
        prefix, e.g:

            c.IPClusterEngines.engine_launcher_class = 'SSH'

        or:

            ipcluster start --engines=MPI

        """
        )
    daemonize = Bool(False, config=True,
        help="""Daemonize the ipcluster program. This implies --log-to-file.
        Not available on Windows.
        """)

    def _daemonize_changed(self, name, old, new):
        if new:
            self.log_to_file = True

    early_shutdown = Integer(30, config=True, help="The timeout (in seconds)")
    _stopping = False

    aliases = Dict(engine_aliases)
    flags = Dict(engine_flags)

    @catch_config_error
    def initialize(self, argv=None):
        super(IPClusterEngines, self).initialize(argv)
        self.init_signal()
        self.init_launchers()

    def init_launchers(self):
        self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet')

    def init_signal(self):
        # Setup signals
        signal.signal(signal.SIGINT, self.sigint_handler)

    def build_launcher(self, clsname, kind=None):
        """import and instantiate a Launcher based on importstring"""
        try:
            klass = find_launcher_class(clsname, kind)
        except (ImportError, KeyError):
            self.log.fatal("Could not import launcher class: %r"%clsname)
            self.exit(1)

        launcher = klass(
            work_dir=u'.', config=self.config, log=self.log,
            profile_dir=self.profile_dir.location, cluster_id=self.cluster_id,
        )
        return launcher

    def engines_started_ok(self):
        self.log.info("Engines appear to have started successfully")
        self.early_shutdown = 0

    def start_engines(self):
        # Some EngineSetLaunchers ignore `n` and use their own engine count, such as SSH:
        n = getattr(self.engine_launcher, 'engine_count', self.n)
        self.log.info("Starting %s Engines with %s", n, self.engine_launcher_class)
        self.engine_launcher.start(self.n)
        self.engine_launcher.on_stop(self.engines_stopped_early)
        if self.early_shutdown:
            ioloop.DelayedCallback(self.engines_started_ok, self.early_shutdown*1000, self.loop).start()

    def engines_stopped_early(self, r):
        if self.early_shutdown and not self._stopping:
            self.log.error("""
            Engines shutdown early, they probably failed to connect.

            Check the engine log files for output.

            If your controller and engines are not on the same machine, you probably
            have to instruct the controller to listen on an interface other than localhost.

            You can set this by adding "--ip='*'" to your ControllerLauncher.controller_args.

            Be sure to read our security docs before instructing your controller to listen on
            a public interface.
            """)
            self.stop_launchers()

        return self.engines_stopped(r)

    def engines_stopped(self, r):
        return self.loop.stop()

    def stop_engines(self):
        if self.engine_launcher.running:
            self.log.info("Stopping Engines...")
            d = self.engine_launcher.stop()
            return d
        else:
            return None

    def stop_launchers(self, r=None):
        if not self._stopping:
            self._stopping = True
            self.log.error("IPython cluster: stopping")
            self.stop_engines()
            # Wait a few seconds to let things shut down.
            dc = ioloop.DelayedCallback(self.loop.stop, 3000, self.loop)
            dc.start()

    def sigint_handler(self, signum, frame):
        self.log.debug("SIGINT received, stopping launchers...")
        self.stop_launchers()

    def start_logging(self):
        # Remove old log files of the controller and engine
        if self.clean_logs:
            log_dir = self.profile_dir.log_dir
            for f in os.listdir(log_dir):
                if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f):
                    os.remove(os.path.join(log_dir, f))
        # This will remove old log files for ipcluster itself
        # super(IPBaseParallelApplication, self).start_logging()

    def start(self):
        """Start the app for the engines subcommand."""
        self.log.info("IPython cluster: started")
        # First see if the cluster is already running

        # Now log and daemonize
        self.log.info(
            'Starting engines with [daemon=%r]' % self.daemonize
        )
        # TODO: Get daemonize working on Windows or as a Windows Server.
        if self.daemonize:
            if os.name=='posix':
                daemonize()

        dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop)
        dc.start()
        # Now write the new pid file AFTER our new forked pid is active.
        # self.write_pid_file()
        try:
            self.loop.start()
        except KeyboardInterrupt:
            pass
        except zmq.ZMQError as e:
            if e.errno == errno.EINTR:
                pass
            else:
                raise

start_aliases = {}
start_aliases.update(engine_aliases)
start_aliases.update(dict(
    delay='IPClusterStart.delay',
    controller = 'IPClusterStart.controller_launcher_class',
))
start_aliases['clean-logs'] = 'IPClusterStart.clean_logs'

class IPClusterStart(IPClusterEngines):

    name = u'ipcluster'
    description = start_help
    examples = _start_examples
    default_log_level = logging.INFO
    auto_create = Bool(True, config=True,
        help="whether to create the profile_dir if it doesn't exist")
    classes = List()
    def _classes_default(self,):
        from IPython.parallel.apps import launcher
        return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers

    clean_logs = Bool(True, config=True,
        help="whether to cleanup old logs before starting")

    delay = CFloat(1., config=True,
        help="delay (in s) between starting the controller and the engines")

    controller_launcher = Any(config=True, help="Deprecated, use controller_launcher_class")
    def _controller_launcher_changed(self, name, old, new):
        if isinstance(new, basestring):
            # old 0.11-style config
            self.log.warn("WARNING: %s.controller_launcher is deprecated as of 0.12,"
                    " use controller_launcher_class" % self.__class__.__name__)
            self.controller_launcher_class = new
    controller_launcher_class = DottedObjectName('LocalControllerLauncher',
        config=True,
        help="""The class for launching a Controller. Change this value if you want
        your controller to also be launched by a batch system, such as PBS,SGE,MPI,etc.

        Each launcher class has its own set of configuration options, for making sure
        it will work in your environment.

        Note that using a batch launcher for the controller *does not* put it
        in the same batch job as the engines, so they will still start separately.

        IPython's bundled examples include:

            Local : start engines locally as subprocesses
            MPI : use mpiexec to launch the controller in an MPI universe
            PBS : use PBS (qsub) to submit the controller to a batch queue
            SGE : use SGE (qsub) to submit the controller to a batch queue
            LSF : use LSF (bsub) to submit the controller to a batch queue
            SSH : use SSH to start the controller
            WindowsHPC : use Windows HPC

        If you are using one of IPython's builtin launchers, you can specify just the
        prefix, e.g:

            c.IPClusterStart.controller_launcher_class = 'SSH'

        or:

            ipcluster start --controller=MPI

        """
        )
    reset = Bool(False, config=True,
        help="Whether to reset config files as part of '--create'."
        )

    # flags = Dict(flags)
    aliases = Dict(start_aliases)

    def init_launchers(self):
        self.controller_launcher = self.build_launcher(self.controller_launcher_class, 'Controller')
        self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet')

    def engines_stopped(self, r):
        """prevent parent.engines_stopped from stopping everything on engine shutdown"""
        pass

    def start_controller(self):
        self.log.info("Starting Controller with %s", self.controller_launcher_class)
        self.controller_launcher.on_stop(self.stop_launchers)
        self.controller_launcher.start()

    def stop_controller(self):
        # self.log.info("In stop_controller")
        if self.controller_launcher and self.controller_launcher.running:
            return self.controller_launcher.stop()

    def stop_launchers(self, r=None):
        if not self._stopping:
            self.stop_controller()
            super(IPClusterStart, self).stop_launchers()

    def start(self):
        """Start the app for the start subcommand."""
        # First see if the cluster is already running
        try:
            pid = self.get_pid_from_file()
        except PIDFileError:
            pass
        else:
            if self.check_pid(pid):
                self.log.critical(
                    'Cluster is already running with [pid=%s]. '
                    'use "ipcluster stop" to stop the cluster.' % pid
                )
                # Here I exit with a unusual exit status that other processes
                # can watch for to learn how I existed.
                self.exit(ALREADY_STARTED)
            else:
                self.remove_pid_file()

        # Now log and daemonize
        self.log.info(
            'Starting ipcluster with [daemon=%r]' % self.daemonize
        )
        # TODO: Get daemonize working on Windows or as a Windows Server.
        if self.daemonize:
            if os.name=='posix':
                daemonize()

        dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop)
        dc.start()
        dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop)
        dc.start()
        # Now write the new pid file AFTER our new forked pid is active.
        self.write_pid_file()
        try:
            self.loop.start()
        except KeyboardInterrupt:
            pass
        except zmq.ZMQError as e:
            if e.errno == errno.EINTR:
                pass
            else:
                raise
        finally:
            self.remove_pid_file()

base='IPython.parallel.apps.ipclusterapp.IPCluster'

class IPClusterApp(Application):
    name = u'ipcluster'
    description = _description
    examples = _main_examples

    subcommands = {
                'start' : (base+'Start', start_help),
                'stop' : (base+'Stop', stop_help),
                'engines' : (base+'Engines', engines_help),
    }

    # no aliases or flags for parent App
    aliases = Dict()
    flags = Dict()

    def start(self):
        if self.subapp is None:
            print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys())
            print
            self.print_description()
            self.print_subcommands()
            self.exit(1)
        else:
            return self.subapp.start()

def launch_new_instance():
    """Create and run the IPython cluster."""
    app = IPClusterApp.instance()
    app.initialize()
    app.start()

if __name__ == '__main__':
    launch_new_instance()


# coding: utf-8
"""
Support for creating GUI apps and starting event loops.

IPython's GUI integration allows interative plotting and GUI usage in IPython
session. IPython has two different types of GUI integration:

1. The terminal based IPython supports GUI event loops through Python's
   PyOS_InputHook. PyOS_InputHook is a hook that Python calls periodically
   whenever raw_input is waiting for a user to type code. We implement GUI
   support in the terminal by setting PyOS_InputHook to a function that
   iterates the event loop for a short while. It is important to note that
   in this situation, the real GUI event loop is NOT run in the normal
   manner, so you can't use the normal means to detect that it is running.
2. In the two process IPython kernel/frontend, the GUI event loop is run in
   the kernel. In this case, the event loop is run in the normal manner by
   calling the function or method of the GUI toolkit that starts the event
   loop.

In addition to starting the GUI event loops in one of these two ways, IPython
will *always* create an appropriate GUI application object when GUi
integration is enabled.

If you want your GUI apps to run in IPython you need to do two things:

1. Test to see if there is already an existing main application object. If
   there is, you should use it. If there is not an existing application object
   you should create one.
2. Test to see if the GUI event loop is running. If it is, you should not
   start it. If the event loop is not running you may start it.

This module contains functions for each toolkit that perform these things
in a consistent manner. Because of how PyOS_InputHook runs the event loop
you cannot detect if the event loop is running using the traditional calls
(such as ``wx.GetApp.IsMainLoopRunning()`` in wxPython). If PyOS_InputHook is
set These methods will return a false negative. That is, they will say the
event loop is not running, when is actually is. To work around this limitation
we proposed the following informal protocol:

* Whenever someone starts the event loop, they *must* set the ``_in_event_loop``
  attribute of the main application object to ``True``. This should be done
  regardless of how the event loop is actually run.
* Whenever someone stops the event loop, they *must* set the ``_in_event_loop``
  attribute of the main application object to ``False``.
* If you want to see if the event loop is running, you *must* use ``hasattr``
  to see if ``_in_event_loop`` attribute has been set. If it is set, you
  *must* use its value. If it has not been set, you can query the toolkit
  in the normal manner.
* If you want GUI support and no one else has created an application or
  started the event loop you *must* do this. We don't want projects to
  attempt to defer these things to someone else if they themselves need it.

The functions below implement this logic for each GUI toolkit. If you need
to create custom application subclasses, you will likely have to modify this
code for your own purposes. This code can be copied into your own project
so you don't have to depend on IPython.

"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# wx
#-----------------------------------------------------------------------------

def get_app_wx(*args, **kwargs):
    """Create a new wx app or return an exiting one."""
    import wx
    app = wx.GetApp()
    if app is None:
        if not kwargs.has_key('redirect'):
            kwargs['redirect'] = False
        app = wx.PySimpleApp(*args, **kwargs)
    return app

def is_event_loop_running_wx(app=None):
    """Is the wx event loop running."""
    if app is None:
        app = get_app_wx()
    if hasattr(app, '_in_event_loop'):
        return app._in_event_loop
    else:
        return app.IsMainLoopRunning()

def start_event_loop_wx(app=None):
    """Start the wx event loop in a consistent manner."""
    if app is None:
        app = get_app_wx()
    if not is_event_loop_running_wx(app):
        app._in_event_loop = True
        app.MainLoop()
        app._in_event_loop = False
    else:
        app._in_event_loop = True

#-----------------------------------------------------------------------------
# qt4
#-----------------------------------------------------------------------------

def get_app_qt4(*args, **kwargs):
    """Create a new qt4 app or return an existing one."""
    from IPython.external.qt_for_kernel import QtGui
    app = QtGui.QApplication.instance()
    if app is None:
        if not args:
            args = ([''],)
        app = QtGui.QApplication(*args, **kwargs)
    return app

def is_event_loop_running_qt4(app=None):
    """Is the qt4 event loop running."""
    if app is None:
        app = get_app_qt4([''])
    if hasattr(app, '_in_event_loop'):
        return app._in_event_loop
    else:
        # Does qt4 provide a other way to detect this?
        return False

def start_event_loop_qt4(app=None):
    """Start the qt4 event loop in a consistent manner."""
    if app is None:
        app = get_app_qt4([''])
    if not is_event_loop_running_qt4(app):
        app._in_event_loop = True
        app.exec_()
        app._in_event_loop = False
    else:
        app._in_event_loop = True

#-----------------------------------------------------------------------------
# Tk
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# gtk
#-----------------------------------------------------------------------------


import os.path, sys, fnmatch
from distutils.command.build_py import build_py as _build_py
from distutils.util import convert_path
from glob import glob

try:
    from distutils.util import Mixin2to3 as _Mixin2to3
    # add support for converting doctests that is missing in 3.1 distutils
    from distutils import log
    from lib2to3.refactor import RefactoringTool, get_fixers_from_package
    import setuptools
    class DistutilsRefactoringTool(RefactoringTool):
        def log_error(self, msg, *args, **kw):
            log.error(msg, *args)

        def log_message(self, msg, *args):
            log.info(msg, *args)

        def log_debug(self, msg, *args):
            log.debug(msg, *args)

    class Mixin2to3(_Mixin2to3):
        def run_2to3(self, files, doctests = False):
            # See of the distribution option has been set, otherwise check the
            # setuptools default.
            if self.distribution.use_2to3 is not True:
                return
            if not files:
                return
            log.info("Fixing "+" ".join(files))
            self.__build_fixer_names()
            self.__exclude_fixers()
            if doctests:
                if setuptools.run_2to3_on_doctests:
                    r = DistutilsRefactoringTool(self.fixer_names)
                    r.refactor(files, write=True, doctests_only=True)
            else:
                _Mixin2to3.run_2to3(self, files)

        def __build_fixer_names(self):
            if self.fixer_names: return
            self.fixer_names = []
            for p in setuptools.lib2to3_fixer_packages:
                self.fixer_names.extend(get_fixers_from_package(p))
            if self.distribution.use_2to3_fixers is not None:
                for p in self.distribution.use_2to3_fixers:
                    self.fixer_names.extend(get_fixers_from_package(p))

        def __exclude_fixers(self):
            excluded_fixers = getattr(self, 'exclude_fixers', [])
            if self.distribution.use_2to3_exclude_fixers is not None:
                excluded_fixers.extend(self.distribution.use_2to3_exclude_fixers)
            for fixer_name in excluded_fixers:
                if fixer_name in self.fixer_names:
                    self.fixer_names.remove(fixer_name)

except ImportError:
    class Mixin2to3:
        def run_2to3(self, files, doctests=True):
            # Nothing done in 2.x
            pass

class build_py(_build_py, Mixin2to3):
    """Enhanced 'build_py' command that includes data files with packages

    The data files are specified via a 'package_data' argument to 'setup()'.
    See 'setuptools.dist.Distribution' for more details.

    Also, this version of the 'build_py' command allows you to specify both
    'py_modules' and 'packages' in the same setup operation.
    """
    def finalize_options(self):
        _build_py.finalize_options(self)
        self.package_data = self.distribution.package_data
        self.exclude_package_data = self.distribution.exclude_package_data or {}
        if 'data_files' in self.__dict__: del self.__dict__['data_files']
        self.__updated_files = []
        self.__doctests_2to3 = []

    def run(self):
        """Build modules, packages, and copy data files to build directory"""
        if not self.py_modules and not self.packages:
            return

        if self.py_modules:
            self.build_modules()

        if self.packages:
            self.build_packages()
            self.build_package_data()

        self.run_2to3(self.__updated_files, False)
        self.run_2to3(self.__updated_files, True)
        self.run_2to3(self.__doctests_2to3, True)

        # Only compile actual .py files, using our base class' idea of what our
        # output files are.
        self.byte_compile(_build_py.get_outputs(self, include_bytecode=0))

    def __getattr__(self,attr):
        if attr=='data_files':  # lazily compute data files
            self.data_files = files = self._get_data_files(); return files
        return _build_py.__getattr__(self,attr)

    def build_module(self, module, module_file, package):
        outfile, copied = _build_py.build_module(self, module, module_file, package)
        if copied:
            self.__updated_files.append(outfile)
        return outfile, copied

    def _get_data_files(self):
        """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""
        self.analyze_manifest()
        data = []
        for package in self.packages or ():
            # Locate package source directory
            src_dir = self.get_package_dir(package)

            # Compute package build directory
            build_dir = os.path.join(*([self.build_lib] + package.split('.')))

            # Length of path to strip from found files
            plen = len(src_dir)+1

            # Strip directory from globbed filenames
            filenames = [
                file[plen:] for file in self.find_data_files(package, src_dir)
                ]
            data.append( (package, src_dir, build_dir, filenames) )
        return data

    def find_data_files(self, package, src_dir):
        """Return filenames for package's data files in 'src_dir'"""
        globs = (self.package_data.get('', [])
                 + self.package_data.get(package, []))
        files = self.manifest_files.get(package, [])[:]
        for pattern in globs:
            # Each pattern has to be converted to a platform-specific path
            files.extend(glob(os.path.join(src_dir, convert_path(pattern))))
        return self.exclude_data_files(package, src_dir, files)

    def build_package_data(self):
        """Copy data files into build directory"""
        lastdir = None
        for package, src_dir, build_dir, filenames in self.data_files:
            for filename in filenames:
                target = os.path.join(build_dir, filename)
                self.mkpath(os.path.dirname(target))
                srcfile = os.path.join(src_dir, filename)
                outf, copied = self.copy_file(srcfile, target)
                srcfile = os.path.abspath(srcfile)
                if copied and srcfile in self.distribution.convert_2to3_doctests:
                    self.__doctests_2to3.append(outf)

    def analyze_manifest(self):
        self.manifest_files = mf = {}
        if not self.distribution.include_package_data:
            return
        src_dirs = {}
        for package in self.packages or ():
            # Locate package source directory
            src_dirs[assert_relative(self.get_package_dir(package))] = package

        self.run_command('egg_info')
        ei_cmd = self.get_finalized_command('egg_info')
        for path in ei_cmd.filelist.files:
            d,f = os.path.split(assert_relative(path))
            prev = None
            oldf = f
            while d and d!=prev and d not in src_dirs:
                prev = d
                d, df = os.path.split(d)
                f = os.path.join(df, f)
            if d in src_dirs:
                if path.endswith('.py') and f==oldf:
                    continue    # it's a module, not data
                mf.setdefault(src_dirs[d],[]).append(path)

    def get_data_files(self): pass  # kludge 2.4 for lazy computation

    if sys.version<"2.4":    # Python 2.4 already has this code
        def get_outputs(self, include_bytecode=1):
            """Return complete list of files copied to the build directory

            This includes both '.py' files and data files, as well as '.pyc'
            and '.pyo' files if 'include_bytecode' is true.  (This method is
            needed for the 'install_lib' command to do its job properly, and to
            generate a correct installation manifest.)
            """
            return _build_py.get_outputs(self, include_bytecode) + [
                os.path.join(build_dir, filename)
                for package, src_dir, build_dir,filenames in self.data_files
                for filename in filenames
                ]

    def check_package(self, package, package_dir):
        """Check namespace packages' __init__ for declare_namespace"""
        try:
            return self.packages_checked[package]
        except KeyError:
            pass

        init_py = _build_py.check_package(self, package, package_dir)
        self.packages_checked[package] = init_py

        if not init_py or not self.distribution.namespace_packages:
            return init_py

        for pkg in self.distribution.namespace_packages:
            if pkg==package or pkg.startswith(package+'.'):
                break
        else:
            return init_py

        f = open(init_py,'rbU')
        if 'declare_namespace'.encode() not in f.read():
            from distutils import log
            log.warn(
               "WARNING: %s is a namespace package, but its __init__.py does\n"
               "not declare_namespace(); setuptools 0.7 will REQUIRE this!\n"
               '(See the setuptools manual under "Namespace Packages" for '
               "details.)\n", package
            )
        f.close()
        return init_py

    def initialize_options(self):
        self.packages_checked={}
        _build_py.initialize_options(self)

    def get_package_dir(self, package):
        res = _build_py.get_package_dir(self, package)
        if self.distribution.src_root is not None:
            return os.path.join(self.distribution.src_root, res)
        return res

    def exclude_data_files(self, package, src_dir, files):
        """Filter filenames for package's data files in 'src_dir'"""
        globs = (self.exclude_package_data.get('', [])
                 + self.exclude_package_data.get(package, []))
        bad = []
        for pattern in globs:
            bad.extend(
                fnmatch.filter(
                    files, os.path.join(src_dir, convert_path(pattern))
                )
            )
        bad = dict.fromkeys(bad)
        seen = {}
        return [
            f for f in files if f not in bad
                and f not in seen and seen.setdefault(f,1)  # ditch dupes
        ]

def assert_relative(path):
    if not os.path.isabs(path):
        return path
    from distutils.errors import DistutilsSetupError
    raise DistutilsSetupError(
"""Error: setup script specifies an absolute path:

    %s

setup() arguments must *always* be /-separated paths relative to the
setup.py directory, *never* absolute paths.
""" % path
    )


"""Module for creating illustrations.

To create an annotated image we need an instance of the
:class:`jicbioimage.illustrate.AnnotatedImage` class.

>>> from jicbioimage.illustrate import AnnotatedImage

Suppose that we have an existing image.

>>> from jicbioimage.core.image import Image
>>> im = Image((50,50))

We can use this image to create an canvas instance populated with the data
as a RGB gray scale image.

>>> canvas = AnnotatedImage.from_grayscale(im)

The :class:`jicbioimage.illustrate.Canvas` instance has built in annotation
functionality.

One can use it to draw crosses.

>>> canvas.draw_cross(10, 20)

One can use it to mask out bitmaps (in the example below with the color cyan).

>>> bitmap = np.zeros((50, 50), dtype=bool)
>>> bitmap[30:40, 30:40] = True
>>> canvas.mask_region(bitmap, color=(0, 255, 255))

One can use it to add text at particular locations on the canvas.

>>> canvas.text_at("Hello", 30, 60)

"""

import os.path

import PIL.ImageFont
import numpy as np
import skimage.draw

import jicbioimage.core.image

__version__ = "0.6.1"

HERE = os.path.dirname(__file__)
DEFAULT_FONT_PATH = os.path.join(HERE, "fonts", "UbuntuMono-R.ttf")

class Canvas(jicbioimage.core.image._BaseImage):
    """Class for building up annotated images."""

    @staticmethod
    def blank_canvas(width, height):
        """Return a blank canvas to annotate.

        :param width: xdim (int)
        :param height: ydim (int)
        :returns: :class:`jicbioimage.illustrate.Canvas`
        """
        canvas = np.zeros((height, width, 3), dtype=np.uint8)
        return canvas.view(Canvas)

    def draw_cross(self, position, color=(255, 0, 0), radius=4):
        """Draw a cross on the canvas.

        :param position: (row, col) tuple
        :param color: RGB tuple
        :param radius: radius of the cross (int)
        """
        y, x = position
        for xmod in np.arange(-radius, radius+1, 1):
            xpos = x + xmod
            if xpos < 0:
                continue  # Negative indices will draw on the opposite side.
            if xpos >= self.shape[1]:
                continue  # Out of bounds.
            self[int(y), int(xpos)] = color
        for ymod in np.arange(-radius, radius+1, 1):
            ypos = y + ymod
            if ypos < 0:
                continue  # Negative indices will draw on the opposite side.
            if ypos >= self.shape[0]:
                continue  # Out of bounds.
            self[int(ypos), int(x)] = color

    def draw_line(self, pos1, pos2, color=(255, 0, 0)):
        """Draw a line between pos1 and pos2 on the canvas.

        :param pos1: position 1 (row, col) tuple
        :param pos2: position 2 (row, col) tuple
        :param color: RGB tuple
        """
        r1, c1 = tuple([int(round(i, 0)) for i in pos1])
        r2, c2 = tuple([int(round(i, 0)) for i in pos2])
        rr, cc = skimage.draw.line(r1, c1, r2, c2)
        self[rr, cc] = color

    def mask_region(self, region, color=(0, 255, 0)):
        """Mask a region with a color.

        :param region: :class:`jicbioimage.core.region.Region`
        :param color: RGB tuple
        """
        self[region] = color

    def text_at(self, text, position, color=(255, 255, 255),
                size=12, antialias=False, center=False):
        """Write text at x, y top left corner position.

        By default the x and y coordinates represent the top left hand corner
        of the text. The text can be centered vertically and horizontally by
        using setting the ``center`` option to ``True``.

        :param text: text to write
        :param position: (row, col) tuple
        :param color: RGB tuple
        :param size: font size
        :param antialias: whether or not the text should be antialiased
        :param center: whether or not the text should be centered on the
                       input coordinate
        """
        def antialias_value(value, normalisation):
            return int(round(value * normalisation))

        def antialias_rgb(color, normalisation):
            return tuple([antialias_value(v, normalisation) for v in color])

        def set_color(xpos, ypos, color):
            try:
                self[ypos, xpos] = color
            except IndexError:
                pass

        y, x = position
        font = PIL.ImageFont.truetype(DEFAULT_FONT_PATH, size=size)
        mask = font.getmask(text)
        width, height = mask.size
        if center:
            x = x - (width // 2)
            y = y - (height // 2)
        for ystep in range(height):
            for xstep in range(width):
                normalisation = mask[ystep * width + xstep] / 255.
                if antialias:
                    if normalisation != 0:
                        rgb_color = antialias_rgb(color, normalisation)
                        set_color(x + xstep, y+ystep, rgb_color)
                else:
                    if normalisation > .5:
                        set_color(x + xstep, y + ystep, color)

class AnnotatedImage(Canvas):
    """Class for building up annotated images."""

    @staticmethod
    def from_grayscale(im, channels_on=(True, True, True)):
        """Return a canvas from a grayscale image.

        :param im: single channel image
        :channels_on: channels to populate with input image
        :returns: :class:`jicbioimage.illustrate.Canvas`
        """
        xdim, ydim = im.shape
        canvas = np.zeros((xdim, ydim, 3), dtype=np.uint8)
        for i, include in enumerate(channels_on):
            if include:
                canvas[:, :, i] = im
        return canvas.view(AnnotatedImage)


import re
import uuid
import datetime
import hashlib
import urllib

from django.conf import settings
from django.utils.encoding import smart_str
from django.contrib.sites.models import Site
from django.core.exceptions import ValidationError
from django.core.validators import validate_email
from django.utils import timezone

simple_email_re = re.compile(r'^\S+@\S+\.\S+$')

def get_uuid(length=32, version=1):
    """
    Returns a unique ID of a given length.
    User `version=2` for cross-systems uniqueness.
    """
    if version == 1:
        return uuid.uuid1().hex[:length]
    else:
        return uuid.uuid4().hex[:length]

def get_integer(value='0'):
    """
    Converts string to integer.
    If string has non-numerical character, it returns None
    """
    try:
        return int(value)
    except ValueError:
        return None

def get_hashed(key):
    """
    Returns a hashed version of a given key.
    One way encryption.
    """
    return hashlib.md5(key.encode('utf-8')).hexdigest()

def get_days_ago(days=0):
    """
    Return X 'days' ago in datetime format
    """
    return (timezone.now() - datetime.timedelta(days))

def get_days_from_now(days=0):
    """
    Return X 'days' from today in datetime format
    """
    return (timezone.now() + datetime.timedelta(days))

def get_dict_to_encoded_url(data):
    """
    Converts a dict to an encoded URL.
    Example: given  data = {'a': 1, 'b': 2}, it returns 'a=1&b=2'
    """
    unicode_data = dict([(k, smart_str(v)) for k, v in data.items()])
    encoded = urllib.urlencode(unicode_data)
    return encoded

def get_encoded_url_to_dict(string):
    """
    Converts an encoded URL to a dict.
    Example: given string = 'a=1&b=2' it returns {'a': 1, 'b': 2}
    """
    data = urllib.parse.parse_qsl(string, keep_blank_values=True)
    data = dict(data)
    return data

def remove_csrf_from_params_dict(data):
    """
    Removes csrf token from a dict
    """
    pd = data
    try:
        del pd['csrfmiddlewaretoken']
    except KeyError:
        pass
    return pd

def get_unique_key_from_get(get_dict):
    """
    Build a unique key from get data
    """
    site = Site.objects.get_current()
    key = get_dict_to_encoded_url(get_dict)
    cache_key = '{}_{}'.format(site.domain, key)
    return hashlib.md5(cache_key).hexdigest()

def get_unique_key_from_post(post_dict):
    """
    Build a unique key from post data
    """
    post_dict = remove_csrf_from_params_dict(post_dict)
    return get_unique_key_from_get(post_dict)

def tobin(deci_num, len=32):
    """
    Given a decimal number, returns a string bitfield of length = len
    Example: given deci_num = 1 and len = 10, it return 0000000001
    """
    bitstr = "".join(map(lambda y: str((deci_num >> y) & 1), range(len - 1, -1, -1)))
    return bitstr

def is_valid_email(email):
    """
    Validates and email address.
    Note: valid emails must follow the <name>@<domain><.extension> patterns.
    """
    try:
        validate_email(email)
    except ValidationError:
        return False
    if simple_email_re.match(email):
        return True
    return False

def get_domain(url):
    """ Returns domain name portion of a URL """
    if 'http' not in url.lower():
        url = 'http://{}'.format(url)
    return urllib.parse.urlparse(url).hostname

def get_url_args(url):
    """ Returns a dictionary from a URL params """
    url_data = urllib.parse.urlparse(url)
    arg_dict = urllib.parse.parse_qs(url_data.query)
    return arg_dict

def get_site_proto():
    """ Retuns the protocol of site, or http """
    proto = getattr(settings, 'PROTOCOL', 'http')
    return proto