from __future__ import absolute_import

import logging
import os

from pip.basecommand import Command
from pip.status_codes import SUCCESS, ERROR
from pip._vendor import pkg_resources

logger = logging.getLogger(__name__)

class ShowCommand(Command):
    """Show information about one or more installed packages."""
    name = 'show'
    usage = """
      %prog [options] <package> ..."""
    summary = 'Show information about installed packages.'

    def __init__(self, *args, **kw):
        super(ShowCommand, self).__init__(*args, **kw)
        self.cmd_opts.add_option(
            '-f', '--files',
            dest='files',
            action='store_true',
            default=False,
            help='Show the full list of installed files for each package.')

        self.parser.insert_option_group(0, self.cmd_opts)

    def run(self, options, args):
        if not args:
            logger.warning('ERROR: Please provide a package name or names.')
            return ERROR
        query = args

        results = search_packages_info(query)
        if not print_results(results, options.files):
            return ERROR
        return SUCCESS

def search_packages_info(query):
    """
    Gather details from installed distributions. Print distribution name,
    version, location, and installed files. Installed files requires a
    pip generated 'installed-files.txt' in the distributions '.egg-info'
    directory.
    """
    installed = dict(
        [(p.project_name.lower(), p) for p in pkg_resources.working_set])
    query_names = [name.lower() for name in query]
    for dist in [installed[pkg] for pkg in query_names if pkg in installed]:
        package = {
            'name': dist.project_name,
            'version': dist.version,
            'location': dist.location,
            'requires': [dep.project_name for dep in dist.requires()],
        }
        file_list = None
        if isinstance(dist, pkg_resources.DistInfoDistribution):
            # RECORDs should be part of .dist-info metadatas
            if dist.has_metadata('RECORD'):
                lines = dist.get_metadata_lines('RECORD')
                paths = [l.split(',')[0] for l in lines]
                paths = [os.path.join(dist.location, p) for p in paths]
                file_list = [os.path.relpath(p, dist.location) for p in paths]
        else:
            # Otherwise use pip's log for .egg-info's
            if dist.has_metadata('installed-files.txt'):
                paths = dist.get_metadata_lines('installed-files.txt')
                paths = [os.path.join(dist.egg_info, p) for p in paths]
                file_list = [os.path.relpath(p, dist.location) for p in paths]

        # use and short-circuit to check for None
        package['files'] = file_list and sorted(file_list)
        yield package

def print_results(distributions, list_all_files):
    """
    Print the informations from installed distributions found.
    """
    results_printed = False
    for dist in distributions:
        results_printed = True
        logger.info("---")
        logger.info("Name: %s" % dist['name'])
        logger.info("Version: %s" % dist['version'])
        logger.info("Location: %s" % dist['location'])
        logger.info("Requires: %s" % ', '.join(dist['requires']))
        if list_all_files:
            logger.info("Files:")
            if dist['files'] is not None:
                for line in dist['files']:
                    logger.info("  %s" % line.strip())
            else:
                logger.info("Cannot locate installed-files.txt")
    return results_printed


"""Print the metadata for one or more Python package distributions.

Usage:  %prog [options] path+

Each 'path' entry can be one of the following:

o a source distribution:  in this case, 'path' should point to an existing
  archive file (.tar.gz, .tar.bz2, or .zip) as generated by 'setup.py sdist'.

o a binary distribution:  in this case, 'path' should point to an existing
  archive file (.egg)

o a "develop" checkout:  in ths case,  'path' should point to a directory
  intialized via 'setup.py develop' (under setuptools).

o an installed package:  in this case, 'path' should be the importable name
  of the package.
"""
try:
    from configparser import ConfigParser
except ImportError:  # pragma: NO COVER
    from ConfigParser import ConfigParser
from csv import writer
import optparse
import os
import sys

from pkginfo import get_metadata

def _parse_options(args=None):
    parser = optparse.OptionParser(usage=__doc__)

    parser.add_option("-m", "--metadata-version", default=None,
                      help="Override metadata version")

    parser.add_option("-f", "--field", dest="fields", action="append",
                      help="Specify an output field (repeatable)",
                      )

    parser.add_option("-d", "--download-url-prefix",
                      dest="download_url_prefix",
                      help="Download URL prefix",
                      )

    parser.add_option("--simple", dest="output", action="store_const",
                      const='simple', default='simple',
                      help="Output as simple key-value pairs",
                      )

    parser.add_option("-s", "--skip", dest="skip", action="store_true",
                      default=True,
                      help="Skip missing values in simple output",
                     )

    parser.add_option("-S", "--no-skip", dest="skip", action="store_false",
                      help="Don't skip missing values in simple output",
                     )

    parser.add_option("--single", dest="output", action="store_const",
                      const='single',
                      help="Output delimited values",
                      )

    parser.add_option("--item-delim", dest="item_delim", action="store",
                      default=';',
                      help="Delimiter for fields in single-line output",
                      )

    parser.add_option("--sequence-delim", dest="sequence_delim",
                      action="store", default=',',
                      help="Delimiter for multi-valued fields",
                      )

    parser.add_option("--csv", dest="output", action="store_const",
                      const='csv',
                      help="Output as CSV",
                      )

    parser.add_option("--ini", dest="output", action="store_const",
                      const='ini',
                      help="Output as INI",
                      )

    options, args = parser.parse_args(args)

    if len(args)==0:
        parser.error("Pass one or more files or directories as arguments.")
    else:
        return options, args

class Base(object):
    _fields = None
    def __init__(self, options):
        if options.fields:
            self._fields = options.fields

    def finish(self):  # pragma: NO COVER
        pass

class Simple(Base):
    def __init__(self, options):
        super(Simple, self).__init__(options)
        self._skip = options.skip

    def __call__(self, meta):
        for field in self._fields or list(meta):
            value = getattr(meta, field)
            if (not self._skip) or (value is not None and value!=()):
                print("%s: %s" % (field, value))

class SingleLine(Base):
    _fields = None
    def __init__(self, options):
        super(SingleLine, self).__init__(options)
        self._item_delim = options.item_delim
        self._sequence_delim = options.sequence_delim

    def __call__(self, meta):
        if self._fields is None:
            self._fields = list(meta)
        values = []
        for field in self._fields:
            value = getattr(meta, field)
            if isinstance(value, (tuple, list)):
                value = self._sequence_delim.join(value)
            else:
                value = str(value)
            values.append(value)
        print(self._item_delim.join(values))

class CSV(Base):
    _writer = None
    def __init__(self, options):
        super(CSV, self).__init__(options)
        self._sequence_delim = options.sequence_delim

    def __call__(self, meta):
        if self._fields is None:
            self._fields = list(meta) # first dist wins
        fields = self._fields
        if self._writer is None:
            self._writer = writer(sys.stdout)
            self._writer.writerow(fields)
        values = []
        for field in fields:
            value = getattr(meta, field)
            if isinstance(value, (tuple, list)):
                value = self._sequence_delim.join(value)
            else:
                value = str(value)
            values.append(value)
        self._writer.writerow(values)

class INI(Base):
    _fields = None
    def __init__(self, options):
        super(INI, self).__init__(options)
        self._parser = ConfigParser()

    def __call__(self, meta):
        name = meta.name
        version = meta.version
        section = '%s-%s' % (name, version)
        if self._parser.has_section(section):
            raise ValueError('Duplicate distribution: %s' % section)
        self._parser.add_section(section)
        for field in self._fields or list(meta):
            value = getattr(meta, field)
            if isinstance(value, (tuple, list)):
                value = '\n\t'.join(value)
            self._parser.set(section, field, value)

    def finish(self):
        self._parser.write(sys.stdout)  # pragma: NO COVER

_FORMATTERS = {
    'simple': Simple,
    'single': SingleLine,
    'csv': CSV,
    'ini': INI,
}

def main(args=None):
    """Entry point for pkginfo tool
    """
    options, paths = _parse_options(args)
    format = getattr(options, 'output', 'simple')
    formatter = _FORMATTERS[format](options)

    for path in paths:
        meta = get_metadata(path, options.metadata_version)
        if meta is None:
            continue

        if options.download_url_prefix:
            if meta.download_url is None:
                filename = os.path.basename(path)
                meta.download_url = '%s/%s' % (options.download_url_prefix,
                                               filename)

        formatter(meta)

    formatter.finish()


# System library imports
from pygments.token import Token, is_token_subtype

class CompletionLexer(object):
    """ Uses Pygments and some auxillary information to lex code snippets for
        symbol contexts.
    """

    # Maps Lexer names to a list of possible name separators
    separator_map = { 'C' : [ '.', '->' ],
                      'C++' : [ '.', '->', '::' ],
                      'Python' : [ '.' ] }

    def __init__(self, lexer):
        """ Create a CompletionLexer using the specified Pygments lexer.
        """
        self.lexer = lexer

    def get_context(self, string):
        """ Assuming the cursor is at the end of the specified string, get the
            context (a list of names) for the symbol at cursor position.
        """
        context = []
        reversed_tokens = list(self._lexer.get_tokens(string))
        reversed_tokens.reverse()

        # Pygments often tacks on a newline when none is specified in the input.
        # Remove this newline.
        if reversed_tokens and reversed_tokens[0][1].endswith('\n') and \
                not string.endswith('\n'):
            reversed_tokens.pop(0)

        current_op = ''
        for token, text in reversed_tokens:

            if is_token_subtype(token, Token.Name):

                # Handle a trailing separator, e.g 'foo.bar.'
                if current_op in self._name_separators:
                    if not context:
                        context.insert(0, '')

                # Handle non-separator operators and punction.
                elif current_op:
                    break

                context.insert(0, text)
                current_op = ''

            # Pygments doesn't understand that, e.g., '->' is a single operator
            # in C++. This is why we have to build up an operator from
            # potentially several tokens.
            elif token is Token.Operator or token is Token.Punctuation:
                current_op = text + current_op

            # Break on anything that is not a Operator, Punctuation, or Name.
            else:
                break

        return context

    def get_lexer(self, lexer):
        return self._lexer

    def set_lexer(self, lexer, name_separators=None):
        self._lexer = lexer
        if name_separators is None:
            self._name_separators = self.separator_map.get(lexer.name, ['.'])
        else:
            self._name_separators = list(name_separators)

    lexer = property(get_lexer, set_lexer)


# encoding: utf-8
"""
An object for managing IPython profile directories.

Authors:

* Brian Granger
* Fernando Perez
* Min RK

"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import os
import shutil
import sys

from IPython.config.configurable import LoggingConfigurable
from IPython.config.loader import Config
from IPython.utils.path import get_ipython_package_dir, expand_path
from IPython.utils.traitlets import List, Unicode, Bool

#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Module errors
#-----------------------------------------------------------------------------

class ProfileDirError(Exception):
    pass

#-----------------------------------------------------------------------------
# Class for managing profile directories
#-----------------------------------------------------------------------------

class ProfileDir(LoggingConfigurable):
    """An object to manage the profile directory and its resources.

    The profile directory is used by all IPython applications, to manage
    configuration, logging and security.

    This object knows how to find, create and manage these directories. This
    should be used by any code that wants to handle profiles.
    """

    security_dir_name = Unicode('security')
    log_dir_name = Unicode('log')
    startup_dir_name = Unicode('startup')
    pid_dir_name = Unicode('pid')
    security_dir = Unicode(u'')
    log_dir = Unicode(u'')
    startup_dir = Unicode(u'')
    pid_dir = Unicode(u'')

    location = Unicode(u'', config=True,
        help="""Set the profile location directly. This overrides the logic used by the
        `profile` option.""",
        )

    _location_isset = Bool(False) # flag for detecting multiply set location

    def _location_changed(self, name, old, new):
        if self._location_isset:
            raise RuntimeError("Cannot set profile location more than once.")
        self._location_isset = True
        if not os.path.isdir(new):
            os.makedirs(new)

        # ensure config files exist:
        self.security_dir = os.path.join(new, self.security_dir_name)
        self.log_dir = os.path.join(new, self.log_dir_name)
        self.startup_dir = os.path.join(new, self.startup_dir_name)
        self.pid_dir = os.path.join(new, self.pid_dir_name)
        self.check_dirs()

    def _log_dir_changed(self, name, old, new):
        self.check_log_dir()

    def check_log_dir(self):
        if not os.path.isdir(self.log_dir):
            os.mkdir(self.log_dir)

    def _startup_dir_changed(self, name, old, new):
        self.check_startup_dir()

    def check_startup_dir(self):
        if not os.path.isdir(self.startup_dir):
            os.mkdir(self.startup_dir)
        readme = os.path.join(self.startup_dir, 'README')
        src = os.path.join(get_ipython_package_dir(), u'config', u'profile', u'README_STARTUP')
        if not os.path.exists(readme):
            shutil.copy(src, readme)

    def _security_dir_changed(self, name, old, new):
        self.check_security_dir()

    def check_security_dir(self):
        if not os.path.isdir(self.security_dir):
            os.mkdir(self.security_dir, 0700)
        else:
            try:
                os.chmod(self.security_dir, 0700)
            except OSError:
                self.log.warn("Could not set security dir permissions to private.")

    def _pid_dir_changed(self, name, old, new):
        self.check_pid_dir()

    def check_pid_dir(self):
        if not os.path.isdir(self.pid_dir):
            os.mkdir(self.pid_dir, 0700)
        else:
            try:
                os.chmod(self.pid_dir, 0700)
            except OSError:
                self.log.warn("Could not set pid dir permissions to private.")

    def check_dirs(self):
        self.check_security_dir()
        self.check_log_dir()
        self.check_pid_dir()
        self.check_startup_dir()

    def copy_config_file(self, config_file, path=None, overwrite=False):
        """Copy a default config file into the active profile directory.

        Default configuration files are kept in :mod:`IPython.config.default`.
        This function moves these from that location to the working profile
        directory.
        """
        dst = os.path.join(self.location, config_file)
        if os.path.isfile(dst) and not overwrite:
            return False
        if path is None:
            path = os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default')
        src = os.path.join(path, config_file)
        shutil.copy(src, dst)
        return True

    @classmethod
    def create_profile_dir(cls, profile_dir, config=None):
        """Create a new profile directory given a full path.

        Parameters
        ----------
        profile_dir : str
            The full path to the profile directory.  If it does exist, it will
            be used.  If not, it will be created.
        """
        return cls(location=profile_dir, config=config)

    @classmethod
    def create_profile_dir_by_name(cls, path, name=u'default', config=None):
        """Create a profile dir by profile name and path.

        Parameters
        ----------
        path : unicode
            The path (directory) to put the profile directory in.
        name : unicode
            The name of the profile.  The name of the profile directory will
            be "profile_<profile>".
        """
        if not os.path.isdir(path):
            raise ProfileDirError('Directory not found: %s' % path)
        profile_dir = os.path.join(path, u'profile_' + name)
        return cls(location=profile_dir, config=config)

    @classmethod
    def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None):
        """Find an existing profile dir by profile name, return its ProfileDir.

        This searches through a sequence of paths for a profile dir.  If it
        is not found, a :class:`ProfileDirError` exception will be raised.

        The search path algorithm is:
        1. ``os.getcwdu()``
        2. ``ipython_dir``

        Parameters
        ----------
        ipython_dir : unicode or str
            The IPython directory to use.
        name : unicode or str
            The name of the profile.  The name of the profile directory
            will be "profile_<profile>".
        """
        dirname = u'profile_' + name
        paths = [os.getcwdu(), ipython_dir]
        for p in paths:
            profile_dir = os.path.join(p, dirname)
            if os.path.isdir(profile_dir):
                return cls(location=profile_dir, config=config)
        else:
            raise ProfileDirError('Profile directory not found in paths: %s' % dirname)

    @classmethod
    def find_profile_dir(cls, profile_dir, config=None):
        """Find/create a profile dir and return its ProfileDir.

        This will create the profile directory if it doesn't exist.

        Parameters
        ----------
        profile_dir : unicode or str
            The path of the profile directory.  This is expanded using
            :func:`IPython.utils.genutils.expand_path`.
        """
        profile_dir = expand_path(profile_dir)
        if not os.path.isdir(profile_dir):
            raise ProfileDirError('Profile directory not found: %s' % profile_dir)
        return cls(location=profile_dir, config=config)


"""
This module contains fixups for using nose under different versions of Python.
"""
import sys
import os
import types
import inspect
import nose.util

__all__ = ['make_instancemethod', 'cmp_to_key', 'sort_list', 'ClassType',
           'TypeType', 'UNICODE_STRINGS', 'unbound_method', 'ismethod',
           'bytes_']

# In Python 3.x, all strings are unicode (the call to 'unicode()' in the 2.x
# source will be replaced with 'str()' when running 2to3, so this test will
# then become true)
UNICODE_STRINGS = (type(unicode()) == type(str()))

# new.instancemethod() is obsolete for new-style classes (Python 3.x)
# We need to use descriptor methods instead.
try:
    import new
    def make_instancemethod(function, instance):
        return new.instancemethod(function.im_func, instance,
                                  instance.__class__)
except ImportError:
    def make_instancemethod(function, instance):
        return function.__get__(instance, instance.__class__)

# To be forward-compatible, we do all list sorts using keys instead of cmp
# functions.  However, part of the unittest.TestLoader API involves a
# user-provideable cmp function, so we need some way to convert that.
def cmp_to_key(mycmp):
    'Convert a cmp= function into a key= function'
    class Key(object):
        def __init__(self, obj):
            self.obj = obj
        def __lt__(self, other):
            return mycmp(self.obj, other.obj) < 0
        def __gt__(self, other):
            return mycmp(self.obj, other.obj) > 0
        def __eq__(self, other):
            return mycmp(self.obj, other.obj) == 0
    return Key

# Python 2.3 also does not support list-sorting by key, so we need to convert
# keys to cmp functions if we're running on old Python..
if sys.version_info < (2, 4):
    def sort_list(l, key, reverse=False):
        if reverse:
            return l.sort(lambda a, b: cmp(key(b), key(a)))
        else:
            return l.sort(lambda a, b: cmp(key(a), key(b)))
else:
    def sort_list(l, key, reverse=False):
        return l.sort(key=key, reverse=reverse)

# In Python 3.x, all objects are "new style" objects descended from 'type', and
# thus types.ClassType and types.TypeType don't exist anymore.  For
# compatibility, we make sure they still work.
if hasattr(types, 'ClassType'):
    ClassType = types.ClassType
    TypeType = types.TypeType
else:
    ClassType = type
    TypeType = type

# The following emulates the behavior (we need) of an 'unbound method' under
# Python 3.x (namely, the ability to have a class associated with a function
# definition so that things can do stuff based on its associated class)
class UnboundMethod:
    def __init__(self, cls, func):
        # Make sure we have all the same attributes as the original function,
        # so that the AttributeSelector plugin will work correctly...
        self.__dict__ = func.__dict__.copy()
        self._func = func
        self.__self__ = UnboundSelf(cls)

    def address(self):
        cls = self.__self__.cls
        modname = cls.__module__
        module = sys.modules[modname]
        filename = getattr(module, '__file__', None)
        if filename is not None:
            filename = os.path.abspath(filename)
        return (nose.util.src(filename), modname, "%s.%s" % (cls.__name__,
                                                        self._func.__name__))

    def __call__(self, *args, **kwargs):
        return self._func(*args, **kwargs)

    def __getattr__(self, attr):
        return getattr(self._func, attr)

    def __repr__(self):
        return '<unbound method %s.%s>' % (self.__self__.cls.__name__,
                                           self._func.__name__)

class UnboundSelf:
    def __init__(self, cls):
        self.cls = cls

    # We have to do this hackery because Python won't let us override the
    # __class__ attribute...
    def __getattribute__(self, attr):
        if attr == '__class__':
            return self.cls
        else:
            return object.__getattribute__(self, attr)

def unbound_method(cls, func):
    if inspect.ismethod(func):
        return func
    if not inspect.isfunction(func):
        raise TypeError('%s is not a function' % (repr(func),))
    return UnboundMethod(cls, func)

def ismethod(obj):
    return inspect.ismethod(obj) or isinstance(obj, UnboundMethod)

# Make a pseudo-bytes function that can be called without the encoding arg:
if sys.version_info >= (3, 0):
    def bytes_(s, encoding='utf8'):
        if isinstance(s, bytes):
            return s
        return bytes(s, encoding)
else:
    def bytes_(s, encoding=None):
        return str(s)


# encoding: utf-8
"""
IO related utilities.
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------
from __future__ import print_function

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
import os
import sys
import tempfile
from StringIO import StringIO

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

class IOStream:

    def __init__(self,stream, fallback=None):
        if not hasattr(stream,'write') or not hasattr(stream,'flush'):
            if fallback is not None:
                stream = fallback
            else:
                raise ValueError("fallback required, but not specified")
        self.stream = stream
        self._swrite = stream.write

        # clone all methods not overridden:
        def clone(meth):
            return not hasattr(self, meth) and not meth.startswith('_')
        for meth in filter(clone, dir(stream)):
            setattr(self, meth, getattr(stream, meth))

    def write(self,data):
        try:
            self._swrite(data)
        except:
            try:
                # print handles some unicode issues which may trip a plain
                # write() call.  Emulate write() by using an empty end
                # argument.
                print(data, end='', file=self.stream)
            except:
                # if we get here, something is seriously broken.
                print('ERROR - failed to write data to stream:', self.stream,
                      file=sys.stderr)

    def writelines(self, lines):
        if isinstance(lines, basestring):
            lines = [lines]
        for line in lines:
            self.write(line)

    # This class used to have a writeln method, but regular files and streams
    # in Python don't have this method. We need to keep this completely
    # compatible so we removed it.

    @property
    def closed(self):
        return self.stream.closed

    def close(self):
        pass

# setup stdin/stdout/stderr to sys.stdin/sys.stdout/sys.stderr
devnull = open(os.devnull, 'a')
stdin = IOStream(sys.stdin, fallback=devnull)
stdout = IOStream(sys.stdout, fallback=devnull)
stderr = IOStream(sys.stderr, fallback=devnull)

class IOTerm:
    """ Term holds the file or file-like objects for handling I/O operations.

    These are normally just sys.stdin, sys.stdout and sys.stderr but for
    Windows they can can replaced to allow editing the strings before they are
    displayed."""

    # In the future, having IPython channel all its I/O operations through
    # this class will make it easier to embed it into other environments which
    # are not a normal terminal (such as a GUI-based shell)
    def __init__(self, stdin=None, stdout=None, stderr=None):
        mymodule = sys.modules[__name__]
        self.stdin  = IOStream(stdin, mymodule.stdin)
        self.stdout = IOStream(stdout, mymodule.stdout)
        self.stderr = IOStream(stderr, mymodule.stderr)

class Tee(object):
    """A class to duplicate an output stream to stdout/err.

    This works in a manner very similar to the Unix 'tee' command.

    When the object is closed or deleted, it closes the original file given to
    it for duplication.
    """
    # Inspired by:
    # http://mail.python.org/pipermail/python-list/2007-May/442737.html

    def __init__(self, file_or_name, mode="w", channel='stdout'):
        """Construct a new Tee object.

        Parameters
        ----------
        file_or_name : filename or open filehandle (writable)
          File that will be duplicated

        mode : optional, valid mode for open().
          If a filename was give, open with this mode.

        channel : str, one of ['stdout', 'stderr']
        """
        if channel not in ['stdout', 'stderr']:
            raise ValueError('Invalid channel spec %s' % channel)

        if hasattr(file_or_name, 'write') and hasattr(file_or_name, 'seek'):
            self.file = file_or_name
        else:
            self.file = open(file_or_name, mode)
        self.channel = channel
        self.ostream = getattr(sys, channel)
        setattr(sys, channel, self)
        self._closed = False

    def close(self):
        """Close the file and restore the channel."""
        self.flush()
        setattr(sys, self.channel, self.ostream)
        self.file.close()
        self._closed = True

    def write(self, data):
        """Write data to both channels."""
        self.file.write(data)
        self.ostream.write(data)
        self.ostream.flush()

    def flush(self):
        """Flush both channels."""
        self.file.flush()
        self.ostream.flush()

    def __del__(self):
        if not self._closed:
            self.close()

def file_read(filename):
    """Read a file and close it.  Returns the file source."""
    fobj = open(filename,'r');
    source = fobj.read();
    fobj.close()
    return source

def file_readlines(filename):
    """Read a file and close it.  Returns the file source using readlines()."""
    fobj = open(filename,'r');
    lines = fobj.readlines();
    fobj.close()
    return lines

def raw_input_multi(header='', ps1='==> ', ps2='..> ',terminate_str = '.'):
    """Take multiple lines of input.

    A list with each line of input as a separate element is returned when a
    termination string is entered (defaults to a single '.'). Input can also
    terminate via EOF (^D in Unix, ^Z-RET in Windows).

    Lines of input which end in \\ are joined into single entries (and a
    secondary continuation prompt is issued as long as the user terminates
    lines with \\). This allows entering very long strings which are still
    meant to be treated as single entities.
    """

    try:
        if header:
            header += '\n'
        lines = [raw_input(header + ps1)]
    except EOFError:
        return []
    terminate = [terminate_str]
    try:
        while lines[-1:] != terminate:
            new_line = raw_input(ps1)
            while new_line.endswith('\\'):
                new_line = new_line[:-1] + raw_input(ps2)
            lines.append(new_line)

        return lines[:-1]  # don't return the termination command
    except EOFError:
        print()
        return lines

def raw_input_ext(prompt='',  ps2='... '):
    """Similar to raw_input(), but accepts extended lines if input ends with \\."""

    line = raw_input(prompt)
    while line.endswith('\\'):
        line = line[:-1] + raw_input(ps2)
    return line

def ask_yes_no(prompt,default=None):
    """Asks a question and returns a boolean (y/n) answer.

    If default is given (one of 'y','n'), it is used if the user input is
    empty. Otherwise the question is repeated until an answer is given.

    An EOF is treated as the default answer.  If there is no default, an
    exception is raised to prevent infinite loops.

    Valid answers are: y/yes/n/no (match is not case sensitive)."""

    answers = {'y':True,'n':False,'yes':True,'no':False}
    ans = None
    while ans not in answers.keys():
        try:
            ans = raw_input(prompt+' ').lower()
            if not ans:  # response was an empty string
                ans = default
        except KeyboardInterrupt:
            pass
        except EOFError:
            if default in answers.keys():
                ans = default
                print()
            else:
                raise

    return answers[ans]

class NLprinter:
    """Print an arbitrarily nested list, indicating index numbers.

    An instance of this class called nlprint is available and callable as a
    function.

    nlprint(list,indent=' ',sep=': ') -> prints indenting each level by 'indent'
    and using 'sep' to separate the index from the value. """

    def __init__(self):
        self.depth = 0

    def __call__(self,lst,pos='',**kw):
        """Prints the nested list numbering levels."""
        kw.setdefault('indent',' ')
        kw.setdefault('sep',': ')
        kw.setdefault('start',0)
        kw.setdefault('stop',len(lst))
        # we need to remove start and stop from kw so they don't propagate
        # into a recursive call for a nested list.
        start = kw['start']; del kw['start']
        stop = kw['stop']; del kw['stop']
        if self.depth == 0 and 'header' in kw.keys():
            print(kw['header'])

        for idx in range(start,stop):
            elem = lst[idx]
            newpos = pos + str(idx)
            if type(elem)==type([]):
                self.depth += 1
                self.__call__(elem, newpos+",", **kw)
                self.depth -= 1
            else:
                print(kw['indent']*self.depth + newpos + kw["sep"] + repr(elem))

nlprint = NLprinter()

def temp_pyfile(src, ext='.py'):
    """Make a temporary python file, return filename and filehandle.

    Parameters
    ----------
    src : string or list of strings (no need for ending newlines if list)
      Source code to be written to the file.

    ext : optional, string
      Extension for the generated file.

    Returns
    -------
    (filename, open filehandle)
      It is the caller's responsibility to close the open file and unlink it.
    """
    fname = tempfile.mkstemp(ext)[1]
    f = open(fname,'w')
    f.write(src)
    f.flush()
    return fname, f

def raw_print(*args, **kw):
    """Raw print to sys.__stdout__, otherwise identical interface to print()."""

    print(*args, sep=kw.get('sep', ' '), end=kw.get('end', '\n'),
          file=sys.__stdout__)
    sys.__stdout__.flush()

def raw_print_err(*args, **kw):
    """Raw print to sys.__stderr__, otherwise identical interface to print()."""

    print(*args, sep=kw.get('sep', ' '), end=kw.get('end', '\n'),
          file=sys.__stderr__)
    sys.__stderr__.flush()

# Short aliases for quick debugging, do NOT use these in production code.
rprint = raw_print
rprinte = raw_print_err

class CapturedIO(object):
    """Simple object for containing captured stdout/err StringIO objects"""

    def __init__(self, stdout, stderr):
        self._stdout = stdout
        self._stderr = stderr

    def __str__(self):
        return self.stdout

    @property
    def stdout(self):
        if not self._stdout:
            return ''
        return self._stdout.getvalue()

    @property
    def stderr(self):
        if not self._stderr:
            return ''
        return self._stderr.getvalue()

    def show(self):
        """write my output to sys.stdout/err as appropriate"""
        sys.stdout.write(self.stdout)
        sys.stderr.write(self.stderr)
        sys.stdout.flush()
        sys.stderr.flush()

    __call__ = show

class capture_output(object):
    """context manager for capturing stdout/err"""
    stdout = True
    stderr = True

    def __init__(self, stdout=True, stderr=True):
        self.stdout = stdout
        self.stderr = stderr

    def __enter__(self):
        self.sys_stdout = sys.stdout
        self.sys_stderr = sys.stderr

        stdout = stderr = False
        if self.stdout:
            stdout = sys.stdout = StringIO()
        if self.stderr:
            stderr = sys.stderr = StringIO()

        return CapturedIO(stdout, stderr)

    def __exit__(self, exc_type, exc_value, traceback):
        sys.stdout = self.sys_stdout
        sys.stderr = self.sys_stderr


#!/usr/bin/env python
"""
A multi-heart Heartbeat system using PUB and ROUTER sockets. pings are sent out on the PUB,
and hearts are tracked based on their DEALER identities.

Authors:

* Min RK
"""
#-----------------------------------------------------------------------------
#  Copyright (C) 2010-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

from __future__ import print_function
import time
import uuid

import zmq
from zmq.devices import ThreadDevice
from zmq.eventloop import ioloop, zmqstream

from IPython.config.configurable import LoggingConfigurable
from IPython.utils.py3compat import str_to_bytes
from IPython.utils.traitlets import Set, Instance, CFloat, Integer

from IPython.parallel.util import log_errors

class Heart(object):
    """A basic heart object for responding to a HeartMonitor.
    This is a simple wrapper with defaults for the most common
    Device model for responding to heartbeats.

    It simply builds a threadsafe zmq.FORWARDER Device, defaulting to using
    SUB/DEALER for in/out.

    You can specify the DEALER's IDENTITY via the optional heart_id argument."""
    device=None
    id=None
    def __init__(self, in_addr, out_addr, in_type=zmq.SUB, out_type=zmq.DEALER, heart_id=None):
        self.device = ThreadDevice(zmq.FORWARDER, in_type, out_type)
        # do not allow the device to share global Context.instance,
        # which is the default behavior in pyzmq > 2.1.10
        self.device.context_factory = zmq.Context

        self.device.daemon=True
        self.device.connect_in(in_addr)
        self.device.connect_out(out_addr)
        if in_type == zmq.SUB:
            self.device.setsockopt_in(zmq.SUBSCRIBE, b"")
        if heart_id is None:
            heart_id = uuid.uuid4().bytes
        self.device.setsockopt_out(zmq.IDENTITY, heart_id)
        self.id = heart_id

    def start(self):
        return self.device.start()

class HeartMonitor(LoggingConfigurable):
    """A basic HeartMonitor class
    pingstream: a PUB stream
    pongstream: an ROUTER stream
    period: the period of the heartbeat in milliseconds"""

    period = Integer(3000, config=True,
        help='The frequency at which the Hub pings the engines for heartbeats '
        '(in ms)',
    )

    pingstream=Instance('zmq.eventloop.zmqstream.ZMQStream')
    pongstream=Instance('zmq.eventloop.zmqstream.ZMQStream')
    loop = Instance('zmq.eventloop.ioloop.IOLoop')
    def _loop_default(self):
        return ioloop.IOLoop.instance()

    # not settable:
    hearts=Set()
    responses=Set()
    on_probation=Set()
    last_ping=CFloat(0)
    _new_handlers = Set()
    _failure_handlers = Set()
    lifetime = CFloat(0)
    tic = CFloat(0)

    def __init__(self, **kwargs):
        super(HeartMonitor, self).__init__(**kwargs)

        self.pongstream.on_recv(self.handle_pong)

    def start(self):
        self.tic = time.time()
        self.caller = ioloop.PeriodicCallback(self.beat, self.period, self.loop)
        self.caller.start()

    def add_new_heart_handler(self, handler):
        """add a new handler for new hearts"""
        self.log.debug("heartbeat::new_heart_handler: %s", handler)
        self._new_handlers.add(handler)

    def add_heart_failure_handler(self, handler):
        """add a new handler for heart failure"""
        self.log.debug("heartbeat::new heart failure handler: %s", handler)
        self._failure_handlers.add(handler)

    def beat(self):
        self.pongstream.flush()
        self.last_ping = self.lifetime

        toc = time.time()
        self.lifetime += toc-self.tic
        self.tic = toc
        self.log.debug("heartbeat::sending %s", self.lifetime)
        goodhearts = self.hearts.intersection(self.responses)
        missed_beats = self.hearts.difference(goodhearts)
        heartfailures = self.on_probation.intersection(missed_beats)
        newhearts = self.responses.difference(goodhearts)
        map(self.handle_new_heart, newhearts)
        map(self.handle_heart_failure, heartfailures)
        self.on_probation = missed_beats.intersection(self.hearts)
        self.responses = set()
        # print self.on_probation, self.hearts
        # self.log.debug("heartbeat::beat %.3f, %i beating hearts", self.lifetime, len(self.hearts))
        self.pingstream.send(str_to_bytes(str(self.lifetime)))
        # flush stream to force immediate socket send
        self.pingstream.flush()

    def handle_new_heart(self, heart):
        if self._new_handlers:
            for handler in self._new_handlers:
                handler(heart)
        else:
            self.log.info("heartbeat::yay, got new heart %s!", heart)
        self.hearts.add(heart)

    def handle_heart_failure(self, heart):
        if self._failure_handlers:
            for handler in self._failure_handlers:
                try:
                    handler(heart)
                except Exception as e:
                    self.log.error("heartbeat::Bad Handler! %s", handler, exc_info=True)
                    pass
        else:
            self.log.info("heartbeat::Heart %s failed :(", heart)
        self.hearts.remove(heart)

    @log_errors
    def handle_pong(self, msg):
        "a heart just beat"
        current = str_to_bytes(str(self.lifetime))
        last = str_to_bytes(str(self.last_ping))
        if msg[1] == current:
            delta = time.time()-self.tic
            # self.log.debug("heartbeat::heart %r took %.2f ms to respond"%(msg[0], 1000*delta))
            self.responses.add(msg[0])
        elif msg[1] == last:
            delta = time.time()-self.tic + (self.lifetime-self.last_ping)
            self.log.warn("heartbeat::heart %r missed a beat, and took %.2f ms to respond", msg[0], 1000*delta)
            self.responses.add(msg[0])
        else:
            self.log.warn("heartbeat::got bad heartbeat (possibly old?): %s (current=%.3f)", msg[1], self.lifetime)

if __name__ == '__main__':
    loop = ioloop.IOLoop.instance()
    context = zmq.Context()
    pub = context.socket(zmq.PUB)
    pub.bind('tcp://127.0.0.1:5555')
    router = context.socket(zmq.ROUTER)
    router.bind('tcp://127.0.0.1:5556')

    outstream = zmqstream.ZMQStream(pub, loop)
    instream = zmqstream.ZMQStream(router, loop)

    hb = HeartMonitor(loop, outstream, instream)

    loop.start()


'''The utils module provides a collection of methods used across the package or of general utility.

'''

import os
import re
import sys
import shutil
import errno
import fnmatch
import traceback
import numpy as np
import pandas as pd
try:
    import cPickle as pickle
except:
    import pickle
import random
import time

def catch(fcn, *args, **kwargs):
    '''try:
          retrun fcn(*args, **kwargs)
       except:
          print traceback
            if 'spit' in kwargs.keys():
                return kwargs['spit']

    Parameters
    ----------
    fcn : function
    *args : unnamed parameters of fcn
    **kwargs : named parameters of fcn
        spit : returns the parameter named return in the exception

    Returns
    -------
    The expected output of fcn or prints the exception traceback

    '''

    try:
        # remove the special kwargs key "spit" and use it to return if it exists
        spit = kwargs.pop('spit')
    except:
        spit = None

    try:
        results = fcn(*args, **kwargs)
        if results:
            return results
    except:
        print traceback.format_exc()
        if spit:
            return spit

def batch_list(sequence, batch_size, mod = 0, randomize = False):
    '''
    Converts a list into a list of lists with equal batch_size.

    Parameters
    ----------
    sequence : list
        list of items to be placed in batches
    batch_size : int
        length of each sub list
    mod : int
        remainder of list length devided by batch_size
        mod = len(sequence) % batch_size
    randomize = bool
        should the initial sequence be randomized before being batched

    '''

    if randomize:
        sequence = random.sample(sequence, len(sequence))

    return [sequence[x:x + batch_size] for x in xrange(0, len(sequence)-mod, batch_size)]

def to_pickle(obj, filename, clean_memory=False):
    '''http://stackoverflow.com/questions/7900944/read-write-classes-to-files-in-an-efficent-way'''

    path, filename = path_to_filename(filename)
    create_dir(path)

    with open(path + filename, "wb") as output:
        pickle.dump(obj, output, pickle.HIGHEST_PROTOCOL)

    if clean_memory:
        obj = None

    # setting the global object to None requires a return assignment
    return obj

def from_pickle(filename, clean_disk=False):
    # to deserialize the object
    with open(filename, "rb") as input:
        obj = pickle.load(input)  # protocol version is auto detected

    if clean_disk:
        os.remove(filename)
    return obj

def path_to_filename(pathfile):
    '''
    Takes a path filename string and returns the split between the path and the filename

    if filename is not given, filename = ''
    if path is not given, path = './'

    '''

    path = pathfile[:pathfile.rfind('/') + 1]
    if path == '':
        path = './'

    filename = pathfile[pathfile.rfind('/') + 1:len(pathfile)]
    if '.' not in filename:
        path = pathfile
        filename = ''

    if (filename == '') and (path[len(path) - 1] != '/'):
        path += '/'

    return path, filename

def add_path_string(root_path='./results', path_string=None):
    rootPath = path_to_filename(rootPath)[0]
    regEx = '[.<>"!,:;*/ -]'
    if pathString is not None:
        return path_to_filename(root_path + re.sub(regEx, '_', path_string))[0]
    else:
        return root_path

def create_dir(path, dir_dict={}):
    '''
    Tries to create a new directory in the given path.
    **create_dir** can also create subfolders according to the dictionnary given as second argument.

    Parameters
    ----------
    path : string
        string giving the path of the location to create the directory, either absolute or relative.
    dir_dict : dictionary, optional
        Dictionary ordering the creation of subfolders. Keys must be strings, and values either None or path dictionaries.
        the default is {}, which means that no subfolders will be created

    Examples
    --------

    >>> path = './project'
    >>> dir_dict = {'dir1':None, 'dir2':{'subdir21':None}}
    >>> utils.create_dir(path, dir_dict)

    will create:

    * *project/dir1*
    * *project/dir2/subdir21*

    in your parent directory.

    '''

    folders = path.split('/')
    folders = [i for i in folders if i != '']
    rootPath = ''
    if folders[0] == 'C:':
        folders = folders[1:]
    count = 0
    for directory in folders:
        count += 1

        # required to handle the dot operators
        if (directory[0] == '.') & (count == 1):
            rootPath = directory
        else:
            rootPath = rootPath + '/' + directory
        try:
            os.makedirs(rootPath)
        # If the file already exists (EEXIST), raise exception and do nothing
        except OSError as exception:
            if exception.errno != errno.EEXIST:
                raise

    for key in dir_dict.keys():
        rootPath = path + "/" + key
        try:
            os.makedirs(rootPath)
        # If the file already exists (EEXIST), raise exception and do nothing
        except OSError as exception:
            if exception.errno != errno.EEXIST:
                raise
        if dir_dict[key] is not None:
            create_dir(rootPath, dir_dict[key])

def Walk(root='.', recurse=True, pattern='*'):
    '''
    Generator for walking a directory tree.
    Starts at specified root folder, returning files that match our pattern.
    Optionally will also recurse through sub-folders.

    Parameters
    ----------
    root : string (default is *'.'*)
        Path for the root folder to look in.
    recurse : bool (default is *True*)
        If *True*, will also look in the subfolders.
    pattern : string (default is :emphasis:`'*'`, which means all the files are concerned)
        The pattern to look for in the files' name.

    Returns
    -------
    generator
        **Walk** yields a generator from the matching files paths.

    '''

    for path, subdirs, files in os.walk(root):
        for name in files:
            if fnmatch.fnmatch(name, pattern):
                yield os.path.join(path, name)
        if not recurse:
            break

def scan_path(root='.', recurse=False, pattern='*'):
    '''
    Runs a loop over the :doc:`Walk<relpy.utils.Walk>` Generator
    to find all file paths in the root directory with the given
    pattern. If recurse is *True*: matching paths are identified
    for all sub directories.

    Parameters
    ----------
    root : string (default is *'.'*)
        Path for the root folder to look in.
    recurse : bool (default is *True*)
        If *True*, will also look in the subfolders.
    pattern : string (default is :emphasis:`'*'`, which means all the files are concerned)
        The pattern to look for in the files' name.

    Returns
    -------
    path_list : list
        list of all the matching files paths.

    '''

    path_list = []
    for path in Walk(root=root, recurse=recurse, pattern=pattern):
        path_list.append(path)
    return path_list

class Timer:

    '''Timer that calculates time remaining for a process and the percent complete

    .. todo::

        Ask for details about the usage

    Parameters
    ----------
    nLoops : integer
    numPrints : integer (default is *100*)
    verbose : bool (default is *True*)

    Attributes
    ----------
    nLoops : integer
    numPrints : integer
    verbose : bool
        if *True*, print values when **loop** is called
    count : integer
    elapsed : float
        elapsed time
    est_end : float
        estimated end
    ti : float
        initial time
    tf : float
        current time
    display_amt : integer

    '''

    def __init__(self, nLoops, numPrints=100, verbose=True):

        self.nLoops = nLoops
        self.numPrints = numPrints
        self.verbose = verbose
        self.count = 0
        self.elapsed = 1
        self.est_end = 1
        self.ti = time.time()
        self.display_amt = 1

    def loop(self):
        '''
        Tracks the time in a loop. The estimated time to completion
        can be calculated and if verbose is set to *True*, the object will print
        estimated time to completion, and percent complete.
        Actived in every loop to keep track'''

        self.count += 1
        self.tf = time.time()
        self.elapsed = self.tf - self.ti

        if self.verbose:
            displayAll(self.elapsed, self.display_amt, self.est_end,
                       self.nLoops, self.count, self.numPrints)

    def fin(self):
        print "Elapsed time: %s  :-)" % str(time.time() - self.ti)

def displayAll(elapsed, display_amt, est_end, nLoops, count, numPrints):
    '''Displays time if verbose is true and count is within the display amount'''

    if numPrints > nLoops:
        display_amt = 1
    else:
        display_amt = round(nLoops / numPrints)

    if count % display_amt == 0:

        avg = elapsed / count
        est_end = round(avg * nLoops)

        (disp_elapsed,
         disp_avg,
         disp_est) = timeUnit(int(round(elapsed)),
                              int(round(avg)),
                              int(round(est_end)))

        print "%s%%" % str(round(count / float(nLoops) * 100)), "@" + str(count),

        totalTime = disp_est[0]
        unit = disp_est[1]

        if str(unit) == "secs":
            remain = totalTime - round(elapsed)
            remainUnit = "secs"
        elif str(unit) == "mins":
            remain = totalTime - round(elapsed) / 60
            remainUnit = "mins"
        elif str(unit) == "hr":
            remain = totalTime - round(elapsed) / 3600
            remainUnit = "hr"

        print "ETA: %s %s" % (str(remain), remainUnit)
        print

    return

def timeUnit(elapsed, avg, est_end):
    '''calculates unit of time to display'''
    minute = 60
    hr = 3600
    day = 86400

    if elapsed <= 3 * minute:
        unit_elapsed = (elapsed, "secs")
    if elapsed > 3 * minute:
        unit_elapsed = ((elapsed / 60), "mins")
    if elapsed > 3 * hr:
        unit_elapsed = ((elapsed / 3600), "hr")

    if avg <= 3 * minute:
        unit_avg = (avg, "secs")
    if avg > 3 * minute:
        unit_avg = ((avg / 60), "mins")
    if avg > 3 * hr:
        unit_avg = ((avg / 3600), "hr")

    if est_end <= 3 * minute:
        unit_estEnd = (est_end, "secs")
    if est_end > 3 * minute:
        unit_estEnd = ((est_end / 60), "mins")
    if est_end > 3 * hr:
        unit_estEnd = ((est_end / 3600), "hr")

    return [unit_elapsed, unit_avg, unit_estEnd]


"""Implements an importer that looks only in specific path (ignoring
sys.path), and uses a per-path cache in addition to sys.modules. This is
necessary because test modules in different directories frequently have the
same names, which means that the first loaded would mask the rest when using
the builtin importer.
"""
import logging
import os
import sys
from nose.config import Config

from imp import find_module, load_module, acquire_lock, release_lock

log = logging.getLogger(__name__)

class Importer(object):
    """An importer class that does only path-specific imports. That
    is, the given module is not searched for on sys.path, but only at
    the path or in the directory specified.
    """
    def __init__(self, config=None):
        if config is None:
            config = Config()
        self.config = config

    def importFromPath(self, path, fqname):
        """Import a dotted-name package whose tail is at path. In other words,
        given foo.bar and path/to/foo/bar.py, import foo from path/to/foo then
        bar from path/to/foo/bar, returning bar.
        """
        # find the base dir of the package
        path_parts = os.path.normpath(os.path.abspath(path)).split(os.sep)
        name_parts = fqname.split('.')
        if path_parts[-1].startswith('__init__'):
            path_parts.pop()
        path_parts = path_parts[:-(len(name_parts))]
        dir_path = os.sep.join(path_parts)
        # then import fqname starting from that dir
        return self.importFromDir(dir_path, fqname)

    def importFromDir(self, dir, fqname):
        """Import a module *only* from path, ignoring sys.path and
        reloading if the version in sys.modules is not the one we want.
        """
        dir = os.path.normpath(os.path.abspath(dir))
        log.debug("Import %s from %s", fqname, dir)

        # FIXME reimplement local per-dir cache?

        # special case for __main__
        if fqname == '__main__':
            return sys.modules[fqname]

        if self.config.addPaths:
            add_path(dir, self.config)

        path = [dir]
        parts = fqname.split('.')
        part_fqname = ''
        mod = parent = fh = None

        for part in parts:
            if part_fqname == '':
                part_fqname = part
            else:
                part_fqname = "%s.%s" % (part_fqname, part)
            try:
                acquire_lock()
                log.debug("find module part %s (%s) in %s",
                          part, part_fqname, path)
                fh, filename, desc = find_module(part, path)
                old = sys.modules.get(part_fqname)
                if old is not None:
                    # test modules frequently have name overlap; make sure
                    # we get a fresh copy of anything we are trying to load
                    # from a new path
                    log.debug("sys.modules has %s as %s", part_fqname, old)
                    if (self.sameModule(old, filename)
                        or (self.config.firstPackageWins and
                            getattr(old, '__path__', None))):
                        mod = old
                    else:
                        del sys.modules[part_fqname]
                        mod = load_module(part_fqname, fh, filename, desc)
                else:
                    mod = load_module(part_fqname, fh, filename, desc)
            finally:
                if fh:
                    fh.close()
                release_lock()
            if parent:
                setattr(parent, part, mod)
            if hasattr(mod, '__path__'):
                path = mod.__path__
            parent = mod
        return mod

    def sameModule(self, mod, filename):
        mod_paths = []
        if hasattr(mod, '__path__'):
            for path in mod.__path__:
                mod_paths.append(os.path.dirname(
                    os.path.normpath(
                    os.path.abspath(path))))
        elif hasattr(mod, '__file__'):
            mod_paths.append(os.path.dirname(
                os.path.normpath(
                os.path.abspath(mod.__file__))))
        else:
            # builtin or other module-like object that
            # doesn't have __file__; must be new
            return False
        new_path = os.path.dirname(os.path.normpath(filename))
        for mod_path in mod_paths:
            log.debug(
                "module already loaded? mod: %s new: %s",
                mod_path, new_path)
            if mod_path == new_path:
                return True
        return False

def add_path(path, config=None):
    """Ensure that the path, or the root of the current package (if
    path is in a package), is in sys.path.
    """

    # FIXME add any src-looking dirs seen too... need to get config for that

    log.debug('Add path %s' % path)
    if not path:
        return []
    added = []
    parent = os.path.dirname(path)
    if (parent
        and os.path.exists(os.path.join(path, '__init__.py'))):
        added.extend(add_path(parent, config))
    elif not path in sys.path:
        log.debug("insert %s into sys.path", path)
        sys.path.insert(0, path)
        added.append(path)
    if config and config.srcDirs:
        for dirname in config.srcDirs:
            dirpath = os.path.join(path, dirname)
            if os.path.isdir(dirpath):
                sys.path.insert(0, dirpath)
                added.append(dirpath)
    return added

def remove_path(path):
    log.debug('Remove path %s' % path)
    if path in sys.path:
        sys.path.remove(path)