"""Raw data collector for Coverage."""

import os, sys, threading

try:
    # Use the C extension code when we can, for speed.
    from coverage.tracer import CTracer         # pylint: disable=F0401,E0611
except ImportError:
    # Couldn't import the C extension, maybe it isn't built.
    if os.getenv('COVERAGE_TEST_TRACER') == 'c':
        # During testing, we use the COVERAGE_TEST_TRACER env var to indicate
        # that we've fiddled with the environment to test this fallback code.
        # If we thought we had a C tracer, but couldn't import it, then exit
        # quickly and clearly instead of dribbling confusing errors. I'm using
        # sys.exit here instead of an exception because an exception here
        # causes all sorts of other noise in unittest.
        sys.stderr.write(
            "*** COVERAGE_TEST_TRACER is 'c' but can't import CTracer!\n"
            )
        sys.exit(1)
    CTracer = None

class PyTracer(object):
    """Python implementation of the raw data tracer."""

    # Because of poor implementations of trace-function-manipulating tools,
    # the Python trace function must be kept very simple.  In particular, there
    # must be only one function ever set as the trace function, both through
    # sys.settrace, and as the return value from the trace function.  Put
    # another way, the trace function must always return itself.  It cannot
    # swap in other functions, or return None to avoid tracing a particular
    # frame.
    #
    # The trace manipulator that introduced this restriction is DecoratorTools,
    # which sets a trace function, and then later restores the pre-existing one
    # by calling sys.settrace with a function it found in the current frame.
    #
    # Systems that use DecoratorTools (or similar trace manipulations) must use
    # PyTracer to get accurate results.  The command-line --timid argument is
    # used to force the use of this tracer.

    def __init__(self):
        self.data = None
        self.should_trace = None
        self.should_trace_cache = None
        self.warn = None
        self.cur_file_data = None
        self.last_line = 0
        self.data_stack = []
        self.last_exc_back = None
        self.last_exc_firstlineno = 0
        self.arcs = False
        self.thread = None
        self.stopped = False

    def _trace(self, frame, event, arg_unused):
        """The trace function passed to sys.settrace."""

        if self.stopped:
            return

        if 0:
            sys.stderr.write("trace event: %s %r @%d\n" % (
                event, frame.f_code.co_filename, frame.f_lineno
            ))

        if self.last_exc_back:
            if frame == self.last_exc_back:
                # Someone forgot a return event.
                if self.arcs and self.cur_file_data:
                    pair = (self.last_line, -self.last_exc_firstlineno)
                    self.cur_file_data[pair] = None
                self.cur_file_data, self.last_line = self.data_stack.pop()
            self.last_exc_back = None

        if event == 'call':
            # Entering a new function context.  Decide if we should trace
            # in this file.
            self.data_stack.append((self.cur_file_data, self.last_line))
            filename = frame.f_code.co_filename
            if filename not in self.should_trace_cache:
                tracename = self.should_trace(filename, frame)
                self.should_trace_cache[filename] = tracename
            else:
                tracename = self.should_trace_cache[filename]
            #print("called, stack is %d deep, tracename is %r" % (
            #               len(self.data_stack), tracename))
            if tracename:
                if tracename not in self.data:
                    self.data[tracename] = {}
                self.cur_file_data = self.data[tracename]
            else:
                self.cur_file_data = None
            # Set the last_line to -1 because the next arc will be entering a
            # code block, indicated by (-1, n).
            self.last_line = -1
        elif event == 'line':
            # Record an executed line.
            if self.cur_file_data is not None:
                if self.arcs:
                    #print("lin", self.last_line, frame.f_lineno)
                    self.cur_file_data[(self.last_line, frame.f_lineno)] = None
                else:
                    #print("lin", frame.f_lineno)
                    self.cur_file_data[frame.f_lineno] = None
            self.last_line = frame.f_lineno
        elif event == 'return':
            if self.arcs and self.cur_file_data:
                first = frame.f_code.co_firstlineno
                self.cur_file_data[(self.last_line, -first)] = None
            # Leaving this function, pop the filename stack.
            self.cur_file_data, self.last_line = self.data_stack.pop()
            #print("returned, stack is %d deep" % (len(self.data_stack)))
        elif event == 'exception':
            #print("exc", self.last_line, frame.f_lineno)
            self.last_exc_back = frame.f_back
            self.last_exc_firstlineno = frame.f_code.co_firstlineno
        return self._trace

    def start(self):
        """Start this Tracer.

        Return a Python function suitable for use with sys.settrace().

        """
        self.thread = threading.currentThread()
        sys.settrace(self._trace)
        return self._trace

    def stop(self):
        """Stop this Tracer."""
        self.stopped = True
        if self.thread != threading.currentThread():
            # Called on a different thread than started us: we can't unhook
            # ourseves, but we've set the flag that we should stop, so we won't
            # do any more tracing.
            return

        if hasattr(sys, "gettrace") and self.warn:
            if sys.gettrace() != self._trace:
                msg = "Trace function changed, measurement is likely wrong: %r"
                self.warn(msg % (sys.gettrace(),))
        #print("Stopping tracer on %s" % threading.current_thread().ident)
        sys.settrace(None)

    def get_stats(self):
        """Return a dictionary of statistics, or None."""
        return None

class Collector(object):
    """Collects trace data.

    Creates a Tracer object for each thread, since they track stack
    information.  Each Tracer points to the same shared data, contributing
    traced data points.

    When the Collector is started, it creates a Tracer for the current thread,
    and installs a function to create Tracers for each new thread started.
    When the Collector is stopped, all active Tracers are stopped.

    Threads started while the Collector is stopped will never have Tracers
    associated with them.

    """

    # The stack of active Collectors.  Collectors are added here when started,
    # and popped when stopped.  Collectors on the stack are paused when not
    # the top, and resumed when they become the top again.
    _collectors = []

    def __init__(self, should_trace, timid, branch, warn):
        """Create a collector.

        `should_trace` is a function, taking a filename, and returning a
        canonicalized filename, or None depending on whether the file should
        be traced or not.

        If `timid` is true, then a slower simpler trace function will be
        used.  This is important for some environments where manipulation of
        tracing functions make the faster more sophisticated trace function not
        operate properly.

        If `branch` is true, then branches will be measured.  This involves
        collecting data on which statements followed each other (arcs).  Use
        `get_arc_data` to get the arc data.

        `warn` is a warning function, taking a single string message argument,
        to be used if a warning needs to be issued.

        """
        self.should_trace = should_trace
        self.warn = warn
        self.branch = branch
        self.reset()

        if timid:
            # Being timid: use the simple Python trace function.
            self._trace_class = PyTracer
        else:
            # Being fast: use the C Tracer if it is available, else the Python
            # trace function.
            self._trace_class = CTracer or PyTracer

    def __repr__(self):
        return "<Collector at 0x%x>" % id(self)

    def tracer_name(self):
        """Return the class name of the tracer we're using."""
        return self._trace_class.__name__

    def reset(self):
        """Clear collected data, and prepare to collect more."""
        # A dictionary mapping filenames to dicts with linenumber keys,
        # or mapping filenames to dicts with linenumber pairs as keys.
        self.data = {}

        # A cache of the results from should_trace, the decision about whether
        # to trace execution in a file. A dict of filename to (filename or
        # None).
        self.should_trace_cache = {}

        # Our active Tracers.
        self.tracers = []

    def _start_tracer(self):
        """Start a new Tracer object, and store it in self.tracers."""
        tracer = self._trace_class()
        tracer.data = self.data
        tracer.arcs = self.branch
        tracer.should_trace = self.should_trace
        tracer.should_trace_cache = self.should_trace_cache
        tracer.warn = self.warn
        fn = tracer.start()
        self.tracers.append(tracer)
        return fn

    # The trace function has to be set individually on each thread before
    # execution begins.  Ironically, the only support the threading module has
    # for running code before the thread main is the tracing function.  So we
    # install this as a trace function, and the first time it's called, it does
    # the real trace installation.

    def _installation_trace(self, frame_unused, event_unused, arg_unused):
        """Called on new threads, installs the real tracer."""
        # Remove ourselves as the trace function
        sys.settrace(None)
        # Install the real tracer.
        fn = self._start_tracer()
        # Invoke the real trace function with the current event, to be sure
        # not to lose an event.
        if fn:
            fn = fn(frame_unused, event_unused, arg_unused)
        # Return the new trace function to continue tracing in this scope.
        return fn

    def start(self):
        """Start collecting trace information."""
        if self._collectors:
            self._collectors[-1].pause()
        self._collectors.append(self)
        #print("Started: %r" % self._collectors, file=sys.stderr)

        # Check to see whether we had a fullcoverage tracer installed.
        traces0 = []
        if hasattr(sys, "gettrace"):
            fn0 = sys.gettrace()
            if fn0:
                tracer0 = getattr(fn0, '__self__', None)
                if tracer0:
                    traces0 = getattr(tracer0, 'traces', [])

        # Install the tracer on this thread.
        fn = self._start_tracer()

        for args in traces0:
            (frame, event, arg), lineno = args
            try:
                fn(frame, event, arg, lineno=lineno)
            except TypeError:
                raise Exception(
                    "fullcoverage must be run with the C trace function."
                )

        # Install our installation tracer in threading, to jump start other
        # threads.
        threading.settrace(self._installation_trace)

    def stop(self):
        """Stop collecting trace information."""
        #print >>sys.stderr, "Stopping: %r" % self._collectors
        assert self._collectors
        assert self._collectors[-1] is self

        self.pause()
        self.tracers = []

        # Remove this Collector from the stack, and resume the one underneath
        # (if any).
        self._collectors.pop()
        if self._collectors:
            self._collectors[-1].resume()

    def pause(self):
        """Pause tracing, but be prepared to `resume`."""
        for tracer in self.tracers:
            tracer.stop()
            stats = tracer.get_stats()
            if stats:
                print("\nCoverage.py tracer stats:")
                for k in sorted(stats.keys()):
                    print("%16s: %s" % (k, stats[k]))
        threading.settrace(None)

    def resume(self):
        """Resume tracing after a `pause`."""
        for tracer in self.tracers:
            tracer.start()
        threading.settrace(self._installation_trace)

    def get_line_data(self):
        """Return the line data collected.

        Data is { filename: { lineno: None, ...}, ...}

        """
        if self.branch:
            # If we were measuring branches, then we have to re-build the dict
            # to show line data.
            line_data = {}
            for f, arcs in self.data.items():
                line_data[f] = ldf = {}
                for l1, _ in list(arcs.keys()):
                    if l1:
                        ldf[l1] = None
            return line_data
        else:
            return self.data

    def get_arc_data(self):
        """Return the arc data collected.

        Data is { filename: { (l1, l2): None, ...}, ...}

        Note that no data is collected or returned if the Collector wasn't
        created with `branch` true.

        """
        if self.branch:
            return self.data
        else:
            return {}


"""The basic dict based notebook format.

Authors:

* Brian Granger
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import pprint
import uuid

from IPython.utils.ipstruct import Struct

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

class NotebookNode(Struct):
    pass

def from_dict(d):
    if isinstance(d, dict):
        newd = NotebookNode()
        for k,v in d.items():
            newd[k] = from_dict(v)
        return newd
    elif isinstance(d, (tuple, list)):
        return [from_dict(i) for i in d]
    else:
        return d

def new_code_cell(code=None, prompt_number=None):
    """Create a new code cell with input and output"""
    cell = NotebookNode()
    cell.cell_type = u'code'
    if code is not None:
        cell.code = unicode(code)
    if prompt_number is not None:
        cell.prompt_number = int(prompt_number)
    return cell

def new_text_cell(text=None):
    """Create a new text cell."""
    cell = NotebookNode()
    if text is not None:
        cell.text = unicode(text)
    cell.cell_type = u'text'
    return cell

def new_notebook(cells=None):
    """Create a notebook by name, id and a list of worksheets."""
    nb = NotebookNode()
    if cells is not None:
        nb.cells = cells
    else:
        nb.cells = []
    return nb


"""Tools so trivial that tracebacks should not descend into them

We define the ``__unittest`` symbol in their module namespace so unittest will
skip them when printing tracebacks, just as it does for their corresponding
methods in ``unittest`` proper.

"""
import re
import unittest

__all__ = ['ok_', 'eq_']

# Use the same flag as unittest itself to prevent descent into these functions:
__unittest = 1

def ok_(expr, msg=None):
    """Shorthand for assert. Saves 3 whole characters!
    """
    if not expr:
        raise AssertionError(msg)

def eq_(a, b, msg=None):
    """Shorthand for 'assert a == b, "%r != %r" % (a, b)
    """
    if not a == b:
        raise AssertionError(msg or "%r != %r" % (a, b))

#
# Expose assert* from unittest.TestCase
# - give them pep8 style names
#
caps = re.compile('([A-Z])')

def pep8(name):
    return caps.sub(lambda m: '_' + m.groups()[0].lower(), name)

class Dummy(unittest.TestCase):
    def nop():
        pass
_t = Dummy('nop')

for at in [ at for at in dir(_t)
            if at.startswith('assert') and not '_' in at ]:
    pepd = pep8(at)
    vars()[pepd] = getattr(_t, at)
    __all__.append(pepd)

del Dummy
del _t
del pep8


# encoding: utf-8

"""Classes and functions for kernel related errors and exceptions.

Authors:

* Brian Granger
* Min RK
"""
from __future__ import print_function

import sys
import traceback

__docformat__ = "restructuredtext en"

# Tell nose to skip this module
__test__ = {}

#-------------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
# Error classes
#-------------------------------------------------------------------------------
class IPythonError(Exception):
    """Base exception that all of our exceptions inherit from.

    This can be raised by code that doesn't have any more specific
    information."""

    pass

# Exceptions associated with the controller objects
class ControllerError(IPythonError): pass

class ControllerCreationError(ControllerError): pass

# Exceptions associated with the Engines
class EngineError(IPythonError): pass

class EngineCreationError(EngineError): pass

class KernelError(IPythonError):
    pass

class NotDefined(KernelError):
    def __init__(self, name):
        self.name = name
        self.args = (name,)

    def __repr__(self):
        return '<NotDefined: %s>' % self.name

    __str__ = __repr__

class QueueCleared(KernelError):
    pass

class IdInUse(KernelError):
    pass

class ProtocolError(KernelError):
    pass

class ConnectionError(KernelError):
    pass

class InvalidEngineID(KernelError):
    pass

class NoEnginesRegistered(KernelError):
    pass

class InvalidClientID(KernelError):
    pass

class InvalidDeferredID(KernelError):
    pass

class SerializationError(KernelError):
    pass

class MessageSizeError(KernelError):
    pass

class PBMessageSizeError(MessageSizeError):
    pass

class ResultNotCompleted(KernelError):
    pass

class ResultAlreadyRetrieved(KernelError):
    pass

class ClientError(KernelError):
    pass

class TaskAborted(KernelError):
    pass

class TaskTimeout(KernelError):
    pass

class NotAPendingResult(KernelError):
    pass

class UnpickleableException(KernelError):
    pass

class AbortedPendingDeferredError(KernelError):
    pass

class InvalidProperty(KernelError):
    pass

class MissingBlockArgument(KernelError):
    pass

class StopLocalExecution(KernelError):
    pass

class SecurityError(KernelError):
    pass

class FileTimeoutError(KernelError):
    pass

class TimeoutError(KernelError):
    pass

class UnmetDependency(KernelError):
    pass

class ImpossibleDependency(UnmetDependency):
    pass

class DependencyTimeout(ImpossibleDependency):
    pass

class InvalidDependency(ImpossibleDependency):
    pass

class RemoteError(KernelError):
    """Error raised elsewhere"""
    ename=None
    evalue=None
    traceback=None
    engine_info=None

    def __init__(self, ename, evalue, traceback, engine_info=None):
        self.ename=ename
        self.evalue=evalue
        self.traceback=traceback
        self.engine_info=engine_info or {}
        self.args=(ename, evalue)

    def __repr__(self):
        engineid = self.engine_info.get('engine_id', ' ')
        return "<Remote[%s]:%s(%s)>"%(engineid, self.ename, self.evalue)

    def __str__(self):
        return "%s(%s)" % (self.ename, self.evalue)

    def render_traceback(self):
        """render traceback to a list of lines"""
        return (self.traceback or "No traceback available").splitlines()

    def _render_traceback_(self):
        """Special method for custom tracebacks within IPython.

        This will be called by IPython instead of displaying the local traceback.

        It should return a traceback rendered as a list of lines.
        """
        return self.render_traceback()

    def print_traceback(self, excid=None):
        """print my traceback"""
        print('\n'.join(self.render_traceback()))

class TaskRejectError(KernelError):
    """Exception to raise when a task should be rejected by an engine.

    This exception can be used to allow a task running on an engine to test
    if the engine (or the user's namespace on the engine) has the needed
    task dependencies.  If not, the task should raise this exception.  For
    the task to be retried on another engine, the task should be created
    with the `retries` argument > 1.

    The advantage of this approach over our older properties system is that
    tasks have full access to the user's namespace on the engines and the
    properties don't have to be managed or tested by the controller.
    """

class CompositeError(RemoteError):
    """Error for representing possibly multiple errors on engines"""
    def __init__(self, message, elist):
        Exception.__init__(self, *(message, elist))
        # Don't use pack_exception because it will conflict with the .message
        # attribute that is being deprecated in 2.6 and beyond.
        self.msg = message
        self.elist = elist
        self.args = [ e[0] for e in elist ]

    def _get_engine_str(self, ei):
        if not ei:
            return '[Engine Exception]'
        else:
            return '[%s:%s]: ' % (ei['engine_id'], ei['method'])

    def _get_traceback(self, ev):
        try:
            tb = ev._ipython_traceback_text
        except AttributeError:
            return 'No traceback available'
        else:
            return tb

    def __str__(self):
        s = str(self.msg)
        for en, ev, etb, ei in self.elist:
            engine_str = self._get_engine_str(ei)
            s = s + '\n' + engine_str + en + ': ' + str(ev)
        return s

    def __repr__(self):
        return "CompositeError(%i)"%len(self.elist)

    def render_traceback(self, excid=None):
        """render one or all of my tracebacks to a list of lines"""
        lines = []
        if excid is None:
            for (en,ev,etb,ei) in self.elist:
                lines.append(self._get_engine_str(ei))
                lines.extend((etb or 'No traceback available').splitlines())
                lines.append('')
        else:
            try:
                en,ev,etb,ei = self.elist[excid]
            except:
                raise IndexError("an exception with index %i does not exist"%excid)
            else:
                lines.append(self._get_engine_str(ei))
                lines.extend((etb or 'No traceback available').splitlines())

        return lines

    def print_traceback(self, excid=None):
        print('\n'.join(self.render_traceback(excid)))

    def raise_exception(self, excid=0):
        try:
            en,ev,etb,ei = self.elist[excid]
        except:
            raise IndexError("an exception with index %i does not exist"%excid)
        else:
            raise RemoteError(en, ev, etb, ei)

def collect_exceptions(rdict_or_list, method='unspecified'):
    """check a result dict for errors, and raise CompositeError if any exist.
    Passthrough otherwise."""
    elist = []
    if isinstance(rdict_or_list, dict):
        rlist = rdict_or_list.values()
    else:
        rlist = rdict_or_list
    for r in rlist:
        if isinstance(r, RemoteError):
            en, ev, etb, ei = r.ename, r.evalue, r.traceback, r.engine_info
            # Sometimes we could have CompositeError in our list.  Just take
            # the errors out of them and put them in our new list.  This
            # has the effect of flattening lists of CompositeErrors into one
            # CompositeError
            if en=='CompositeError':
                for e in ev.elist:
                    elist.append(e)
            else:
                elist.append((en, ev, etb, ei))
    if len(elist)==0:
        return rdict_or_list
    else:
        msg = "one or more exceptions from call to method: %s" % (method)
        # This silliness is needed so the debugger has access to the exception
        # instance (e in this case)
        try:
            raise CompositeError(msg, elist)
        except CompositeError as e:
            raise e

def wrap_exception(engine_info={}):
    etype, evalue, tb = sys.exc_info()
    stb = traceback.format_exception(etype, evalue, tb)
    exc_content = {
        'status' : 'error',
        'traceback' : stb,
        'ename' : unicode(etype.__name__),
        'evalue' : unicode(evalue),
        'engine_info' : engine_info
    }
    return exc_content

def unwrap_exception(content):
    err = RemoteError(content['ename'], content['evalue'],
                ''.join(content['traceback']),
                content.get('engine_info', {}))
    return err


"""Core control stuff for Coverage."""

import atexit, os, random, socket, sys

from coverage.annotate import AnnotateReporter
from coverage.backward import string_class, iitems, sorted  # pylint: disable=W0622
from coverage.codeunit import code_unit_factory, CodeUnit
from coverage.collector import Collector
from coverage.config import CoverageConfig
from coverage.data import CoverageData
from coverage.debug import DebugControl
from coverage.files import FileLocator, TreeMatcher, FnmatchMatcher
from coverage.files import PathAliases, find_python_files, prep_patterns
from coverage.html import HtmlReporter
from coverage.misc import CoverageException, bool_or_none, join_regex
from coverage.misc import file_be_gone
from coverage.results import Analysis, Numbers
from coverage.summary import SummaryReporter
from coverage.xmlreport import XmlReporter

# Pypy has some unusual stuff in the "stdlib".  Consider those locations
# when deciding where the stdlib is.
try:
    import _structseq       # pylint: disable=F0401
except ImportError:
    _structseq = None

class coverage(object):
    """Programmatic access to coverage.py.

    To use::

        from coverage import coverage

        cov = coverage()
        cov.start()
        #.. call your code ..
        cov.stop()
        cov.html_report(directory='covhtml')

    """
    def __init__(self, data_file=None, data_suffix=None, cover_pylib=None,
                auto_data=False, timid=None, branch=None, config_file=True,
                source=None, omit=None, include=None, debug=None,
                debug_file=None):
        """
        `data_file` is the base name of the data file to use, defaulting to
        ".coverage".  `data_suffix` is appended (with a dot) to `data_file` to
        create the final file name.  If `data_suffix` is simply True, then a
        suffix is created with the machine and process identity included.

        `cover_pylib` is a boolean determining whether Python code installed
        with the Python interpreter is measured.  This includes the Python
        standard library and any packages installed with the interpreter.

        If `auto_data` is true, then any existing data file will be read when
        coverage measurement starts, and data will be saved automatically when
        measurement stops.

        If `timid` is true, then a slower and simpler trace function will be
        used.  This is important for some environments where manipulation of
        tracing functions breaks the faster trace function.

        If `branch` is true, then branch coverage will be measured in addition
        to the usual statement coverage.

        `config_file` determines what config file to read.  If it is a string,
        it is the name of the config file to read.  If it is True, then a
        standard file is read (".coveragerc").  If it is False, then no file is
        read.

        `source` is a list of file paths or package names.  Only code located
        in the trees indicated by the file paths or package names will be
        measured.

        `include` and `omit` are lists of filename patterns. Files that match
        `include` will be measured, files that match `omit` will not.  Each
        will also accept a single string argument.

        `debug` is a list of strings indicating what debugging information is
        desired. `debug_file` is the file to write debug messages to,
        defaulting to stderr.

        """
        from coverage import __version__

        # A record of all the warnings that have been issued.
        self._warnings = []

        # Build our configuration from a number of sources:
        # 1: defaults:
        self.config = CoverageConfig()

        # 2: from the coveragerc file:
        if config_file:
            if config_file is True:
                config_file = ".coveragerc"
            try:
                self.config.from_file(config_file)
            except ValueError:
                _, err, _ = sys.exc_info()
                raise CoverageException(
                    "Couldn't read config file %s: %s" % (config_file, err)
                    )

        # 3: from environment variables:
        self.config.from_environment('COVERAGE_OPTIONS')
        env_data_file = os.environ.get('COVERAGE_FILE')
        if env_data_file:
            self.config.data_file = env_data_file

        # 4: from constructor arguments:
        self.config.from_args(
            data_file=data_file, cover_pylib=cover_pylib, timid=timid,
            branch=branch, parallel=bool_or_none(data_suffix),
            source=source, omit=omit, include=include, debug=debug,
            )

        # Create and configure the debugging controller.
        self.debug = DebugControl(self.config.debug, debug_file or sys.stderr)

        self.auto_data = auto_data

        # _exclude_re is a dict mapping exclusion list names to compiled
        # regexes.
        self._exclude_re = {}
        self._exclude_regex_stale()

        self.file_locator = FileLocator()

        # The source argument can be directories or package names.
        self.source = []
        self.source_pkgs = []
        for src in self.config.source or []:
            if os.path.exists(src):
                self.source.append(self.file_locator.canonical_filename(src))
            else:
                self.source_pkgs.append(src)

        self.omit = prep_patterns(self.config.omit)
        self.include = prep_patterns(self.config.include)

        self.collector = Collector(
            self._should_trace, timid=self.config.timid,
            branch=self.config.branch, warn=self._warn
            )

        # Suffixes are a bit tricky.  We want to use the data suffix only when
        # collecting data, not when combining data.  So we save it as
        # `self.run_suffix` now, and promote it to `self.data_suffix` if we
        # find that we are collecting data later.
        if data_suffix or self.config.parallel:
            if not isinstance(data_suffix, string_class):
                # if data_suffix=True, use .machinename.pid.random
                data_suffix = True
        else:
            data_suffix = None
        self.data_suffix = None
        self.run_suffix = data_suffix

        # Create the data file.  We do this at construction time so that the
        # data file will be written into the directory where the process
        # started rather than wherever the process eventually chdir'd to.
        self.data = CoverageData(
            basename=self.config.data_file,
            collector="coverage v%s" % __version__,
            debug=self.debug,
            )

        # The dirs for files considered "installed with the interpreter".
        self.pylib_dirs = []
        if not self.config.cover_pylib:
            # Look at where some standard modules are located. That's the
            # indication for "installed with the interpreter". In some
            # environments (virtualenv, for example), these modules may be
            # spread across a few locations. Look at all the candidate modules
            # we've imported, and take all the different ones.
            for m in (atexit, os, random, socket, _structseq):
                if m is not None and hasattr(m, "__file__"):
                    m_dir = self._canonical_dir(m)
                    if m_dir not in self.pylib_dirs:
                        self.pylib_dirs.append(m_dir)

        # To avoid tracing the coverage code itself, we skip anything located
        # where we are.
        self.cover_dir = self._canonical_dir(__file__)

        # The matchers for _should_trace.
        self.source_match = None
        self.pylib_match = self.cover_match = None
        self.include_match = self.omit_match = None

        # Set the reporting precision.
        Numbers.set_precision(self.config.precision)

        # Is it ok for no data to be collected?
        self._warn_no_data = True
        self._warn_unimported_source = True

        # State machine variables:
        # Have we started collecting and not stopped it?
        self._started = False
        # Have we measured some data and not harvested it?
        self._measured = False

        atexit.register(self._atexit)

    def _canonical_dir(self, morf):
        """Return the canonical directory of the module or file `morf`."""
        return os.path.split(CodeUnit(morf, self.file_locator).filename)[0]

    def _source_for_file(self, filename):
        """Return the source file for `filename`."""
        if not filename.endswith(".py"):
            if filename[-4:-1] == ".py":
                filename = filename[:-1]
            elif filename.endswith("$py.class"): # jython
                filename = filename[:-9] + ".py"
        return filename

    def _should_trace_with_reason(self, filename, frame):
        """Decide whether to trace execution in `filename`, with a reason.

        This function is called from the trace function.  As each new file name
        is encountered, this function determines whether it is traced or not.

        Returns a pair of values:  the first indicates whether the file should
        be traced: it's a canonicalized filename if it should be traced, None
        if it should not.  The second value is a string, the resason for the
        decision.

        """
        if not filename:
            # Empty string is pretty useless
            return None, "empty string isn't a filename"

        if filename.startswith('<'):
            # Lots of non-file execution is represented with artificial
            # filenames like "<string>", "<doctest readme.txt[0]>", or
            # "<exec_function>".  Don't ever trace these executions, since we
            # can't do anything with the data later anyway.
            return None, "not a real filename"

        self._check_for_packages()

        # Compiled Python files have two filenames: frame.f_code.co_filename is
        # the filename at the time the .pyc was compiled.  The second name is
        # __file__, which is where the .pyc was actually loaded from.  Since
        # .pyc files can be moved after compilation (for example, by being
        # installed), we look for __file__ in the frame and prefer it to the
        # co_filename value.
        dunder_file = frame.f_globals.get('__file__')
        if dunder_file:
            filename = self._source_for_file(dunder_file)

        # Jython reports the .class file to the tracer, use the source file.
        if filename.endswith("$py.class"):
            filename = filename[:-9] + ".py"

        canonical = self.file_locator.canonical_filename(filename)

        # If the user specified source or include, then that's authoritative
        # about the outer bound of what to measure and we don't have to apply
        # any canned exclusions. If they didn't, then we have to exclude the
        # stdlib and coverage.py directories.
        if self.source_match:
            if not self.source_match.match(canonical):
                return None, "falls outside the --source trees"
        elif self.include_match:
            if not self.include_match.match(canonical):
                return None, "falls outside the --include trees"
        else:
            # If we aren't supposed to trace installed code, then check if this
            # is near the Python standard library and skip it if so.
            if self.pylib_match and self.pylib_match.match(canonical):
                return None, "is in the stdlib"

            # We exclude the coverage code itself, since a little of it will be
            # measured otherwise.
            if self.cover_match and self.cover_match.match(canonical):
                return None, "is part of coverage.py"

        # Check the file against the omit pattern.
        if self.omit_match and self.omit_match.match(canonical):
            return None, "is inside an --omit pattern"

        return canonical, "because we love you"

    def _should_trace(self, filename, frame):
        """Decide whether to trace execution in `filename`.

        Calls `_should_trace_with_reason`, and returns just the decision.

        """
        canonical, reason = self._should_trace_with_reason(filename, frame)
        if self.debug.should('trace'):
            if not canonical:
                msg = "Not tracing %r: %s" % (filename, reason)
            else:
                msg = "Tracing %r" % (filename,)
            self.debug.write(msg)
        return canonical

    def _warn(self, msg):
        """Use `msg` as a warning."""
        self._warnings.append(msg)
        sys.stderr.write("Coverage.py warning: %s\n" % msg)

    def _check_for_packages(self):
        """Update the source_match matcher with latest imported packages."""
        # Our self.source_pkgs attribute is a list of package names we want to
        # measure.  Each time through here, we see if we've imported any of
        # them yet.  If so, we add its file to source_match, and we don't have
        # to look for that package any more.
        if self.source_pkgs:
            found = []
            for pkg in self.source_pkgs:
                try:
                    mod = sys.modules[pkg]
                except KeyError:
                    continue

                found.append(pkg)

                try:
                    pkg_file = mod.__file__
                except AttributeError:
                    pkg_file = None
                else:
                    d, f = os.path.split(pkg_file)
                    if f.startswith('__init__'):
                        # This is actually a package, return the directory.
                        pkg_file = d
                    else:
                        pkg_file = self._source_for_file(pkg_file)
                    pkg_file = self.file_locator.canonical_filename(pkg_file)
                    if not os.path.exists(pkg_file):
                        pkg_file = None

                if pkg_file:
                    self.source.append(pkg_file)
                    self.source_match.add(pkg_file)
                else:
                    self._warn("Module %s has no Python source." % pkg)

            for pkg in found:
                self.source_pkgs.remove(pkg)

    def use_cache(self, usecache):
        """Control the use of a data file (incorrectly called a cache).

        `usecache` is true or false, whether to read and write data on disk.

        """
        self.data.usefile(usecache)

    def load(self):
        """Load previously-collected coverage data from the data file."""
        self.collector.reset()
        self.data.read()

    def start(self):
        """Start measuring code coverage.

        Coverage measurement actually occurs in functions called after `start`
        is invoked.  Statements in the same scope as `start` won't be measured.

        Once you invoke `start`, you must also call `stop` eventually, or your
        process might not shut down cleanly.

        """
        if self.run_suffix:
            # Calling start() means we're running code, so use the run_suffix
            # as the data_suffix when we eventually save the data.
            self.data_suffix = self.run_suffix
        if self.auto_data:
            self.load()

        # Create the matchers we need for _should_trace
        if self.source or self.source_pkgs:
            self.source_match = TreeMatcher(self.source)
        else:
            if self.cover_dir:
                self.cover_match = TreeMatcher([self.cover_dir])
            if self.pylib_dirs:
                self.pylib_match = TreeMatcher(self.pylib_dirs)
        if self.include:
            self.include_match = FnmatchMatcher(self.include)
        if self.omit:
            self.omit_match = FnmatchMatcher(self.omit)

        # The user may want to debug things, show info if desired.
        if self.debug.should('config'):
            self.debug.write("Configuration values:")
            config_info = sorted(self.config.__dict__.items())
            self.debug.write_formatted_info(config_info)

        if self.debug.should('sys'):
            self.debug.write("Debugging info:")
            self.debug.write_formatted_info(self.sysinfo())

        self.collector.start()
        self._started = True
        self._measured = True

    def stop(self):
        """Stop measuring code coverage."""
        self._started = False
        self.collector.stop()

    def _atexit(self):
        """Clean up on process shutdown."""
        if self._started:
            self.stop()
        if self.auto_data:
            self.save()

    def erase(self):
        """Erase previously-collected coverage data.

        This removes the in-memory data collected in this session as well as
        discarding the data file.

        """
        self.collector.reset()
        self.data.erase()

    def clear_exclude(self, which='exclude'):
        """Clear the exclude list."""
        setattr(self.config, which + "_list", [])
        self._exclude_regex_stale()

    def exclude(self, regex, which='exclude'):
        """Exclude source lines from execution consideration.

        A number of lists of regular expressions are maintained.  Each list
        selects lines that are treated differently during reporting.

        `which` determines which list is modified.  The "exclude" list selects
        lines that are not considered executable at all.  The "partial" list
        indicates lines with branches that are not taken.

        `regex` is a regular expression.  The regex is added to the specified
        list.  If any of the regexes in the list is found in a line, the line
        is marked for special treatment during reporting.

        """
        excl_list = getattr(self.config, which + "_list")
        excl_list.append(regex)
        self._exclude_regex_stale()

    def _exclude_regex_stale(self):
        """Drop all the compiled exclusion regexes, a list was modified."""
        self._exclude_re.clear()

    def _exclude_regex(self, which):
        """Return a compiled regex for the given exclusion list."""
        if which not in self._exclude_re:
            excl_list = getattr(self.config, which + "_list")
            self._exclude_re[which] = join_regex(excl_list)
        return self._exclude_re[which]

    def get_exclude_list(self, which='exclude'):
        """Return a list of excluded regex patterns.

        `which` indicates which list is desired.  See `exclude` for the lists
        that are available, and their meaning.

        """
        return getattr(self.config, which + "_list")

    def save(self):
        """Save the collected coverage data to the data file."""
        data_suffix = self.data_suffix
        if data_suffix is True:
            # If data_suffix was a simple true value, then make a suffix with
            # plenty of distinguishing information.  We do this here in
            # `save()` at the last minute so that the pid will be correct even
            # if the process forks.
            extra = ""
            if _TEST_NAME_FILE:
                f = open(_TEST_NAME_FILE)
                test_name = f.read()
                f.close()
                extra = "." + test_name
            data_suffix = "%s%s.%s.%06d" % (
                socket.gethostname(), extra, os.getpid(),
                random.randint(0, 999999)
                )

        self._harvest_data()
        self.data.write(suffix=data_suffix)

    def combine(self):
        """Combine together a number of similarly-named coverage data files.

        All coverage data files whose name starts with `data_file` (from the
        coverage() constructor) will be read, and combined together into the
        current measurements.

        """
        aliases = None
        if self.config.paths:
            aliases = PathAliases(self.file_locator)
            for paths in self.config.paths.values():
                result = paths[0]
                for pattern in paths[1:]:
                    aliases.add(pattern, result)
        self.data.combine_parallel_data(aliases=aliases)

    def _harvest_data(self):
        """Get the collected data and reset the collector.

        Also warn about various problems collecting data.

        """
        if not self._measured:
            return

        self.data.add_line_data(self.collector.get_line_data())
        self.data.add_arc_data(self.collector.get_arc_data())
        self.collector.reset()

        # If there are still entries in the source_pkgs list, then we never
        # encountered those packages.
        if self._warn_unimported_source:
            for pkg in self.source_pkgs:
                self._warn("Module %s was never imported." % pkg)

        # Find out if we got any data.
        summary = self.data.summary()
        if not summary and self._warn_no_data:
            self._warn("No data was collected.")

        # Find files that were never executed at all.
        for src in self.source:
            for py_file in find_python_files(src):
                py_file = self.file_locator.canonical_filename(py_file)

                if self.omit_match and self.omit_match.match(py_file):
                    # Turns out this file was omitted, so don't pull it back
                    # in as unexecuted.
                    continue

                self.data.touch_file(py_file)

        self._measured = False

    # Backward compatibility with version 1.
    def analysis(self, morf):
        """Like `analysis2` but doesn't return excluded line numbers."""
        f, s, _, m, mf = self.analysis2(morf)
        return f, s, m, mf

    def analysis2(self, morf):
        """Analyze a module.

        `morf` is a module or a filename.  It will be analyzed to determine
        its coverage statistics.  The return value is a 5-tuple:

        * The filename for the module.
        * A list of line numbers of executable statements.
        * A list of line numbers of excluded statements.
        * A list of line numbers of statements not run (missing from
          execution).
        * A readable formatted string of the missing line numbers.

        The analysis uses the source file itself and the current measured
        coverage data.

        """
        analysis = self._analyze(morf)
        return (
            analysis.filename,
            sorted(analysis.statements),
            sorted(analysis.excluded),
            sorted(analysis.missing),
            analysis.missing_formatted(),
            )

    def _analyze(self, it):
        """Analyze a single morf or code unit.

        Returns an `Analysis` object.

        """
        self._harvest_data()
        if not isinstance(it, CodeUnit):
            it = code_unit_factory(it, self.file_locator)[0]

        return Analysis(self, it)

    def report(self, morfs=None, show_missing=True, ignore_errors=None,
                file=None,                          # pylint: disable=W0622
                omit=None, include=None
                ):
        """Write a summary report to `file`.

        Each module in `morfs` is listed, with counts of statements, executed
        statements, missing statements, and a list of lines missed.

        `include` is a list of filename patterns.  Modules whose filenames
        match those patterns will be included in the report. Modules matching
        `omit` will not be included in the report.

        Returns a float, the total percentage covered.

        """
        self._harvest_data()
        self.config.from_args(
            ignore_errors=ignore_errors, omit=omit, include=include,
            show_missing=show_missing,
            )
        reporter = SummaryReporter(self, self.config)
        return reporter.report(morfs, outfile=file)

    def annotate(self, morfs=None, directory=None, ignore_errors=None,
                    omit=None, include=None):
        """Annotate a list of modules.

        Each module in `morfs` is annotated.  The source is written to a new
        file, named with a ",cover" suffix, with each line prefixed with a
        marker to indicate the coverage of the line.  Covered lines have ">",
        excluded lines have "-", and missing lines have "!".

        See `coverage.report()` for other arguments.

        """
        self._harvest_data()
        self.config.from_args(
            ignore_errors=ignore_errors, omit=omit, include=include
            )
        reporter = AnnotateReporter(self, self.config)
        reporter.report(morfs, directory=directory)

    def html_report(self, morfs=None, directory=None, ignore_errors=None,
                    omit=None, include=None, extra_css=None, title=None):
        """Generate an HTML report.

        The HTML is written to `directory`.  The file "index.html" is the
        overview starting point, with links to more detailed pages for
        individual modules.

        `extra_css` is a path to a file of other CSS to apply on the page.
        It will be copied into the HTML directory.

        `title` is a text string (not HTML) to use as the title of the HTML
        report.

        See `coverage.report()` for other arguments.

        Returns a float, the total percentage covered.

        """
        self._harvest_data()
        self.config.from_args(
            ignore_errors=ignore_errors, omit=omit, include=include,
            html_dir=directory, extra_css=extra_css, html_title=title,
            )
        reporter = HtmlReporter(self, self.config)
        return reporter.report(morfs)

    def xml_report(self, morfs=None, outfile=None, ignore_errors=None,
                    omit=None, include=None):
        """Generate an XML report of coverage results.

        The report is compatible with Cobertura reports.

        Each module in `morfs` is included in the report.  `outfile` is the
        path to write the file to, "-" will write to stdout.

        See `coverage.report()` for other arguments.

        Returns a float, the total percentage covered.

        """
        self._harvest_data()
        self.config.from_args(
            ignore_errors=ignore_errors, omit=omit, include=include,
            xml_output=outfile,
            )
        file_to_close = None
        delete_file = False
        if self.config.xml_output:
            if self.config.xml_output == '-':
                outfile = sys.stdout
            else:
                outfile = open(self.config.xml_output, "w")
                file_to_close = outfile
        try:
            try:
                reporter = XmlReporter(self, self.config)
                return reporter.report(morfs, outfile=outfile)
            except CoverageException:
                delete_file = True
                raise
        finally:
            if file_to_close:
                file_to_close.close()
                if delete_file:
                    file_be_gone(self.config.xml_output)

    def sysinfo(self):
        """Return a list of (key, value) pairs showing internal information."""

        import coverage as covmod
        import platform, re

        try:
            implementation = platform.python_implementation()
        except AttributeError:
            implementation = "unknown"

        info = [
            ('version', covmod.__version__),
            ('coverage', covmod.__file__),
            ('cover_dir', self.cover_dir),
            ('pylib_dirs', self.pylib_dirs),
            ('tracer', self.collector.tracer_name()),
            ('config_files', self.config.attempted_config_files),
            ('configs_read', self.config.config_files),
            ('data_path', self.data.filename),
            ('python', sys.version.replace('\n', '')),
            ('platform', platform.platform()),
            ('implementation', implementation),
            ('executable', sys.executable),
            ('cwd', os.getcwd()),
            ('path', sys.path),
            ('environment', sorted([
                ("%s = %s" % (k, v)) for k, v in iitems(os.environ)
                    if re.search(r"^COV|^PY", k)
                ])),
            ('command_line', " ".join(getattr(sys, 'argv', ['???']))),
            ]
        if self.source_match:
            info.append(('source_match', self.source_match.info()))
        if self.include_match:
            info.append(('include_match', self.include_match.info()))
        if self.omit_match:
            info.append(('omit_match', self.omit_match.info()))
        if self.cover_match:
            info.append(('cover_match', self.cover_match.info()))
        if self.pylib_match:
            info.append(('pylib_match', self.pylib_match.info()))

        return info

def process_startup():
    """Call this at Python startup to perhaps measure coverage.

    If the environment variable COVERAGE_PROCESS_START is defined, coverage
    measurement is started.  The value of the variable is the config file
    to use.

    There are two ways to configure your Python installation to invoke this
    function when Python starts:

    #. Create or append to sitecustomize.py to add these lines::

        import coverage
        coverage.process_startup()

    #. Create a .pth file in your Python installation containing::

        import coverage; coverage.process_startup()

    """
    cps = os.environ.get("COVERAGE_PROCESS_START")
    if cps:
        cov = coverage(config_file=cps, auto_data=True)
        cov.start()
        cov._warn_no_data = False
        cov._warn_unimported_source = False

# A hack for debugging testing in subprocesses.
_TEST_NAME_FILE = "" #"/tmp/covtest.txt"


# -*- coding: utf-8 -*-
"""Top-level display functions for displaying object in different formats.

Authors:

* Brian Granger
"""

#-----------------------------------------------------------------------------
#       Copyright (C) 2008-2011 The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from __future__ import print_function

from xml.dom import minidom

from .displaypub import (
    publish_pretty, publish_html,
    publish_latex, publish_svg,
    publish_png, publish_json,
    publish_javascript, publish_jpeg
)

#-----------------------------------------------------------------------------
# Main functions
#-----------------------------------------------------------------------------

def display(*objs, **kwargs):
    """Display a Python object in all frontends.

    By default all representations will be computed and sent to the frontends.
    Frontends can decide which representation is used and how.

    Parameters
    ----------
    objs : tuple of objects
        The Python objects to display.
    include : list or tuple, optional
        A list of format type strings (MIME types) to include in the
        format data dict. If this is set *only* the format types included
        in this list will be computed.
    exclude : list or tuple, optional
        A list of format type string (MIME types) to exclue in the format
        data dict. If this is set all format types will be computed,
        except for those included in this argument.
    """
    include = kwargs.get('include')
    exclude = kwargs.get('exclude')

    from IPython.core.interactiveshell import InteractiveShell
    inst = InteractiveShell.instance()
    format = inst.display_formatter.format
    publish = inst.display_pub.publish

    for obj in objs:
        format_dict = format(obj, include=include, exclude=exclude)
        publish('IPython.core.display.display', format_dict)

def display_pretty(*objs, **kwargs):
    """Display the pretty (default) representation of an object.

    Parameters
    ----------
    objs : tuple of objects
        The Python objects to display, or if raw=True raw text data to
        display.
    raw : bool
        Are the data objects raw data or Python objects that need to be
        formatted before display? [default: False]
    """
    raw = kwargs.pop('raw',False)
    if raw:
        for obj in objs:
            publish_pretty(obj)
    else:
        display(*objs, include=['text/plain'])

def display_html(*objs, **kwargs):
    """Display the HTML representation of an object.

    Parameters
    ----------
    objs : tuple of objects
        The Python objects to display, or if raw=True raw HTML data to
        display.
    raw : bool
        Are the data objects raw data or Python objects that need to be
        formatted before display? [default: False]
    """
    raw = kwargs.pop('raw',False)
    if raw:
        for obj in objs:
            publish_html(obj)
    else:
        display(*objs, include=['text/plain','text/html'])

def display_svg(*objs, **kwargs):
    """Display the SVG representation of an object.

    Parameters
    ----------
    objs : tuple of objects
        The Python objects to display, or if raw=True raw svg data to
        display.
    raw : bool
        Are the data objects raw data or Python objects that need to be
        formatted before display? [default: False]
    """
    raw = kwargs.pop('raw',False)
    if raw:
        for obj in objs:
            publish_svg(obj)
    else:
        display(*objs, include=['text/plain','image/svg+xml'])

def display_png(*objs, **kwargs):
    """Display the PNG representation of an object.

    Parameters
    ----------
    objs : tuple of objects
        The Python objects to display, or if raw=True raw png data to
        display.
    raw : bool
        Are the data objects raw data or Python objects that need to be
        formatted before display? [default: False]
    """
    raw = kwargs.pop('raw',False)
    if raw:
        for obj in objs:
            publish_png(obj)
    else:
        display(*objs, include=['text/plain','image/png'])

def display_jpeg(*objs, **kwargs):
    """Display the JPEG representation of an object.

    Parameters
    ----------
    objs : tuple of objects
        The Python objects to display, or if raw=True raw JPEG data to
        display.
    raw : bool
        Are the data objects raw data or Python objects that need to be
        formatted before display? [default: False]
    """
    raw = kwargs.pop('raw',False)
    if raw:
        for obj in objs:
            publish_jpeg(obj)
    else:
        display(*objs, include=['text/plain','image/jpeg'])

def display_latex(*objs, **kwargs):
    """Display the LaTeX representation of an object.

    Parameters
    ----------
    objs : tuple of objects
        The Python objects to display, or if raw=True raw latex data to
        display.
    raw : bool
        Are the data objects raw data or Python objects that need to be
        formatted before display? [default: False]
    """
    raw = kwargs.pop('raw',False)
    if raw:
        for obj in objs:
            publish_latex(obj)
    else:
        display(*objs, include=['text/plain','text/latex'])

def display_json(*objs, **kwargs):
    """Display the JSON representation of an object.

    Note that not many frontends support displaying JSON.

    Parameters
    ----------
    objs : tuple of objects
        The Python objects to display, or if raw=True raw json data to
        display.
    raw : bool
        Are the data objects raw data or Python objects that need to be
        formatted before display? [default: False]
    """
    raw = kwargs.pop('raw',False)
    if raw:
        for obj in objs:
            publish_json(obj)
    else:
        display(*objs, include=['text/plain','application/json'])

def display_javascript(*objs, **kwargs):
    """Display the Javascript representation of an object.

    Parameters
    ----------
    objs : tuple of objects
        The Python objects to display, or if raw=True raw javascript data to
        display.
    raw : bool
        Are the data objects raw data or Python objects that need to be
        formatted before display? [default: False]
    """
    raw = kwargs.pop('raw',False)
    if raw:
        for obj in objs:
            publish_javascript(obj)
    else:
        display(*objs, include=['text/plain','application/javascript'])

#-----------------------------------------------------------------------------
# Smart classes
#-----------------------------------------------------------------------------

class DisplayObject(object):
    """An object that wraps data to be displayed."""

    _read_flags = 'r'

    def __init__(self, data=None, url=None, filename=None):
        """Create a display object given raw data.

        When this object is returned by an expression or passed to the
        display function, it will result in the data being displayed
        in the frontend. The MIME type of the data should match the
        subclasses used, so the Png subclass should be used for 'image/png'
        data. If the data is a URL, the data will first be downloaded
        and then displayed. If

        Parameters
        ----------
        data : unicode, str or bytes
            The raw data or a URL to download the data from.
        url : unicode
            A URL to download the data from.
        filename : unicode
            Path to a local file to load the data from.
        """
        if data is not None and data.startswith('http'):
            self.url = data
            self.filename = None
            self.data = None
        else:
            self.data = data
            self.url = url
            self.filename = None if filename is None else unicode(filename)
        self.reload()

    def reload(self):
        """Reload the raw data from file or URL."""
        if self.filename is not None:
            with open(self.filename, self._read_flags) as f:
                self.data = f.read()
        elif self.url is not None:
            try:
                import urllib2
                response = urllib2.urlopen(self.url)
                self.data = response.read()
                # extract encoding from header, if there is one:
                encoding = None
                for sub in response.headers['content-type'].split(';'):
                    sub = sub.strip()
                    if sub.startswith('charset'):
                        encoding = sub.split('=')[-1].strip()
                        break
                # decode data, if an encoding was specified
                if encoding:
                    self.data = self.data.decode(encoding, 'replace')
            except:
                self.data = None

class Pretty(DisplayObject):

    def _repr_pretty_(self):
        return self.data

class HTML(DisplayObject):

    def _repr_html_(self):
        return self.data

class Math(DisplayObject):

    def _repr_latex_(self):
        s = self.data.strip('$')
        return "$$%s$$" % s

class Latex(DisplayObject):

    def _repr_latex_(self):
        return self.data

class SVG(DisplayObject):

    # wrap data in a property, which extracts the <svg> tag, discarding
    # document headers
    _data = None

    @property
    def data(self):
        return self._data

    @data.setter
    def data(self, svg):
        if svg is None:
            self._data = None
            return
        # parse into dom object
        x = minidom.parseString(svg)
        # get svg tag (should be 1)
        found_svg = x.getElementsByTagName('svg')
        if found_svg:
            svg = found_svg[0].toxml()
        else:
            # fallback on the input, trust the user
            # but this is probably an error.
            pass
        self._data = svg

    def _repr_svg_(self):
        return self.data

class JSON(DisplayObject):

    def _repr_json_(self):
        return self.data

css_t = """$("head").append($("<link/>").attr({
  rel:  "stylesheet",
  type: "text/css",
  href: "%s"
}));
"""

lib_t1 = """$.getScript("%s", function () {
"""
lib_t2 = """});
"""

class Javascript(DisplayObject):

    def __init__(self, data=None, url=None, filename=None, lib=None, css=None):
        """Create a Javascript display object given raw data.

        When this object is returned by an expression or passed to the
        display function, it will result in the data being displayed
        in the frontend. If the data is a URL, the data will first be
        downloaded and then displayed.

        In the Notebook, the containing element will be available as `element`,
        and jQuery will be available.  The output area starts hidden, so if
        the js appends content to `element` that should be visible, then
        it must call `container.show()` to unhide the area.

        Parameters
        ----------
        data : unicode, str or bytes
            The Javascript source code or a URL to download it from.
        url : unicode
            A URL to download the data from.
        filename : unicode
            Path to a local file to load the data from.
        lib : list or str
            A sequence of Javascript library URLs to load asynchronously before
            running the source code. The full URLs of the libraries should
            be given. A single Javascript library URL can also be given as a
            string.
        css: : list or str
            A sequence of css files to load before running the source code.
            The full URLs of the css files should be give. A single css URL
            can also be given as a string.
        """
        if isinstance(lib, basestring):
            lib = [lib]
        elif lib is None:
            lib = []
        if isinstance(css, basestring):
            css = [css]
        elif css is None:
            css = []
        if not isinstance(lib, (list,tuple)):
            raise TypeError('expected sequence, got: %r' % lib)
        if not isinstance(css, (list,tuple)):
            raise TypeError('expected sequence, got: %r' % css)
        self.lib = lib
        self.css = css
        super(Javascript, self).__init__(data=data, url=url, filename=filename)

    def _repr_javascript_(self):
        r = ''
        for c in self.css:
            r += css_t % c
        for l in self.lib:
            r += lib_t1 % l
        r += self.data
        r += lib_t2*len(self.lib)
        return r

class Image(DisplayObject):

    _read_flags = 'rb'

    def __init__(self, data=None, url=None, filename=None, format=u'png', embed=None):
        """Create a display an PNG/JPEG image given raw data.

        When this object is returned by an expression or passed to the
        display function, it will result in the image being displayed
        in the frontend.

        Parameters
        ----------
        data : unicode, str or bytes
            The raw data or a URL to download the data from.
        url : unicode
            A URL to download the data from.
        filename : unicode
            Path to a local file to load the data from.
        format : unicode
            The format of the image data (png/jpeg/jpg). If a filename or URL is given
            for format will be inferred from the filename extension.
        embed : bool
            Should the image data be embedded using a data URI (True) or be
            loaded using an <img> tag. Set this to True if you want the image
            to be viewable later with no internet connection in the notebook.

            Default is `True`, unless the keyword argument `url` is set, then
            default value is `False`.

            Note that QtConsole is not able to display images if `embed` is set to `False`

        Examples
        --------
        # embed implicitly True, works in qtconsole and notebook
        Image('http://www.google.fr/images/srpr/logo3w.png')

        # embed implicitly False, does not works in qtconsole but works in notebook if
        # internet connection available
        Image(url='http://www.google.fr/images/srpr/logo3w.png')

        """
        if filename is not None:
            ext = self._find_ext(filename)
        elif url is not None:
            ext = self._find_ext(url)
        elif data.startswith('http'):
            ext = self._find_ext(data)
        else:
            ext = None
        if ext is not None:
            if ext == u'jpg' or ext == u'jpeg':
                format = u'jpeg'
            if ext == u'png':
                format = u'png'
        self.format = unicode(format).lower()
        self.embed = embed if embed is not None else (url is None)
        super(Image, self).__init__(data=data, url=url, filename=filename)

    def reload(self):
        """Reload the raw data from file or URL."""
        if self.embed:
            super(Image,self).reload()

    def _repr_html_(self):
        if not self.embed:
            return u'<img src="%s" />' % self.url

    def _repr_png_(self):
        if self.embed and self.format == u'png':
            return self.data

    def _repr_jpeg_(self):
        if self.embed and (self.format == u'jpeg' or self.format == u'jpg'):
            return self.data

    def _find_ext(self, s):
        return unicode(s.split('.')[-1].lower())

def clear_output(stdout=True, stderr=True, other=True):
    """Clear the output of the current cell receiving output.

    Optionally, each of stdout/stderr or other non-stream data (e.g. anything
    produced by display()) can be excluded from the clear event.

    By default, everything is cleared.

    Parameters
    ----------
    stdout : bool [default: True]
        Whether to clear stdout.
    stderr : bool [default: True]
        Whether to clear stderr.
    other : bool [default: True]
        Whether to clear everything else that is not stdout/stderr
        (e.g. figures,images,HTML, any result of display()).
    """
    from IPython.core.interactiveshell import InteractiveShell
    if InteractiveShell.initialized():
        InteractiveShell.instance().display_pub.clear_output(
            stdout=stdout, stderr=stderr, other=other,
        )
    else:
        from IPython.utils import io
        if stdout:
            print('\033[2K\r', file=io.stdout, end='')
            io.stdout.flush()
        if stderr:
            print('\033[2K\r', file=io.stderr, end='')
            io.stderr.flush()


from __future__ import absolute_import

import datetime
import errno
import json
import logging
import os.path
import sys

from pip._vendor import lockfile
from pip._vendor import pkg_resources

from pip.compat import total_seconds
from pip.index import PyPI
from pip.locations import USER_CACHE_DIR, running_under_virtualenv
from pip.utils.filesystem import check_path_owner

SELFCHECK_DATE_FMT = "%Y-%m-%dT%H:%M:%SZ"

logger = logging.getLogger(__name__)

class VirtualenvSelfCheckState(object):
    def __init__(self):
        self.statefile_path = os.path.join(sys.prefix, "pip-selfcheck.json")

        # Load the existing state
        try:
            with open(self.statefile_path) as statefile:
                self.state = json.load(statefile)
        except (IOError, ValueError):
            self.state = {}

    def save(self, pypi_version, current_time):
        # Attempt to write out our version check file
        with open(self.statefile_path, "w") as statefile:
            json.dump(
                {
                    "last_check": current_time.strftime(SELFCHECK_DATE_FMT),
                    "pypi_version": pypi_version,
                },
                statefile,
                sort_keys=True,
                separators=(",", ":")
            )

class GlobalSelfCheckState(object):
    def __init__(self):
        self.statefile_path = os.path.join(USER_CACHE_DIR, "selfcheck.json")

        # Load the existing state
        try:
            with open(self.statefile_path) as statefile:
                self.state = json.load(statefile)[sys.prefix]
        except (IOError, ValueError, KeyError):
            self.state = {}

    def save(self, pypi_version, current_time):
        # Check to make sure that we own the directory
        if not check_path_owner(os.path.dirname(self.statefile_path)):
            return

        # Now that we've ensured the directory is owned by this user, we'll go
        # ahead and make sure that all our directories are created.
        try:
            os.makedirs(os.path.dirname(self.statefile_path))
        except OSError as exc:
            if exc.errno != errno.EEXIST:
                raise

        # Attempt to write out our version check file
        with lockfile.LockFile(self.statefile_path):
            if os.path.exists(self.statefile_path):
                with open(self.statefile_path) as statefile:
                    state = json.load(statefile)
            else:
                state = {}

            state[sys.prefix] = {
                "last_check": current_time.strftime(SELFCHECK_DATE_FMT),
                "pypi_version": pypi_version,
            }

            with open(self.statefile_path, "w") as statefile:
                json.dump(state, statefile, sort_keys=True,
                          separators=(",", ":"))

def load_selfcheck_statefile():
    if running_under_virtualenv():
        return VirtualenvSelfCheckState()
    else:
        return GlobalSelfCheckState()

def pip_version_check(session):
    """Check for an update for pip.

    Limit the frequency of checks to once per week. State is stored either in
    the active virtualenv or in the user's USER_CACHE_DIR keyed off the prefix
    of the pip script path.
    """
    import pip  # imported here to prevent circular imports
    pypi_version = None

    try:
        state = load_selfcheck_statefile()

        current_time = datetime.datetime.utcnow()
        # Determine if we need to refresh the state
        if "last_check" in state.state and "pypi_version" in state.state:
            last_check = datetime.datetime.strptime(
                state.state["last_check"],
                SELFCHECK_DATE_FMT
            )
            if total_seconds(current_time - last_check) < 7 * 24 * 60 * 60:
                pypi_version = state.state["pypi_version"]

        # Refresh the version if we need to or just see if we need to warn
        if pypi_version is None:
            resp = session.get(
                PyPI.pip_json_url,
                headers={"Accept": "application/json"},
            )
            resp.raise_for_status()
            pypi_version = resp.json()["info"]["version"]

            # save that we've performed a check
            state.save(pypi_version, current_time)

        pip_version = pkg_resources.parse_version(pip.__version__)

        # Determine if our pypi_version is older
        if pip_version < pkg_resources.parse_version(pypi_version):
            logger.warning(
                "You are using pip version %s, however version %s is "
                "available.\nYou should consider upgrading via the "
                "'pip install --upgrade pip' command." % (pip.__version__,
                                                          pypi_version)
            )

    except Exception:
        logger.debug(
            "There was an error checking the latest version of pip",
            exc_info=True,
        )


"""Posix-specific implementation of process utilities.

This file is only meant to be imported by process.py, not by end-users.
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2010-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
from __future__ import print_function

# Stdlib
import subprocess as sp
import sys

from IPython.external import pexpect

# Our own
from .autoattr import auto_attr
from ._process_common import getoutput, arg_split
from IPython.utils import text
from IPython.utils import py3compat
from IPython.utils.encoding import DEFAULT_ENCODING

#-----------------------------------------------------------------------------
# Function definitions
#-----------------------------------------------------------------------------

def _find_cmd(cmd):
    """Find the full path to a command using which."""

    path = sp.Popen(['/usr/bin/env', 'which', cmd],
                    stdout=sp.PIPE, stderr=sp.PIPE).communicate()[0]
    return py3compat.bytes_to_str(path)

class ProcessHandler(object):
    """Execute subprocesses under the control of pexpect.
    """
    # Timeout in seconds to wait on each reading of the subprocess' output.
    # This should not be set too low to avoid cpu overusage from our side,
    # since we read in a loop whose period is controlled by this timeout.
    read_timeout = 0.05

    # Timeout to give a process if we receive SIGINT, between sending the
    # SIGINT to the process and forcefully terminating it.
    terminate_timeout = 0.2

    # File object where stdout and stderr of the subprocess will be written
    logfile = None

    # Shell to call for subprocesses to execute
    sh = None

    @auto_attr
    def sh(self):
        sh = pexpect.which('sh')
        if sh is None:
            raise OSError('"sh" shell not found')
        return sh

    def __init__(self, logfile=None, read_timeout=None, terminate_timeout=None):
        """Arguments are used for pexpect calls."""
        self.read_timeout = (ProcessHandler.read_timeout if read_timeout is
                             None else read_timeout)
        self.terminate_timeout = (ProcessHandler.terminate_timeout if
                                  terminate_timeout is None else
                                  terminate_timeout)
        self.logfile = sys.stdout if logfile is None else logfile

    def getoutput(self, cmd):
        """Run a command and return its stdout/stderr as a string.

        Parameters
        ----------
        cmd : str
          A command to be executed in the system shell.

        Returns
        -------
        output : str
          A string containing the combination of stdout and stderr from the
        subprocess, in whatever order the subprocess originally wrote to its
        file descriptors (so the order of the information in this string is the
        correct order as would be seen if running the command in a terminal).
        """
        try:
            return pexpect.run(self.sh, args=['-c', cmd]).replace('\r\n', '\n')
        except KeyboardInterrupt:
            print('^C', file=sys.stderr, end='')

    def getoutput_pexpect(self, cmd):
        """Run a command and return its stdout/stderr as a string.

        Parameters
        ----------
        cmd : str
          A command to be executed in the system shell.

        Returns
        -------
        output : str
          A string containing the combination of stdout and stderr from the
        subprocess, in whatever order the subprocess originally wrote to its
        file descriptors (so the order of the information in this string is the
        correct order as would be seen if running the command in a terminal).
        """
        try:
            return pexpect.run(self.sh, args=['-c', cmd]).replace('\r\n', '\n')
        except KeyboardInterrupt:
            print('^C', file=sys.stderr, end='')

    def system(self, cmd):
        """Execute a command in a subshell.

        Parameters
        ----------
        cmd : str
          A command to be executed in the system shell.

        Returns
        -------
        int : child's exitstatus
        """
        # Get likely encoding for the output.
        enc = DEFAULT_ENCODING

        # Patterns to match on the output, for pexpect.  We read input and
        # allow either a short timeout or EOF
        patterns = [pexpect.TIMEOUT, pexpect.EOF]
        # the index of the EOF pattern in the list.
        # even though we know it's 1, this call means we don't have to worry if
        # we change the above list, and forget to change this value:
        EOF_index = patterns.index(pexpect.EOF)
        # The size of the output stored so far in the process output buffer.
        # Since pexpect only appends to this buffer, each time we print we
        # record how far we've printed, so that next time we only print *new*
        # content from the buffer.
        out_size = 0
        try:
            # Since we're not really searching the buffer for text patterns, we
            # can set pexpect's search window to be tiny and it won't matter.
            # We only search for the 'patterns' timeout or EOF, which aren't in
            # the text itself.
            #child = pexpect.spawn(pcmd, searchwindowsize=1)
            if hasattr(pexpect, 'spawnb'):
                child = pexpect.spawnb(self.sh, args=['-c', cmd]) # Pexpect-U
            else:
                child = pexpect.spawn(self.sh, args=['-c', cmd])  # Vanilla Pexpect
            flush = sys.stdout.flush
            while True:
                # res is the index of the pattern that caused the match, so we
                # know whether we've finished (if we matched EOF) or not
                res_idx = child.expect_list(patterns, self.read_timeout)
                print(child.before[out_size:].decode(enc, 'replace'), end='')
                flush()
                if res_idx==EOF_index:
                    break
                # Update the pointer to what we've already printed
                out_size = len(child.before)
        except KeyboardInterrupt:
            # We need to send ^C to the process.  The ascii code for '^C' is 3
            # (the character is known as ETX for 'End of Text', see
            # curses.ascii.ETX).
            child.sendline(chr(3))
            # Read and print any more output the program might produce on its
            # way out.
            try:
                out_size = len(child.before)
                child.expect_list(patterns, self.terminate_timeout)
                print(child.before[out_size:].decode(enc, 'replace'), end='')
                sys.stdout.flush()
            except KeyboardInterrupt:
                # Impatient users tend to type it multiple times
                pass
            finally:
                # Ensure the subprocess really is terminated
                child.terminate(force=True)
        # add isalive check, to ensure exitstatus is set:
        child.isalive()
        return child.exitstatus

# Make system() with a functional interface for outside use.  Note that we use
# getoutput() from the _common utils, which is built on top of popen(). Using
# pexpect to get subprocess output produces difficult to parse output, since
# programs think they are talking to a tty and produce highly formatted output
# (ls is a good example) that makes them hard.
system = ProcessHandler().system


"""Utility for forwarding file read events over a zmq socket.

This is necessary because select on Windows only supports sockets, not FDs.

Authors:

* MinRK

"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import uuid
import zmq

from threading import Thread

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

class ForwarderThread(Thread):
    def __init__(self, sock, fd):
        Thread.__init__(self)
        self.daemon=True
        self.sock = sock
        self.fd = fd

    def run(self):
        """Loop through lines in self.fd, and send them over self.sock."""
        line = self.fd.readline()
        # allow for files opened in unicode mode
        if isinstance(line, unicode):
            send = self.sock.send_unicode
        else:
            send = self.sock.send
        while line:
            send(line)
            line = self.fd.readline()
        # line == '' means EOF
        self.fd.close()
        self.sock.close()

def forward_read_events(fd, context=None):
    """Forward read events from an FD over a socket.

    This method wraps a file in a socket pair, so it can
    be polled for read events by select (specifically zmq.eventloop.ioloop)
    """
    if context is None:
        context = zmq.Context.instance()
    push = context.socket(zmq.PUSH)
    push.setsockopt(zmq.LINGER, -1)
    pull = context.socket(zmq.PULL)
    addr='inproc://%s'%uuid.uuid4()
    push.bind(addr)
    pull.connect(addr)
    forwarder = ForwarderThread(push, fd)
    forwarder.start()
    return pull

__all__ = ['forward_read_events']