"""Code parsing for Coverage."""

import dis, re, sys, token, tokenize

from coverage.backward import set, sorted, StringIO # pylint: disable=W0622
from coverage.backward import open_source, range    # pylint: disable=W0622
from coverage.backward import reversed              # pylint: disable=W0622
from coverage.backward import bytes_to_ints
from coverage.bytecode import ByteCodes, CodeObjects
from coverage.misc import nice_pair, expensive, join_regex
from coverage.misc import CoverageException, NoSource, NotPython

class CodeParser(object):
    """Parse code to find executable lines, excluded lines, etc."""

    def __init__(self, text=None, filename=None, exclude=None):
        """
        Source can be provided as `text`, the text itself, or `filename`, from
        which the text will be read.  Excluded lines are those that match
        `exclude`, a regex.

        """
        assert text or filename, "CodeParser needs either text or filename"
        self.filename = filename or "<code>"
        self.text = text
        if not self.text:
            try:
                sourcef = open_source(self.filename)
                try:
                    self.text = sourcef.read()
                finally:
                    sourcef.close()
            except IOError:
                _, err, _ = sys.exc_info()
                raise NoSource(
                    "No source for code: '%s': %s" % (self.filename, err)
                    )

        # Scrap the BOM if it exists.
        if self.text and ord(self.text[0]) == 0xfeff:
            self.text = self.text[1:]

        self.exclude = exclude

        self.show_tokens = False

        # The text lines of the parsed code.
        self.lines = self.text.split('\n')

        # The line numbers of excluded lines of code.
        self.excluded = set()

        # The line numbers of docstring lines.
        self.docstrings = set()

        # The line numbers of class definitions.
        self.classdefs = set()

        # A dict mapping line numbers to (lo,hi) for multi-line statements.
        self.multiline = {}

        # The line numbers that start statements.
        self.statement_starts = set()

        # Lazily-created ByteParser
        self._byte_parser = None

    def _get_byte_parser(self):
        """Create a ByteParser on demand."""
        if not self._byte_parser:
            self._byte_parser = \
                            ByteParser(text=self.text, filename=self.filename)
        return self._byte_parser
    byte_parser = property(_get_byte_parser)

    def lines_matching(self, *regexes):
        """Find the lines matching one of a list of regexes.

        Returns a set of line numbers, the lines that contain a match for one
        of the regexes in `regexes`.  The entire line needn't match, just a
        part of it.

        """
        regex_c = re.compile(join_regex(regexes))
        matches = set()
        for i, ltext in enumerate(self.lines):
            if regex_c.search(ltext):
                matches.add(i+1)
        return matches

    def _raw_parse(self):
        """Parse the source to find the interesting facts about its lines.

        A handful of member fields are updated.

        """
        # Find lines which match an exclusion pattern.
        if self.exclude:
            self.excluded = self.lines_matching(self.exclude)

        # Tokenize, to find excluded suites, to find docstrings, and to find
        # multi-line statements.
        indent = 0
        exclude_indent = 0
        excluding = False
        prev_toktype = token.INDENT
        first_line = None
        empty = True

        tokgen = generate_tokens(self.text)
        for toktype, ttext, (slineno, _), (elineno, _), ltext in tokgen:
            if self.show_tokens:                # pragma: not covered
                print("%10s %5s %-20r %r" % (
                    tokenize.tok_name.get(toktype, toktype),
                    nice_pair((slineno, elineno)), ttext, ltext
                    ))
            if toktype == token.INDENT:
                indent += 1
            elif toktype == token.DEDENT:
                indent -= 1
            elif toktype == token.NAME and ttext == 'class':
                # Class definitions look like branches in the byte code, so
                # we need to exclude them.  The simplest way is to note the
                # lines with the 'class' keyword.
                self.classdefs.add(slineno)
            elif toktype == token.OP and ttext == ':':
                if not excluding and elineno in self.excluded:
                    # Start excluding a suite.  We trigger off of the colon
                    # token so that the #pragma comment will be recognized on
                    # the same line as the colon.
                    exclude_indent = indent
                    excluding = True
            elif toktype == token.STRING and prev_toktype == token.INDENT:
                # Strings that are first on an indented line are docstrings.
                # (a trick from trace.py in the stdlib.) This works for
                # 99.9999% of cases.  For the rest (!) see:
                # http://stackoverflow.com/questions/1769332/x/1769794#1769794
                self.docstrings.update(range(slineno, elineno+1))
            elif toktype == token.NEWLINE:
                if first_line is not None and elineno != first_line:
                    # We're at the end of a line, and we've ended on a
                    # different line than the first line of the statement,
                    # so record a multi-line range.
                    rng = (first_line, elineno)
                    for l in range(first_line, elineno+1):
                        self.multiline[l] = rng
                first_line = None

            if ttext.strip() and toktype != tokenize.COMMENT:
                # A non-whitespace token.
                empty = False
                if first_line is None:
                    # The token is not whitespace, and is the first in a
                    # statement.
                    first_line = slineno
                    # Check whether to end an excluded suite.
                    if excluding and indent <= exclude_indent:
                        excluding = False
                    if excluding:
                        self.excluded.add(elineno)

            prev_toktype = toktype

        # Find the starts of the executable statements.
        if not empty:
            self.statement_starts.update(self.byte_parser._find_statements())

    def first_line(self, line):
        """Return the first line number of the statement including `line`."""
        rng = self.multiline.get(line)
        if rng:
            first_line = rng[0]
        else:
            first_line = line
        return first_line

    def first_lines(self, lines, *ignores):
        """Map the line numbers in `lines` to the correct first line of the
        statement.

        Skip any line mentioned in any of the sequences in `ignores`.

        Returns a set of the first lines.

        """
        ignore = set()
        for ign in ignores:
            ignore.update(ign)
        lset = set()
        for l in lines:
            if l in ignore:
                continue
            new_l = self.first_line(l)
            if new_l not in ignore:
                lset.add(new_l)
        return lset

    def parse_source(self):
        """Parse source text to find executable lines, excluded lines, etc.

        Return values are 1) a set of executable line numbers, and 2) a set of
        excluded line numbers.

        Reported line numbers are normalized to the first line of multi-line
        statements.

        """
        try:
            self._raw_parse()
        except (tokenize.TokenError, IndentationError):
            _, tokerr, _ = sys.exc_info()
            msg, lineno = tokerr.args
            raise NotPython(
                "Couldn't parse '%s' as Python source: '%s' at %s" %
                    (self.filename, msg, lineno)
                )

        excluded_lines = self.first_lines(self.excluded)
        lines = self.first_lines(
            self.statement_starts,
            excluded_lines,
            self.docstrings
        )

        return lines, excluded_lines

    def arcs(self):
        """Get information about the arcs available in the code.

        Returns a sorted list of line number pairs.  Line numbers have been
        normalized to the first line of multiline statements.

        """
        all_arcs = []
        for l1, l2 in self.byte_parser._all_arcs():
            fl1 = self.first_line(l1)
            fl2 = self.first_line(l2)
            if fl1 != fl2:
                all_arcs.append((fl1, fl2))
        return sorted(all_arcs)
    arcs = expensive(arcs)

    def exit_counts(self):
        """Get a mapping from line numbers to count of exits from that line.

        Excluded lines are excluded.

        """
        excluded_lines = self.first_lines(self.excluded)
        exit_counts = {}
        for l1, l2 in self.arcs():
            if l1 < 0:
                # Don't ever report -1 as a line number
                continue
            if l1 in excluded_lines:
                # Don't report excluded lines as line numbers.
                continue
            if l2 in excluded_lines:
                # Arcs to excluded lines shouldn't count.
                continue
            if l1 not in exit_counts:
                exit_counts[l1] = 0
            exit_counts[l1] += 1

        # Class definitions have one extra exit, so remove one for each:
        for l in self.classdefs:
            # Ensure key is there: classdefs can include excluded lines.
            if l in exit_counts:
                exit_counts[l] -= 1

        return exit_counts
    exit_counts = expensive(exit_counts)

## Opcodes that guide the ByteParser.

def _opcode(name):
    """Return the opcode by name from the dis module."""
    return dis.opmap[name]

def _opcode_set(*names):
    """Return a set of opcodes by the names in `names`."""
    s = set()
    for name in names:
        try:
            s.add(_opcode(name))
        except KeyError:
            pass
    return s

# Opcodes that leave the code object.
OPS_CODE_END = _opcode_set('RETURN_VALUE')

# Opcodes that unconditionally end the code chunk.
OPS_CHUNK_END = _opcode_set(
    'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'RETURN_VALUE', 'RAISE_VARARGS',
    'BREAK_LOOP', 'CONTINUE_LOOP',
    )

# Opcodes that unconditionally begin a new code chunk.  By starting new chunks
# with unconditional jump instructions, we neatly deal with jumps to jumps
# properly.
OPS_CHUNK_BEGIN = _opcode_set('JUMP_ABSOLUTE', 'JUMP_FORWARD')

# Opcodes that push a block on the block stack.
OPS_PUSH_BLOCK = _opcode_set(
    'SETUP_LOOP', 'SETUP_EXCEPT', 'SETUP_FINALLY', 'SETUP_WITH'
    )

# Block types for exception handling.
OPS_EXCEPT_BLOCKS = _opcode_set('SETUP_EXCEPT', 'SETUP_FINALLY')

# Opcodes that pop a block from the block stack.
OPS_POP_BLOCK = _opcode_set('POP_BLOCK')

# Opcodes that have a jump destination, but aren't really a jump.
OPS_NO_JUMP = OPS_PUSH_BLOCK

# Individual opcodes we need below.
OP_BREAK_LOOP = _opcode('BREAK_LOOP')
OP_END_FINALLY = _opcode('END_FINALLY')
OP_COMPARE_OP = _opcode('COMPARE_OP')
COMPARE_EXCEPTION = 10  # just have to get this const from the code.
OP_LOAD_CONST = _opcode('LOAD_CONST')
OP_RETURN_VALUE = _opcode('RETURN_VALUE')

class ByteParser(object):
    """Parse byte codes to understand the structure of code."""

    def __init__(self, code=None, text=None, filename=None):
        if code:
            self.code = code
            self.text = text
        else:
            if not text:
                assert filename, "If no code or text, need a filename"
                sourcef = open_source(filename)
                try:
                    text = sourcef.read()
                finally:
                    sourcef.close()
            self.text = text

            try:
                # Python 2.3 and 2.4 don't like partial last lines, so be sure
                # the text ends nicely for them.
                self.code = compile(text + '\n', filename, "exec")
            except SyntaxError:
                _, synerr, _ = sys.exc_info()
                raise NotPython(
                    "Couldn't parse '%s' as Python source: '%s' at line %d" %
                        (filename, synerr.msg, synerr.lineno)
                    )

        # Alternative Python implementations don't always provide all the
        # attributes on code objects that we need to do the analysis.
        for attr in ['co_lnotab', 'co_firstlineno', 'co_consts', 'co_code']:
            if not hasattr(self.code, attr):
                raise CoverageException(
                    "This implementation of Python doesn't support code "
                    "analysis.\n"
                    "Run coverage.py under CPython for this command."
                    )

    def child_parsers(self):
        """Iterate over all the code objects nested within this one.

        The iteration includes `self` as its first value.

        """
        children = CodeObjects(self.code)
        return [ByteParser(code=c, text=self.text) for c in children]

    def _bytes_lines(self):
        """Map byte offsets to line numbers in `code`.

        Uses co_lnotab described in Python/compile.c to map byte offsets to
        line numbers.  Produces a sequence: (b0, l0), (b1, l1), ...

        Only byte offsets that correspond to line numbers are included in the
        results.

        """
        # Adapted from dis.py in the standard library.
        byte_increments = bytes_to_ints(self.code.co_lnotab[0::2])
        line_increments = bytes_to_ints(self.code.co_lnotab[1::2])

        last_line_num = None
        line_num = self.code.co_firstlineno
        byte_num = 0
        for byte_incr, line_incr in zip(byte_increments, line_increments):
            if byte_incr:
                if line_num != last_line_num:
                    yield (byte_num, line_num)
                    last_line_num = line_num
                byte_num += byte_incr
            line_num += line_incr
        if line_num != last_line_num:
            yield (byte_num, line_num)

    def _find_statements(self):
        """Find the statements in `self.code`.

        Produce a sequence of line numbers that start statements.  Recurses
        into all code objects reachable from `self.code`.

        """
        for bp in self.child_parsers():
            # Get all of the lineno information from this code.
            for _, l in bp._bytes_lines():
                yield l

    def _block_stack_repr(self, block_stack):
        """Get a string version of `block_stack`, for debugging."""
        blocks = ", ".join(
            ["(%s, %r)" % (dis.opname[b[0]], b[1]) for b in block_stack]
        )
        return "[" + blocks + "]"

    def _split_into_chunks(self):
        """Split the code object into a list of `Chunk` objects.

        Each chunk is only entered at its first instruction, though there can
        be many exits from a chunk.

        Returns a list of `Chunk` objects.

        """
        # The list of chunks so far, and the one we're working on.
        chunks = []
        chunk = None

        # A dict mapping byte offsets of line starts to the line numbers.
        bytes_lines_map = dict(self._bytes_lines())

        # The block stack: loops and try blocks get pushed here for the
        # implicit jumps that can occur.
        # Each entry is a tuple: (block type, destination)
        block_stack = []

        # Some op codes are followed by branches that should be ignored.  This
        # is a count of how many ignores are left.
        ignore_branch = 0

        # We have to handle the last two bytecodes specially.
        ult = penult = None

        # Get a set of all of the jump-to points.
        jump_to = set()
        bytecodes = list(ByteCodes(self.code.co_code))
        for bc in bytecodes:
            if bc.jump_to >= 0:
                jump_to.add(bc.jump_to)

        chunk_lineno = 0

        # Walk the byte codes building chunks.
        for bc in bytecodes:
            # Maybe have to start a new chunk
            start_new_chunk = False
            first_chunk = False
            if bc.offset in bytes_lines_map:
                # Start a new chunk for each source line number.
                start_new_chunk = True
                chunk_lineno = bytes_lines_map[bc.offset]
                first_chunk = True
            elif bc.offset in jump_to:
                # To make chunks have a single entrance, we have to make a new
                # chunk when we get to a place some bytecode jumps to.
                start_new_chunk = True
            elif bc.op in OPS_CHUNK_BEGIN:
                # Jumps deserve their own unnumbered chunk.  This fixes
                # problems with jumps to jumps getting confused.
                start_new_chunk = True

            if not chunk or start_new_chunk:
                if chunk:
                    chunk.exits.add(bc.offset)
                chunk = Chunk(bc.offset, chunk_lineno, first_chunk)
                chunks.append(chunk)

            # Look at the opcode
            if bc.jump_to >= 0 and bc.op not in OPS_NO_JUMP:
                if ignore_branch:
                    # Someone earlier wanted us to ignore this branch.
                    ignore_branch -= 1
                else:
                    # The opcode has a jump, it's an exit for this chunk.
                    chunk.exits.add(bc.jump_to)

            if bc.op in OPS_CODE_END:
                # The opcode can exit the code object.
                chunk.exits.add(-self.code.co_firstlineno)
            if bc.op in OPS_PUSH_BLOCK:
                # The opcode adds a block to the block_stack.
                block_stack.append((bc.op, bc.jump_to))
            if bc.op in OPS_POP_BLOCK:
                # The opcode pops a block from the block stack.
                block_stack.pop()
            if bc.op in OPS_CHUNK_END:
                # This opcode forces the end of the chunk.
                if bc.op == OP_BREAK_LOOP:
                    # A break is implicit: jump where the top of the
                    # block_stack points.
                    chunk.exits.add(block_stack[-1][1])
                chunk = None
            if bc.op == OP_END_FINALLY:
                # For the finally clause we need to find the closest exception
                # block, and use its jump target as an exit.
                for block in reversed(block_stack):
                    if block[0] in OPS_EXCEPT_BLOCKS:
                        chunk.exits.add(block[1])
                        break
            if bc.op == OP_COMPARE_OP and bc.arg == COMPARE_EXCEPTION:
                # This is an except clause.  We want to overlook the next
                # branch, so that except's don't count as branches.
                ignore_branch += 1

            penult = ult
            ult = bc

        if chunks:
            # The last two bytecodes could be a dummy "return None" that
            # shouldn't be counted as real code. Every Python code object seems
            # to end with a return, and a "return None" is inserted if there
            # isn't an explicit return in the source.
            if ult and penult:
                if penult.op == OP_LOAD_CONST and ult.op == OP_RETURN_VALUE:
                    if self.code.co_consts[penult.arg] is None:
                        # This is "return None", but is it dummy?  A real line
                        # would be a last chunk all by itself.
                        if chunks[-1].byte != penult.offset:
                            ex = -self.code.co_firstlineno
                            # Split the last chunk
                            last_chunk = chunks[-1]
                            last_chunk.exits.remove(ex)
                            last_chunk.exits.add(penult.offset)
                            chunk = Chunk(
                                penult.offset, last_chunk.line, False
                            )
                            chunk.exits.add(ex)
                            chunks.append(chunk)

            # Give all the chunks a length.
            chunks[-1].length = bc.next_offset - chunks[-1].byte # pylint: disable=W0631,C0301
            for i in range(len(chunks)-1):
                chunks[i].length = chunks[i+1].byte - chunks[i].byte

        #self.validate_chunks(chunks)
        return chunks

    def validate_chunks(self, chunks):
        """Validate the rule that chunks have a single entrance."""
        # starts is the entrances to the chunks
        starts = set([ch.byte for ch in chunks])
        for ch in chunks:
            assert all([(ex in starts or ex < 0) for ex in ch.exits])

    def _arcs(self):
        """Find the executable arcs in the code.

        Yields pairs: (from,to).  From and to are integer line numbers.  If
        from is < 0, then the arc is an entrance into the code object.  If to
        is < 0, the arc is an exit from the code object.

        """
        chunks = self._split_into_chunks()

        # A map from byte offsets to chunks jumped into.
        byte_chunks = dict([(c.byte, c) for c in chunks])

        # There's always an entrance at the first chunk.
        yield (-1, byte_chunks[0].line)

        # Traverse from the first chunk in each line, and yield arcs where
        # the trace function will be invoked.
        for chunk in chunks:
            if not chunk.first:
                continue

            chunks_considered = set()
            chunks_to_consider = [chunk]
            while chunks_to_consider:
                # Get the chunk we're considering, and make sure we don't
                # consider it again
                this_chunk = chunks_to_consider.pop()
                chunks_considered.add(this_chunk)

                # For each exit, add the line number if the trace function
                # would be triggered, or add the chunk to those being
                # considered if not.
                for ex in this_chunk.exits:
                    if ex < 0:
                        yield (chunk.line, ex)
                    else:
                        next_chunk = byte_chunks[ex]
                        if next_chunk in chunks_considered:
                            continue

                        # The trace function is invoked if visiting the first
                        # bytecode in a line, or if the transition is a
                        # backward jump.
                        backward_jump = next_chunk.byte < this_chunk.byte
                        if next_chunk.first or backward_jump:
                            if next_chunk.line != chunk.line:
                                yield (chunk.line, next_chunk.line)
                        else:
                            chunks_to_consider.append(next_chunk)

    def _all_chunks(self):
        """Returns a list of `Chunk` objects for this code and its children.

        See `_split_into_chunks` for details.

        """
        chunks = []
        for bp in self.child_parsers():
            chunks.extend(bp._split_into_chunks())

        return chunks

    def _all_arcs(self):
        """Get the set of all arcs in this code object and its children.

        See `_arcs` for details.

        """
        arcs = set()
        for bp in self.child_parsers():
            arcs.update(bp._arcs())

        return arcs

class Chunk(object):
    """A sequence of byte codes with a single entrance.

    To analyze byte code, we have to divide it into chunks, sequences of byte
    codes such that each chunk has only one entrance, the first instruction in
    the block.

    This is almost the CS concept of `basic block`_, except that we're willing
    to have many exits from a chunk, and "basic block" is a more cumbersome
    term.

    .. _basic block: http://en.wikipedia.org/wiki/Basic_block

    `line` is the source line number containing this chunk.

    `first` is true if this is the first chunk in the source line.

    An exit < 0 means the chunk can leave the code (return).  The exit is
    the negative of the starting line number of the code block.

    """
    def __init__(self, byte, line, first):
        self.byte = byte
        self.line = line
        self.first = first
        self.length = 0
        self.exits = set()

    def __repr__(self):
        if self.first:
            bang = "!"
        else:
            bang = ""
        return "<%d+%d @%d%s %r>" % (
            self.byte, self.length, self.line, bang, list(self.exits)
            )

class CachedTokenizer(object):
    """A one-element cache around tokenize.generate_tokens.

    When reporting, coverage.py tokenizes files twice, once to find the
    structure of the file, and once to syntax-color it.  Tokenizing is
    expensive, and easily cached.

    This is a one-element cache so that our twice-in-a-row tokenizing doesn't
    actually tokenize twice.

    """
    def __init__(self):
        self.last_text = None
        self.last_tokens = None

    def generate_tokens(self, text):
        """A stand-in for `tokenize.generate_tokens`."""
        if text != self.last_text:
            self.last_text = text
            self.last_tokens = list(
                tokenize.generate_tokens(StringIO(text).readline)
            )
        return self.last_tokens

# Create our generate_tokens cache as a callable replacement function.
generate_tokens = CachedTokenizer().generate_tokens


"""If you have Ned Batchelder's coverage_ module installed, you may activate a
coverage report with the ``--with-coverage`` switch or NOSE_WITH_COVERAGE
environment variable. The coverage report will cover any python source module
imported after the start of the test run, excluding modules that match
testMatch. If you want to include those modules too, use the ``--cover-tests``
switch, or set the NOSE_COVER_TESTS environment variable to a true value. To
restrict the coverage report to modules from a particular package or packages,
use the ``--cover-packages`` switch or the NOSE_COVER_PACKAGES environment
variable.

.. _coverage: http://www.nedbatchelder.com/code/modules/coverage.html
"""
import logging
import re
import sys
import StringIO
from nose.plugins.base import Plugin
from nose.util import src, tolist

log = logging.getLogger(__name__)

class Coverage(Plugin):
    """
    Activate a coverage report using Ned Batchelder's coverage module.
    """
    coverTests = False
    coverPackages = None
    coverInstance = None
    coverErase = False
    coverMinPercentage = None
    score = 200
    status = {}

    def options(self, parser, env):
        """
        Add options to command line.
        """
        super(Coverage, self).options(parser, env)
        parser.add_option("--cover-package", action="append",
                          default=env.get('NOSE_COVER_PACKAGE'),
                          metavar="PACKAGE",
                          dest="cover_packages",
                          help="Restrict coverage output to selected packages "
                          "[NOSE_COVER_PACKAGE]")
        parser.add_option("--cover-erase", action="store_true",
                          default=env.get('NOSE_COVER_ERASE'),
                          dest="cover_erase",
                          help="Erase previously collected coverage "
                          "statistics before run")
        parser.add_option("--cover-tests", action="store_true",
                          dest="cover_tests",
                          default=env.get('NOSE_COVER_TESTS'),
                          help="Include test modules in coverage report "
                          "[NOSE_COVER_TESTS]")
        parser.add_option("--cover-min-percentage", action="store",
                          dest="cover_min_percentage",
                          default=env.get('NOSE_COVER_MIN_PERCENTAGE'),
                          help="Minimum percentage of coverage for tests"
                          "to pass [NOSE_COVER_MIN_PERCENTAGE]")
        parser.add_option("--cover-inclusive", action="store_true",
                          dest="cover_inclusive",
                          default=env.get('NOSE_COVER_INCLUSIVE'),
                          help="Include all python files under working "
                          "directory in coverage report.  Useful for "
                          "discovering holes in test coverage if not all "
                          "files are imported by the test suite. "
                          "[NOSE_COVER_INCLUSIVE]")
        parser.add_option("--cover-html", action="store_true",
                          default=env.get('NOSE_COVER_HTML'),
                          dest='cover_html',
                          help="Produce HTML coverage information")
        parser.add_option('--cover-html-dir', action='store',
                          default=env.get('NOSE_COVER_HTML_DIR', 'cover'),
                          dest='cover_html_dir',
                          metavar='DIR',
                          help='Produce HTML coverage information in dir')
        parser.add_option("--cover-branches", action="store_true",
                          default=env.get('NOSE_COVER_BRANCHES'),
                          dest="cover_branches",
                          help="Include branch coverage in coverage report "
                          "[NOSE_COVER_BRANCHES]")
        parser.add_option("--cover-xml", action="store_true",
                          default=env.get('NOSE_COVER_XML'),
                          dest="cover_xml",
                          help="Produce XML coverage information")
        parser.add_option("--cover-xml-file", action="store",
                          default=env.get('NOSE_COVER_XML_FILE', 'coverage.xml'),
                          dest="cover_xml_file",
                          metavar="FILE",
                          help="Produce XML coverage information in file")

    def configure(self, options, conf):
        """
        Configure plugin.
        """
        try:
            self.status.pop('active')
        except KeyError:
            pass
        super(Coverage, self).configure(options, conf)
        if conf.worker:
            return
        if self.enabled:
            try:
                import coverage
            except ImportError:
                log.error("Coverage not available: "
                          "unable to import coverage module")
                self.enabled = False
                return
        self.conf = conf
        self.coverErase = options.cover_erase
        self.coverTests = options.cover_tests
        self.coverPackages = []
        if options.cover_packages:
            for pkgs in [tolist(x) for x in options.cover_packages]:
                self.coverPackages.extend(pkgs)
        self.coverInclusive = options.cover_inclusive
        if self.coverPackages:
            log.info("Coverage report will include only packages: %s",
                     self.coverPackages)
        self.coverHtmlDir = None
        if options.cover_html:
            self.coverHtmlDir = options.cover_html_dir
            log.debug('Will put HTML coverage report in %s', self.coverHtmlDir)
        self.coverBranches = options.cover_branches
        self.coverXmlFile = None
        if options.cover_min_percentage:
            self.coverMinPercentage = int(options.cover_min_percentage.rstrip('%'))
        if options.cover_xml:
            self.coverXmlFile = options.cover_xml_file
            log.debug('Will put XML coverage report in %s', self.coverXmlFile)
        if self.enabled:
            self.status['active'] = True
            self.coverInstance = coverage.coverage(auto_data=False,
                branch=self.coverBranches, data_suffix=None)

    def begin(self):
        """
        Begin recording coverage information.
        """
        log.debug("Coverage begin")
        self.skipModules = sys.modules.keys()[:]
        if self.coverErase:
            log.debug("Clearing previously collected coverage statistics")
            self.coverInstance.combine()
            self.coverInstance.erase()
        self.coverInstance.exclude('#pragma[: ]+[nN][oO] [cC][oO][vV][eE][rR]')
        self.coverInstance.load()
        self.coverInstance.start()

    def report(self, stream):
        """
        Output code coverage report.
        """
        log.debug("Coverage report")
        self.coverInstance.stop()
        self.coverInstance.combine()
        self.coverInstance.save()
        modules = [module
                    for name, module in sys.modules.items()
                    if self.wantModuleCoverage(name, module)]
        log.debug("Coverage report will cover modules: %s", modules)
        self.coverInstance.report(modules, file=stream)
        if self.coverHtmlDir:
            log.debug("Generating HTML coverage report")
            self.coverInstance.html_report(modules, self.coverHtmlDir)
        if self.coverXmlFile:
            log.debug("Generating XML coverage report")
            self.coverInstance.xml_report(modules, self.coverXmlFile)

        # make sure we have minimum required coverage
        if self.coverMinPercentage:
            f = StringIO.StringIO()
            self.coverInstance.report(modules, file=f)
            m = re.search(r'-------\s\w+\s+\d+\s+\d+\s+(\d+)%\s+\d*\s{0,1}$', f.getvalue())
            if m:
                percentage = int(m.groups()[0])
                if percentage < self.coverMinPercentage:
                    log.error('TOTAL Coverage did not reach minimum '
                              'required: %d%%' % self.coverMinPercentage)
                    sys.exit(1)
            else:
                log.error("No total percentage was found in coverage output, "
                          "something went wrong.")

    def wantModuleCoverage(self, name, module):
        if not hasattr(module, '__file__'):
            log.debug("no coverage of %s: no __file__", name)
            return False
        module_file = src(module.__file__)
        if not module_file or not module_file.endswith('.py'):
            log.debug("no coverage of %s: not a python file", name)
            return False
        if self.coverPackages:
            for package in self.coverPackages:
                if (re.findall(r'^%s\b' % re.escape(package), name)
                    and (self.coverTests
                         or not self.conf.testMatch.search(name))):
                    log.debug("coverage for %s", name)
                    return True
        if name in self.skipModules:
            log.debug("no coverage for %s: loaded before coverage start",
                      name)
            return False
        if self.conf.testMatch.search(name) and not self.coverTests:
            log.debug("no coverage for %s: is a test", name)
            return False
        # accept any package that passed the previous tests, unless
        # coverPackages is on -- in that case, if we wanted this
        # module, we would have already returned True
        return not self.coverPackages

    def wantFile(self, file, package=None):
        """If inclusive coverage enabled, return true for all source files
        in wanted packages.
        """
        if self.coverInclusive:
            if file.endswith(".py"):
                if package and self.coverPackages:
                    for want in self.coverPackages:
                        if package.startswith(want):
                            return True
                else:
                    return True
        return None


"""PyPI and direct package downloading"""
import sys, os.path, re, urlparse, urllib, urllib2, shutil, random, socket, cStringIO
import base64
import httplib
from pkg_resources import *
from distutils import log
from distutils.errors import DistutilsError
try:
    from hashlib import md5
except ImportError:
    from md5 import md5
from fnmatch import translate

EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$')
HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I)
# this is here to fix emacs' cruddy broken syntax highlighting
PYPI_MD5 = re.compile(
    '([^<]+)\n\s+\\(md5\\)'
)
URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match
EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split()

__all__ = [
    'PackageIndex', 'distros_for_url', 'parse_bdist_wininst',
    'interpret_distro_name',
]

_SOCKET_TIMEOUT = 15

def parse_bdist_wininst(name):
    """Return (base,pyversion) or (None,None) for possible .exe name"""

    lower = name.lower()
    base, py_ver, plat = None, None, None

    if lower.endswith('.exe'):
        if lower.endswith('.win32.exe'):
            base = name[:-10]
            plat = 'win32'
        elif lower.startswith('.win32-py',-16):
            py_ver = name[-7:-4]
            base = name[:-16]
            plat = 'win32'
        elif lower.endswith('.win-amd64.exe'):
            base = name[:-14]
            plat = 'win-amd64'
        elif lower.startswith('.win-amd64-py',-20):
            py_ver = name[-7:-4]
            base = name[:-20]
            plat = 'win-amd64'
    return base,py_ver,plat

def egg_info_for_url(url):
    scheme, server, path, parameters, query, fragment = urlparse.urlparse(url)
    base = urllib2.unquote(path.split('/')[-1])
    if '#' in base: base, fragment = base.split('#',1)
    return base,fragment

def distros_for_url(url, metadata=None):
    """Yield egg or source distribution objects that might be found at a URL"""
    base, fragment = egg_info_for_url(url)
    for dist in distros_for_location(url, base, metadata): yield dist
    if fragment:
        match = EGG_FRAGMENT.match(fragment)
        if match:
            for dist in interpret_distro_name(
                url, match.group(1), metadata, precedence = CHECKOUT_DIST
            ):
                yield dist

def distros_for_location(location, basename, metadata=None):
    """Yield egg or source distribution objects based on basename"""
    if basename.endswith('.egg.zip'):
        basename = basename[:-4]    # strip the .zip
    if basename.endswith('.egg') and '-' in basename:
        # only one, unambiguous interpretation
        return [Distribution.from_location(location, basename, metadata)]

    if basename.endswith('.exe'):
        win_base, py_ver, platform = parse_bdist_wininst(basename)
        if win_base is not None:
            return interpret_distro_name(
                location, win_base, metadata, py_ver, BINARY_DIST, platform
            )

    # Try source distro extensions (.zip, .tgz, etc.)
    #
    for ext in EXTENSIONS:
        if basename.endswith(ext):
            basename = basename[:-len(ext)]
            return interpret_distro_name(location, basename, metadata)
    return []  # no extension matched

def distros_for_filename(filename, metadata=None):
    """Yield possible egg or source distribution objects based on a filename"""
    return distros_for_location(
        normalize_path(filename), os.path.basename(filename), metadata
    )

def interpret_distro_name(location, basename, metadata,
    py_version=None, precedence=SOURCE_DIST, platform=None
):
    """Generate alternative interpretations of a source distro name

    Note: if `location` is a filesystem filename, you should call
    ``pkg_resources.normalize_path()`` on it before passing it to this
    routine!
    """
    # Generate alternative interpretations of a source distro name
    # Because some packages are ambiguous as to name/versions split
    # e.g. "adns-python-1.1.0", "egenix-mx-commercial", etc.
    # So, we generate each possible interepretation (e.g. "adns, python-1.1.0"
    # "adns-python, 1.1.0", and "adns-python-1.1.0, no version").  In practice,
    # the spurious interpretations should be ignored, because in the event
    # there's also an "adns" package, the spurious "python-1.1.0" version will
    # compare lower than any numeric version number, and is therefore unlikely
    # to match a request for it.  It's still a potential problem, though, and
    # in the long run PyPI and the distutils should go for "safe" names and
    # versions in distribution archive names (sdist and bdist).

    parts = basename.split('-')
    if not py_version:
        for i,p in enumerate(parts[2:]):
            if len(p)==5 and p.startswith('py2.'):
                return # It's a bdist_dumb, not an sdist -- bail out

    for p in range(1,len(parts)+1):
        yield Distribution(
            location, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]),
            py_version=py_version, precedence = precedence,
            platform = platform
        )

REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I)
# this line is here to fix emacs' cruddy broken syntax highlighting

def find_external_links(url, page):
    """Find rel="homepage" and rel="download" links in `page`, yielding URLs"""

    for match in REL.finditer(page):
        tag, rel = match.groups()
        rels = map(str.strip, rel.lower().split(','))
        if 'homepage' in rels or 'download' in rels:
            for match in HREF.finditer(tag):
                yield urlparse.urljoin(url, htmldecode(match.group(1)))

    for tag in ("Home Page", "Download URL"):
        pos = page.find(tag)
        if pos!=-1:
            match = HREF.search(page,pos)
            if match:
                yield urlparse.urljoin(url, htmldecode(match.group(1)))

user_agent = "Python-urllib/%s distribute/%s" % (
    sys.version[:3], require('distribute')[0].version
)

class PackageIndex(Environment):
    """A distribution index that scans web pages for download URLs"""

    def __init__(self, index_url="http://pypi.python.org/simple", hosts=('*',),
        *args, **kw
    ):
        Environment.__init__(self,*args,**kw)
        self.index_url = index_url + "/"[:not index_url.endswith('/')]
        self.scanned_urls = {}
        self.fetched_urls = {}
        self.package_pages = {}
        self.allows = re.compile('|'.join(map(translate,hosts))).match
        self.to_scan = []

    def process_url(self, url, retrieve=False):
        """Evaluate a URL as a possible download, and maybe retrieve it"""
        if url in self.scanned_urls and not retrieve:
            return
        self.scanned_urls[url] = True
        if not URL_SCHEME(url):
            self.process_filename(url)
            return
        else:
            dists = list(distros_for_url(url))
            if dists:
                if not self.url_ok(url):
                    return
                self.debug("Found link: %s", url)

        if dists or not retrieve or url in self.fetched_urls:
            map(self.add, dists)
            return  # don't need the actual page

        if not self.url_ok(url):
            self.fetched_urls[url] = True
            return

        self.info("Reading %s", url)
        f = self.open_url(url, "Download error on %s: %%s -- Some packages may not be found!" % url)
        if f is None: return
        self.fetched_urls[url] = self.fetched_urls[f.url] = True

        if 'html' not in f.headers.get('content-type', '').lower():
            f.close()   # not html, we can't process it
            return

        base = f.url     # handle redirects
        page = f.read()
        if not isinstance(page, str): # We are in Python 3 and got bytes. We want str.
            if isinstance(f, urllib2.HTTPError):
                # Errors have no charset, assume latin1:
                charset = 'latin-1'
            else:
                charset = f.headers.get_param('charset') or 'latin-1'
            page = page.decode(charset, "ignore")
        f.close()
        for match in HREF.finditer(page):
            link = urlparse.urljoin(base, htmldecode(match.group(1)))
            self.process_url(link)
        if url.startswith(self.index_url) and getattr(f,'code',None)!=404:
            page = self.process_index(url, page)

    def process_filename(self, fn, nested=False):
        # process filenames or directories
        if not os.path.exists(fn):
            self.warn("Not found: %s", fn)
            return

        if os.path.isdir(fn) and not nested:
            path = os.path.realpath(fn)
            for item in os.listdir(path):
                self.process_filename(os.path.join(path,item), True)

        dists = distros_for_filename(fn)
        if dists:
            self.debug("Found: %s", fn)
            map(self.add, dists)

    def url_ok(self, url, fatal=False):
        s = URL_SCHEME(url)
        if (s and s.group(1).lower()=='file') or self.allows(urlparse.urlparse(url)[1]):
            return True
        msg = "\nLink to % s ***BLOCKED*** by --allow-hosts\n"
        if fatal:
            raise DistutilsError(msg % url)
        else:
            self.warn(msg, url)

    def scan_egg_links(self, search_path):
        for item in search_path:
            if os.path.isdir(item):
                for entry in os.listdir(item):
                    if entry.endswith('.egg-link'):
                        self.scan_egg_link(item, entry)

    def scan_egg_link(self, path, entry):
        lines = filter(None, map(str.strip, open(os.path.join(path, entry))))
        if len(lines)==2:
            for dist in find_distributions(os.path.join(path, lines[0])):
                dist.location = os.path.join(path, *lines)
                dist.precedence = SOURCE_DIST
                self.add(dist)

    def process_index(self,url,page):
        """Process the contents of a PyPI page"""
        def scan(link):
            # Process a URL to see if it's for a package page
            if link.startswith(self.index_url):
                parts = map(
                    urllib2.unquote, link[len(self.index_url):].split('/')
                )
                if len(parts)==2 and '#' not in parts[1]:
                    # it's a package page, sanitize and index it
                    pkg = safe_name(parts[0])
                    ver = safe_version(parts[1])
                    self.package_pages.setdefault(pkg.lower(),{})[link] = True
                    return to_filename(pkg), to_filename(ver)
            return None, None

        # process an index page into the package-page index
        for match in HREF.finditer(page):
            try:
                scan( urlparse.urljoin(url, htmldecode(match.group(1))) )
            except ValueError:
                pass

        pkg, ver = scan(url)   # ensure this page is in the page index
        if pkg:
            # process individual package page
            for new_url in find_external_links(url, page):
                # Process the found URL
                base, frag = egg_info_for_url(new_url)
                if base.endswith('.py') and not frag:
                    if ver:
                        new_url+='#egg=%s-%s' % (pkg,ver)
                    else:
                        self.need_version_info(url)
                self.scan_url(new_url)

            return PYPI_MD5.sub(
                lambda m: '%s' % m.group(1,3,2), page
            )
        else:
            return ""   # no sense double-scanning non-package pages

    def need_version_info(self, url):
        self.scan_all(
            "Page at %s links to .py file(s) without version info; an index "
            "scan is required.", url
        )

    def scan_all(self, msg=None, *args):
        if self.index_url not in self.fetched_urls:
            if msg: self.warn(msg,*args)
            self.info(
                "Scanning index of all packages (this may take a while)"
            )
        self.scan_url(self.index_url)

    def find_packages(self, requirement):
        self.scan_url(self.index_url + requirement.unsafe_name+'/')

        if not self.package_pages.get(requirement.key):
            # Fall back to safe version of the name
            self.scan_url(self.index_url + requirement.project_name+'/')

        if not self.package_pages.get(requirement.key):
            # We couldn't find the target package, so search the index page too
            self.not_found_in_index(requirement)

        for url in list(self.package_pages.get(requirement.key,())):
            # scan each page that might be related to the desired package
            self.scan_url(url)

    def obtain(self, requirement, installer=None):
        self.prescan(); self.find_packages(requirement)
        for dist in self[requirement.key]:
            if dist in requirement:
                return dist
            self.debug("%s does not match %s", requirement, dist)
        return super(PackageIndex, self).obtain(requirement,installer)

    def check_md5(self, cs, info, filename, tfp):
        if re.match('md5=[0-9a-f]{32}$', info):
            self.debug("Validating md5 checksum for %s", filename)
            if cs.hexdigest()<>info[4:]:
                tfp.close()
                os.unlink(filename)
                raise DistutilsError(
                    "MD5 validation failed for "+os.path.basename(filename)+
                    "; possible download problem?"
                )

    def add_find_links(self, urls):
        """Add `urls` to the list that will be prescanned for searches"""
        for url in urls:
            if (
                self.to_scan is None        # if we have already "gone online"
                or not URL_SCHEME(url)      # or it's a local file/directory
                or url.startswith('file:')
                or list(distros_for_url(url))   # or a direct package link
            ):
                # then go ahead and process it now
                self.scan_url(url)
            else:
                # otherwise, defer retrieval till later
                self.to_scan.append(url)

    def prescan(self):
        """Scan urls scheduled for prescanning (e.g. --find-links)"""
        if self.to_scan:
            map(self.scan_url, self.to_scan)
        self.to_scan = None     # from now on, go ahead and process immediately

    def not_found_in_index(self, requirement):
        if self[requirement.key]:   # we've seen at least one distro
            meth, msg = self.info, "Couldn't retrieve index page for %r"
        else:   # no distros seen for this name, might be misspelled
            meth, msg = (self.warn,
                "Couldn't find index page for %r (maybe misspelled?)")
        meth(msg, requirement.unsafe_name)
        self.scan_all()

    def download(self, spec, tmpdir):
        """Locate and/or download `spec` to `tmpdir`, returning a local path

        `spec` may be a ``Requirement`` object, or a string containing a URL,
        an existing local filename, or a project/version requirement spec
        (i.e. the string form of a ``Requirement`` object).  If it is the URL
        of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one
        that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is
        automatically created alongside the downloaded file.

        If `spec` is a ``Requirement`` object or a string containing a
        project/version requirement spec, this method returns the location of
        a matching distribution (possibly after downloading it to `tmpdir`).
        If `spec` is a locally existing file or directory name, it is simply
        returned unchanged.  If `spec` is a URL, it is downloaded to a subpath
        of `tmpdir`, and the local filename is returned.  Various errors may be
        raised if a problem occurs during downloading.
        """
        if not isinstance(spec,Requirement):
            scheme = URL_SCHEME(spec)
            if scheme:
                # It's a url, download it to tmpdir
                found = self._download_url(scheme.group(1), spec, tmpdir)
                base, fragment = egg_info_for_url(spec)
                if base.endswith('.py'):
                    found = self.gen_setup(found,fragment,tmpdir)
                return found
            elif os.path.exists(spec):
                # Existing file or directory, just return it
                return spec
            else:
                try:
                    spec = Requirement.parse(spec)
                except ValueError:
                    raise DistutilsError(
                        "Not a URL, existing file, or requirement spec: %r" %
                        (spec,)
                    )
        return getattr(self.fetch_distribution(spec, tmpdir),'location',None)

    def fetch_distribution(self,
        requirement, tmpdir, force_scan=False, source=False, develop_ok=False,
        local_index=None
    ):
        """Obtain a distribution suitable for fulfilling `requirement`

        `requirement` must be a ``pkg_resources.Requirement`` instance.
        If necessary, or if the `force_scan` flag is set, the requirement is
        searched for in the (online) package index as well as the locally
        installed packages.  If a distribution matching `requirement` is found,
        the returned distribution's ``location`` is the value you would have
        gotten from calling the ``download()`` method with the matching
        distribution's URL or filename.  If no matching distribution is found,
        ``None`` is returned.

        If the `source` flag is set, only source distributions and source
        checkout links will be considered.  Unless the `develop_ok` flag is
        set, development and system eggs (i.e., those using the ``.egg-info``
        format) will be ignored.
        """

        # process a Requirement
        self.info("Searching for %s", requirement)
        skipped = {}
        dist = None

        def find(req, env=None):
            if env is None:
                env = self
            # Find a matching distribution; may be called more than once

            for dist in env[req.key]:

                if dist.precedence==DEVELOP_DIST and not develop_ok:
                    if dist not in skipped:
                        self.warn("Skipping development or system egg: %s",dist)
                        skipped[dist] = 1
                    continue

                if dist in req and (dist.precedence<=SOURCE_DIST or not source):
                    self.info("Best match: %s", dist)
                    return dist.clone(
                        location=self.download(dist.location, tmpdir)
                    )

        if force_scan:
            self.prescan()
            self.find_packages(requirement)
            dist = find(requirement)

        if local_index is not None:
            dist = dist or find(requirement, local_index)

        if dist is None and self.to_scan is not None:
            self.prescan()
            dist = find(requirement)

        if dist is None and not force_scan:
            self.find_packages(requirement)
            dist = find(requirement)

        if dist is None:
            self.warn(
                "No local packages or download links found for %s%s",
                (source and "a source distribution of " or ""),
                requirement,
            )
        return dist

    def fetch(self, requirement, tmpdir, force_scan=False, source=False):
        """Obtain a file suitable for fulfilling `requirement`

        DEPRECATED; use the ``fetch_distribution()`` method now instead.  For
        backward compatibility, this routine is identical but returns the
        ``location`` of the downloaded distribution instead of a distribution
        object.
        """
        dist = self.fetch_distribution(requirement,tmpdir,force_scan,source)
        if dist is not None:
            return dist.location
        return None

    def gen_setup(self, filename, fragment, tmpdir):
        match = EGG_FRAGMENT.match(fragment)
        dists = match and [d for d in
            interpret_distro_name(filename, match.group(1), None) if d.version
        ] or []

        if len(dists)==1:   # unambiguous ``#egg`` fragment
            basename = os.path.basename(filename)

            # Make sure the file has been downloaded to the temp dir.
            if os.path.dirname(filename) != tmpdir:
                dst = os.path.join(tmpdir, basename)
                from setuptools.command.easy_install import samefile
                if not samefile(filename, dst):
                    shutil.copy2(filename, dst)
                    filename=dst

            file = open(os.path.join(tmpdir, 'setup.py'), 'w')
            file.write(
                "from setuptools import setup\n"
                "setup(name=%r, version=%r, py_modules=[%r])\n"
                % (
                    dists[0].project_name, dists[0].version,
                    os.path.splitext(basename)[0]
                )
            )
            file.close()
            return filename

        elif match:
            raise DistutilsError(
                "Can't unambiguously interpret project/version identifier %r; "
                "any dashes in the name or version should be escaped using "
                "underscores. %r" % (fragment,dists)
            )
        else:
            raise DistutilsError(
                "Can't process plain .py files without an '#egg=name-version'"
                " suffix to enable automatic setup script generation."
            )

    dl_blocksize = 8192
    def _download_to(self, url, filename):
        self.info("Downloading %s", url)
        # Download the file
        fp, tfp, info = None, None, None
        try:
            if '#' in url:
                url, info = url.split('#', 1)
            fp = self.open_url(url)
            if isinstance(fp, urllib2.HTTPError):
                raise DistutilsError(
                    "Can't download %s: %s %s" % (url, fp.code,fp.msg)
                )
            cs = md5()
            headers = fp.info()
            blocknum = 0
            bs = self.dl_blocksize
            size = -1
            if "content-length" in headers:
                # Some servers return multiple Content-Length headers :(
                content_length = headers.get("Content-Length")
                size = int(content_length)
                self.reporthook(url, filename, blocknum, bs, size)
            tfp = open(filename,'wb')
            while True:
                block = fp.read(bs)
                if block:
                    cs.update(block)
                    tfp.write(block)
                    blocknum += 1
                    self.reporthook(url, filename, blocknum, bs, size)
                else:
                    break
            if info: self.check_md5(cs, info, filename, tfp)
            return headers
        finally:
            if fp: fp.close()
            if tfp: tfp.close()

    def reporthook(self, url, filename, blocknum, blksize, size):
        pass    # no-op

    def open_url(self, url, warning=None):
        if url.startswith('file:'):
            return local_open(url)
        try:
            return open_with_auth(url)
        except (ValueError, httplib.InvalidURL), v:
            msg = ' '.join([str(arg) for arg in v.args])
            if warning:
                self.warn(warning, msg)
            else:
                raise DistutilsError('%s %s' % (url, msg))
        except urllib2.HTTPError, v:
            return v
        except urllib2.URLError, v:
            if warning:
                self.warn(warning, v.reason)
            else:
                raise DistutilsError("Download error for %s: %s"
                                     % (url, v.reason))
        except httplib.BadStatusLine, v:
            if warning:
                self.warn(warning, v.line)
            else:
                raise DistutilsError('%s returned a bad status line. '
                                     'The server might be down, %s' % \
                                             (url, v.line))
        except httplib.HTTPException, v:
            if warning:
                self.warn(warning, v)
            else:
                raise DistutilsError("Download error for %s: %s"
                                     % (url, v))

    def _download_url(self, scheme, url, tmpdir):
        # Determine download filename
        #
        name = filter(None,urlparse.urlparse(url)[2].split('/'))
        if name:
            name = name[-1]
            while '..' in name:
                name = name.replace('..','.').replace('\\','_')
        else:
            name = "__downloaded__"    # default if URL has no path contents

        if name.endswith('.egg.zip'):
            name = name[:-4]    # strip the extra .zip before download

        filename = os.path.join(tmpdir,name)

        # Download the file
        #
        if scheme=='svn' or scheme.startswith('svn+'):
            return self._download_svn(url, filename)
        elif scheme=='git' or scheme.startswith('git+'):
            return self._download_git(url, filename)
        elif scheme.startswith('hg+'):
            return self._download_hg(url, filename)
        elif scheme=='file':
            return urllib.url2pathname(urlparse.urlparse(url)[2])
        else:
            self.url_ok(url, True)   # raises error if not allowed
            return self._attempt_download(url, filename)

    def scan_url(self, url):
        self.process_url(url, True)

    def _attempt_download(self, url, filename):
        headers = self._download_to(url, filename)
        if 'html' in headers.get('content-type','').lower():
            return self._download_html(url, headers, filename)
        else:
            return filename

    def _download_html(self, url, headers, filename):
        file = open(filename)
        for line in file:
            if line.strip():
                # Check for a subversion index page
                if re.search(r'<title>([^- ]+ - )?Revision \d+:', line):
                    # it's a subversion index page:
                    file.close()
                    os.unlink(filename)
                    return self._download_svn(url, filename)
                break   # not an index page
        file.close()
        os.unlink(filename)
        raise DistutilsError("Unexpected HTML page found at "+url)

    def _download_svn(self, url, filename):
        url = url.split('#',1)[0]   # remove any fragment for svn's sake
        self.info("Doing subversion checkout from %s to %s", url, filename)
        os.system("svn checkout -q %s %s" % (url, filename))
        return filename

    def _vcs_split_rev_from_url(self, url, pop_prefix=False):
        scheme, netloc, path, query, frag = urlparse.urlsplit(url)

        scheme = scheme.split('+', 1)[-1]

        # Some fragment identification fails
        path = path.split('#',1)[0]

        rev = None
        if '@' in path:
            path, rev = path.rsplit('@', 1)

        # Also, discard fragment
        url = urlparse.urlunsplit((scheme, netloc, path, query, ''))

        return url, rev

    def _download_git(self, url, filename):
        filename = filename.split('#',1)[0]
        url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True)

        self.info("Doing git clone from %s to %s", url, filename)
        os.system("git clone --quiet %s %s" % (url, filename))

        if rev is not None:
            self.info("Checking out %s", rev)
            os.system("(cd %s && git checkout --quiet %s)" % (
                filename,
                rev,
            ))

        return filename

    def _download_hg(self, url, filename):
        filename = filename.split('#',1)[0]
        url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True)

        self.info("Doing hg clone from %s to %s", url, filename)
        os.system("hg clone --quiet %s %s" % (url, filename))

        if rev is not None:
            self.info("Updating to %s", rev)
            os.system("(cd %s && hg up -C -r %s >&-)" % (
                filename,
                rev,
            ))

        return filename

    def debug(self, msg, *args):
        log.debug(msg, *args)

    def info(self, msg, *args):
        log.info(msg, *args)

    def warn(self, msg, *args):
        log.warn(msg, *args)

# This pattern matches a character entity reference (a decimal numeric
# references, a hexadecimal numeric reference, or a named reference).
entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub

def uchr(c):
    if not isinstance(c, int):
        return c
    if c>255: return unichr(c)
    return chr(c)

def decode_entity(match):
    what = match.group(1)
    if what.startswith('#x'):
        what = int(what[2:], 16)
    elif what.startswith('#'):
        what = int(what[1:])
    else:
        from htmlentitydefs import name2codepoint
        what = name2codepoint.get(what, match.group(0))
    return uchr(what)

def htmldecode(text):
    """Decode HTML entities in the given text."""
    return entity_sub(decode_entity, text)

def socket_timeout(timeout=15):
    def _socket_timeout(func):
        def _socket_timeout(*args, **kwargs):
            old_timeout = socket.getdefaulttimeout()
            socket.setdefaulttimeout(timeout)
            try:
                return func(*args, **kwargs)
            finally:
                socket.setdefaulttimeout(old_timeout)
        return _socket_timeout
    return _socket_timeout

def _encode_auth(auth):
    """
    A function compatible with Python 2.3-3.3 that will encode
    auth from a URL suitable for an HTTP header.
    >>> _encode_auth('username%3Apassword')
    u'dXNlcm5hbWU6cGFzc3dvcmQ='
    """
    auth_s = urllib2.unquote(auth)
    # convert to bytes
    auth_bytes = auth_s.encode()
    # use the legacy interface for Python 2.3 support
    encoded_bytes = base64.encodestring(auth_bytes)
    # convert back to a string
    encoded = encoded_bytes.decode()
    # strip the trailing carriage return
    return encoded.rstrip()

def open_with_auth(url):
    """Open a urllib2 request, handling HTTP authentication"""

    scheme, netloc, path, params, query, frag = urlparse.urlparse(url)

    # Double scheme does not raise on Mac OS X as revealed by a
    # failing test. We would expect "nonnumeric port". Refs #20.
    if netloc.endswith(':'):
        raise httplib.InvalidURL("nonnumeric port: ''")

    if scheme in ('http', 'https'):
        auth, host = urllib2.splituser(netloc)
    else:
        auth = None

    if auth:
        auth = "Basic " + _encode_auth(auth)
        new_url = urlparse.urlunparse((scheme,host,path,params,query,frag))
        request = urllib2.Request(new_url)
        request.add_header("Authorization", auth)
    else:
        request = urllib2.Request(url)

    request.add_header('User-Agent', user_agent)
    fp = urllib2.urlopen(request)

    if auth:
        # Put authentication info back into request URL if same host,
        # so that links found on the page will work
        s2, h2, path2, param2, query2, frag2 = urlparse.urlparse(fp.url)
        if s2==scheme and h2==host:
            fp.url = urlparse.urlunparse((s2,netloc,path2,param2,query2,frag2))

    return fp

# adding a timeout to avoid freezing package_index
open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth)

def fix_sf_url(url):
    return url      # backward compatibility

def local_open(url):
    """Read a local path, with special support for directories"""
    scheme, server, path, param, query, frag = urlparse.urlparse(url)
    filename = urllib.url2pathname(path)
    if os.path.isfile(filename):
        return urllib2.urlopen(url)
    elif path.endswith('/') and os.path.isdir(filename):
        files = []
        for f in os.listdir(filename):
            if f=='index.html':
                fp = open(os.path.join(filename,f),'rb')
                body = fp.read()
                fp.close()
                break
            elif os.path.isdir(os.path.join(filename,f)):
                f+='/'
            files.append("%s" % (f,f))
        else:
            body = ("<title>%s</title>" % url) + \
                "%s" % '\n'.join(files)
        status, message = 200, "OK"
    else:
        status, message, body = 404, "Path not found", "Not found"

    return urllib2.HTTPError(url, status, message,
            {'content-type':'text/html'}, cStringIO.StringIO(body))

# this line is a kludge to keep the trailing blank lines for pje's editor


import logging
from logging import INFO, DEBUG, WARN, ERROR, FATAL

import zmq
from zmq.log.handlers import PUBHandler

class EnginePUBHandler(PUBHandler):
    """A simple PUBHandler subclass that sets root_topic"""
    engine=None

    def __init__(self, engine, *args, **kwargs):
        PUBHandler.__init__(self,*args, **kwargs)
        self.engine = engine

    @property
    def root_topic(self):
        """this is a property, in case the handler is created
        before the engine gets registered with an id"""
        if isinstance(getattr(self.engine, 'id', None), int):
            return "engine.%i"%self.engine.id
        else:
            return "engine"


# -*- coding: utf-8 -*-
"""
Class which mimics a module.

Needed to allow pickle to correctly resolve namespaces during IPython
sessions.
"""

#*****************************************************************************
#       Copyright (C) 2002-2004 Fernando Perez. <fperez@colorado.edu>
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#*****************************************************************************

import types

def init_fakemod_dict(fm,adict=None):
    """Initialize a FakeModule instance __dict__.

    Kept as a standalone function and not a method so the FakeModule API can
    remain basically empty.

    This should be considered for private IPython use, used in managing
    namespaces for %run.

    Parameters
    ----------

    fm : FakeModule instance

    adict : dict, optional
    """

    dct = {}
    # It seems pydoc (and perhaps others) needs any module instance to
    # implement a __nonzero__ method, so we add it if missing:
    dct.setdefault('__nonzero__',lambda : True)
    dct.setdefault('__file__',__file__)

    if adict is not None:
        dct.update(adict)

    # Hard assignment of the object's __dict__.  This is nasty but deliberate.
    fm.__dict__.clear()
    fm.__dict__.update(dct)

class FakeModule(types.ModuleType):
    """Simple class with attribute access to fake a module.

    This is not meant to replace a module, but to allow inserting a fake
    module in sys.modules so that systems which rely on run-time module
    importing (like shelve and pickle) work correctly in interactive IPython
    sessions.

    Do NOT use this code for anything other than this IPython private hack."""

    def __init__(self,adict=None):

        # tmp to force __dict__ instance creation, else self.__dict__ fails
        self.__iptmp = None
        # cleanup our temp trick
        del self.__iptmp
        # Now, initialize the actual data in the instance dict.
        init_fakemod_dict(self,adict)


from django.template import Context, Template

def render_template(content, context):
    """ renders context aware template """
    rendered = Template(content).render(Context(context))
    return rendered


"""
This plugin captures stdout during test execution. If the test fails
or raises an error, the captured output will be appended to the error
or failure output. It is enabled by default but can be disabled with
the options ``-s`` or ``--nocapture``.

:Options:
  ``--nocapture``
    Don't capture stdout (any stdout output will be printed immediately)

"""
import logging
import os
import sys
from nose.plugins.base import Plugin
from nose.util import ln
from StringIO import StringIO

log = logging.getLogger(__name__)

class Capture(Plugin):
    """
    Output capture plugin. Enabled by default. Disable with ``-s`` or
    ``--nocapture``. This plugin captures stdout during test execution,
    appending any output captured to the error or failure output,
    should the test fail or raise an error.
    """
    enabled = True
    env_opt = 'NOSE_NOCAPTURE'
    name = 'capture'
    score = 500

    def __init__(self):
        self.stdout = []
        self._buf = None

    def options(self, parser, env):
        """Register commandline options
        """
        parser.add_option(
            "-s", "--nocapture", action="store_false",
            default=not env.get(self.env_opt), dest="capture",
            help="Don't capture stdout (any stdout output "
            "will be printed immediately) [NOSE_NOCAPTURE]")

    def configure(self, options, conf):
        """Configure plugin. Plugin is enabled by default.
        """
        self.conf = conf
        if not options.capture:
            self.enabled = False

    def afterTest(self, test):
        """Clear capture buffer.
        """
        self.end()
        self._buf = None

    def begin(self):
        """Replace sys.stdout with capture buffer.
        """
        self.start() # get an early handle on sys.stdout

    def beforeTest(self, test):
        """Flush capture buffer.
        """
        self.start()

    def formatError(self, test, err):
        """Add captured output to error report.
        """
        test.capturedOutput = output = self.buffer
        self._buf = None
        if not output:
            # Don't return None as that will prevent other
            # formatters from formatting and remove earlier formatters
            # formats, instead return the err we got
            return err
        ec, ev, tb = err
        return (ec, self.addCaptureToErr(ev, output), tb)

    def formatFailure(self, test, err):
        """Add captured output to failure report.
        """
        return self.formatError(test, err)

    def addCaptureToErr(self, ev, output):
        if isinstance(ev, Exception):
            if hasattr(ev, '__unicode__'):
                # 2.6+
                ev = unicode(ev)
            else:
                # 2.5-
                if not hasattr(ev, 'message'):
                    # 2.4
                    msg = len(ev.args) and ev.args[0] or ''
                else:
                    msg = ev.message
                if (isinstance(msg, basestring) and
                    not isinstance(msg, unicode)):
                    msg = msg.decode('utf8', 'replace')
                ev = u'%s: %s' % (ev.__class__.__name__, msg)
        if not isinstance(output, unicode):
            output = output.decode('utf8', 'replace')
        return u'\n'.join([ev, ln(u'>> begin captured stdout <<'),
                           output, ln(u'>> end captured stdout <<')])

    def start(self):
        self.stdout.append(sys.stdout)
        self._buf = StringIO()
        sys.stdout = self._buf

    def end(self):
        if self.stdout:
            sys.stdout = self.stdout.pop()

    def finalize(self, result):
        """Restore stdout.
        """
        while self.stdout:
            self.end()

    def _get_buffer(self):
        if self._buf is not None:
            return self._buf.getvalue()

    buffer = property(_get_buffer, None, None,
                      """Captured stdout output.""")


from django import template

register = template.Library()

@register.assignment_tag()
def sizeof(collection):
    """
    Usage:
    {% sizeof mylist as mylistsize %}
    """

    size = len(collection)
    return size

@register.filter
def splitBy(data, num):
    """ Turn a list to list of list """
    return [data[i:i + num] for i in range(0, len(data), num)]


"""Code for converting notebooks to and from the v2 format.

Authors:

* Brian Granger
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from .nbbase import (
    new_code_cell, new_text_cell, new_worksheet, new_notebook, new_output,
    nbformat, nbformat_minor
)

from IPython.nbformat import v2

#-----------------------------------------------------------------------------
# Code
#-----------------------------------------------------------------------------

def convert_to_this_nbformat(nb, orig_version=2, orig_minor=0):
    """Convert a notebook to the v3 format.

    Parameters
    ----------
    nb : NotebookNode
        The Python representation of the notebook to convert.
    orig_version : int
        The original version of the notebook to convert.
    orig_minor : int
        The original minor version of the notebook to convert (only relevant for v >= 3).
    """
    if orig_version == 1:
        nb = v2.convert_to_this_nbformat(nb)
        orig_version = 2
    if orig_version == 2:
        # Mark the original nbformat so consumers know it has been converted.
        nb.nbformat = nbformat
        nb.nbformat_minor = nbformat_minor

        nb.orig_nbformat = 2
        return nb
    elif orig_version == 3:
        if orig_minor != nbformat_minor:
            nb.orig_nbformat_minor = orig_minor
        nb.nbformat_minor = nbformat_minor
        return nb
    else:
        raise ValueError('Cannot convert a notebook from v%s to v3' % orig_version)


""" Style utilities, templates, and defaults for syntax highlighting widgets.
"""
#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from colorsys import rgb_to_hls
from pygments.styles import get_style_by_name
from pygments.token import Token

#-----------------------------------------------------------------------------
# Constants
#-----------------------------------------------------------------------------

# The default light style sheet: black text on a white background.
default_light_style_template = '''
    QPlainTextEdit, QTextEdit { background-color: %(bgcolor)s;
            color: %(fgcolor)s ;
            selection-background-color: %(select)s}
    .error { color: red; }
    .in-prompt { color: navy; }
    .in-prompt-number { font-weight: bold; }
    .out-prompt { color: darkred; }
    .out-prompt-number { font-weight: bold; }
    .inverted { background-color: %(fgcolor)s ; color:%(bgcolor)s;}
'''
default_light_style_sheet = default_light_style_template%dict(
                bgcolor='white', fgcolor='black', select="#ccc")
default_light_syntax_style = 'default'

# The default dark style sheet: white text on a black background.
default_dark_style_template = '''
    QPlainTextEdit, QTextEdit { background-color: %(bgcolor)s;
            color: %(fgcolor)s ;
            selection-background-color: %(select)s}
    QFrame { border: 1px solid grey; }
    .error { color: red; }
    .in-prompt { color: lime; }
    .in-prompt-number { color: lime; font-weight: bold; }
    .out-prompt { color: red; }
    .out-prompt-number { color: red; font-weight: bold; }
    .inverted { background-color: %(fgcolor)s ; color:%(bgcolor)s;}
'''
default_dark_style_sheet = default_dark_style_template%dict(
                bgcolor='black', fgcolor='white', select="#555")
default_dark_syntax_style = 'monokai'

# The default monochrome
default_bw_style_sheet = '''
    QPlainTextEdit, QTextEdit { background-color: white;
            color: black ;
            selection-background-color: #cccccc}
    .in-prompt-number { font-weight: bold; }
    .out-prompt-number { font-weight: bold; }
    .inverted { background-color: black ; color: white;}
'''
default_bw_syntax_style = 'bw'

def hex_to_rgb(color):
    """Convert a hex color to rgb integer tuple."""
    if color.startswith('#'):
        color = color[1:]
    if len(color) == 3:
        color = ''.join([c*2 for c in color])
    if len(color) != 6:
        return False
    try:
        r = int(color[:2],16)
        g = int(color[2:4],16)
        b = int(color[4:],16)
    except ValueError:
        return False
    else:
        return r,g,b

def dark_color(color):
    """Check whether a color is 'dark'.

    Currently, this is simply whether the luminance is <50%"""
    rgb = hex_to_rgb(color)
    if rgb:
        return rgb_to_hls(*rgb)[1] < 128
    else: # default to False
        return False

def dark_style(stylename):
    """Guess whether the background of the style with name 'stylename'
    counts as 'dark'."""
    return dark_color(get_style_by_name(stylename).background_color)

def get_colors(stylename):
    """Construct the keys to be used building the base stylesheet
    from a templatee."""
    style = get_style_by_name(stylename)
    fgcolor = style.style_for_token(Token.Text)['color'] or ''
    if len(fgcolor) in (3,6):
        # could be 'abcdef' or 'ace' hex, which needs '#' prefix
        try:
            int(fgcolor, 16)
        except TypeError:
            pass
        else:
            fgcolor = "#"+fgcolor

    return dict(
        bgcolor = style.background_color,
        select = style.highlight_color,
        fgcolor = fgcolor
    )

def sheet_from_template(name, colors='lightbg'):
    """Use one of the base templates, and set bg/fg/select colors."""
    colors = colors.lower()
    if colors=='lightbg':
        return default_light_style_template%get_colors(name)
    elif colors=='linux':
        return default_dark_style_template%get_colors(name)
    elif colors=='nocolor':
        return default_bw_style_sheet
    else:
        raise KeyError("No such color scheme: %s"%colors)


""" Defines miscellaneous Qt-related helper classes and functions.
"""

# Standard library imports.
import inspect

# System library imports.
from IPython.external.qt import QtCore, QtGui

# IPython imports.
from IPython.utils.traitlets import HasTraits, TraitType

#-----------------------------------------------------------------------------
# Metaclasses
#-----------------------------------------------------------------------------

MetaHasTraits = type(HasTraits)
MetaQObject = type(QtCore.QObject)

class MetaQObjectHasTraits(MetaQObject, MetaHasTraits):
    """ A metaclass that inherits from the metaclasses of HasTraits and QObject.

    Using this metaclass allows a class to inherit from both HasTraits and
    QObject. Using SuperQObject instead of QObject is highly recommended. See
    QtKernelManager for an example.
    """
    def __new__(mcls, name, bases, classdict):
        # FIXME: this duplicates the code from MetaHasTraits.
        # I don't think a super() call will help me here.
        for k,v in classdict.iteritems():
            if isinstance(v, TraitType):
                v.name = k
            elif inspect.isclass(v):
                if issubclass(v, TraitType):
                    vinst = v()
                    vinst.name = k
                    classdict[k] = vinst
        cls = MetaQObject.__new__(mcls, name, bases, classdict)
        return cls

    def __init__(mcls, name, bases, classdict):
        # Note: super() did not work, so we explicitly call these.
        MetaQObject.__init__(mcls, name, bases, classdict)
        MetaHasTraits.__init__(mcls, name, bases, classdict)

#-----------------------------------------------------------------------------
# Classes
#-----------------------------------------------------------------------------

class SuperQObject(QtCore.QObject):
    """ Permits the use of super() in class hierarchies that contain QObject.

    Unlike QObject, SuperQObject does not accept a QObject parent. If it did,
    super could not be emulated properly (all other classes in the heierarchy
    would have to accept the parent argument--they don't, of course, because
    they don't inherit QObject.)

    This class is primarily useful for attaching signals to existing non-Qt
    classes. See QtKernelManager for an example.
    """

    def __new__(cls, *args, **kw):
        # We initialize QObject as early as possible. Without this, Qt complains
        # if SuperQObject is not the first class in the super class list.
        inst = QtCore.QObject.__new__(cls)
        QtCore.QObject.__init__(inst)
        return inst

    def __init__(self, *args, **kw):
        # Emulate super by calling the next method in the MRO, if there is one.
        mro = self.__class__.mro()
        for qt_class in QtCore.QObject.mro():
            mro.remove(qt_class)
        next_index = mro.index(SuperQObject) + 1
        if next_index < len(mro):
            mro[next_index].__init__(self, *args, **kw)

#-----------------------------------------------------------------------------
# Functions
#-----------------------------------------------------------------------------

def get_font(family, fallback=None):
    """Return a font of the requested family, using fallback as alternative.

    If a fallback is provided, it is used in case the requested family isn't
    found.  If no fallback is given, no alternative is chosen and Qt's internal
    algorithms may automatically choose a fallback font.

    Parameters
    ----------
    family : str
      A font name.
    fallback : str
      A font name.

    Returns
    -------
    font : QFont object
    """
    font = QtGui.QFont(family)
    # Check whether we got what we wanted using QFontInfo, since exactMatch()
    # is overly strict and returns false in too many cases.
    font_info = QtGui.QFontInfo(font)
    if fallback is not None and font_info.family() != family:
        font = QtGui.QFont(fallback)
    return font