"""Routines related to PyPI, indexes"""
from __future__ import absolute_import

import logging
import cgi
import sys
import os
import re
import mimetypes
import posixpath
import warnings

from pip._vendor.six.moves.urllib import parse as urllib_parse
from pip._vendor.six.moves.urllib import request as urllib_request

from pip.compat import ipaddress
from pip.utils import Inf, cached_property, normalize_name, splitext
from pip.utils.deprecation import RemovedInPip7Warning, RemovedInPip8Warning
from pip.utils.logging import indent_log
from pip.exceptions import (
    DistributionNotFound, BestVersionAlreadyInstalled, InvalidWheelFilename,
    UnsupportedWheel,
)
from pip.download import url_to_path, path_to_url
from pip.models import PyPI
from pip.wheel import Wheel, wheel_ext
from pip.pep425tags import supported_tags, supported_tags_noarch, get_platform
from pip.req.req_requirement import InstallationCandidate
from pip._vendor import html5lib, requests, pkg_resources, six
from pip._vendor.packaging.version import parse as parse_version
from pip._vendor.requests.exceptions import SSLError

__all__ = ['PackageFinder']

# Taken from Chrome's list of secure origins (See: http://bit.ly/1qrySKC)
SECURE_ORIGINS = [
    # protocol, hostname, port
    ("https", "*", "*"),
    ("*", "localhost", "*"),
    ("*", "127.0.0.0/8", "*"),
    ("*", "::1/128", "*"),
    ("file", "*", None),
]

logger = logging.getLogger(__name__)

class PackageFinder(object):
    """This finds packages.

    This is meant to match easy_install's technique for looking for
    packages, by reading pages and looking for appropriate links
    """

    def __init__(self, find_links, index_urls,
                 use_wheel=True, allow_external=(), allow_unverified=(),
                 allow_all_external=False, allow_all_prereleases=False,
                 trusted_hosts=None, process_dependency_links=False,
                 session=None):
        if session is None:
            raise TypeError(
                "PackageFinder() missing 1 required keyword argument: "
                "'session'"
            )

        self.find_links = find_links
        self.index_urls = index_urls
        self.dependency_links = []

        # These are boring links that have already been logged somehow:
        self.logged_links = set()

        self.use_wheel = use_wheel

        # Do we allow (safe and verifiable) externally hosted files?
        self.allow_external = set(normalize_name(n) for n in allow_external)

        # Which names are allowed to install insecure and unverifiable files?
        self.allow_unverified = set(
            normalize_name(n) for n in allow_unverified
        )

        # Anything that is allowed unverified is also allowed external
        self.allow_external |= self.allow_unverified

        # Do we allow all (safe and verifiable) externally hosted files?
        self.allow_all_external = allow_all_external

        # Domains that we won't emit warnings for when not using HTTPS
        self.secure_origins = [
            ("*", host, "*")
            for host in (trusted_hosts if trusted_hosts else [])
        ]

        # Stores if we ignored any external links so that we can instruct
        #   end users how to install them if no distributions are available
        self.need_warn_external = False

        # Stores if we ignored any unsafe links so that we can instruct
        #   end users how to install them if no distributions are available
        self.need_warn_unverified = False

        # Do we want to allow _all_ pre-releases?
        self.allow_all_prereleases = allow_all_prereleases

        # Do we process dependency links?
        self.process_dependency_links = process_dependency_links

        # The Session we'll use to make requests
        self.session = session

    def add_dependency_links(self, links):
        # # FIXME: this shouldn't be global list this, it should only
        # # apply to requirements of the package that specifies the
        # # dependency_links value
        # # FIXME: also, we should track comes_from (i.e., use Link)
        if self.process_dependency_links:
            warnings.warn(
                "Dependency Links processing has been deprecated and will be "
                "removed in a future release.",
                RemovedInPip7Warning,
            )
            self.dependency_links.extend(links)

    def _sort_locations(self, locations):
        """
        Sort locations into "files" (archives) and "urls", and return
        a pair of lists (files,urls)
        """
        files = []
        urls = []

        # puts the url for the given file path into the appropriate list
        def sort_path(path):
            url = path_to_url(path)
            if mimetypes.guess_type(url, strict=False)[0] == 'text/html':
                urls.append(url)
            else:
                files.append(url)

        for url in locations:

            is_local_path = os.path.exists(url)
            is_file_url = url.startswith('file:')
            is_find_link = url in self.find_links

            if is_local_path or is_file_url:
                if is_local_path:
                    path = url
                else:
                    path = url_to_path(url)
                if is_find_link and os.path.isdir(path):
                    path = os.path.realpath(path)
                    for item in os.listdir(path):
                        sort_path(os.path.join(path, item))
                elif is_file_url and os.path.isdir(path):
                    urls.append(url)
                elif os.path.isfile(path):
                    sort_path(path)
            else:
                urls.append(url)

        return files, urls

    def _candidate_sort_key(self, candidate):
        """
        Function used to generate link sort key for link tuples.
        The greater the return value, the more preferred it is.
        If not finding wheels, then sorted by version only.
        If finding wheels, then the sort order is by version, then:
          1. existing installs
          2. wheels ordered via Wheel.support_index_min()
          3. source archives
        Note: it was considered to embed this logic into the Link
              comparison operators, but then different sdist links
              with the same version, would have to be considered equal
        """
        if self.use_wheel:
            support_num = len(supported_tags)
            if candidate.location == INSTALLED_VERSION:
                pri = 1
            elif candidate.location.ext == wheel_ext:
                # can raise InvalidWheelFilename
                wheel = Wheel(candidate.location.filename)
                if not wheel.supported():
                    raise UnsupportedWheel(
                        "%s is not a supported wheel for this platform. It "
                        "can't be sorted." % wheel.filename
                    )
                pri = -(wheel.support_index_min())
            else:  # sdist
                pri = -(support_num)
            return (candidate.version, pri)
        else:
            return candidate.version

    def _sort_versions(self, applicable_versions):
        """
        Bring the latest version (and wheels) to the front, but maintain the
        existing ordering as secondary. See the docstring for `_link_sort_key`
        for details. This function is isolated for easier unit testing.
        """
        return sorted(
            applicable_versions,
            key=self._candidate_sort_key,
            reverse=True
        )

    def _validate_secure_origin(self, logger, location):
        # Determine if this url used a secure transport mechanism
        parsed = urllib_parse.urlparse(str(location))
        origin = (parsed.scheme, parsed.hostname, parsed.port)

        # Determine if our origin is a secure origin by looking through our
        # hardcoded list of secure origins, as well as any additional ones
        # configured on this PackageFinder instance.
        for secure_origin in (SECURE_ORIGINS + self.secure_origins):
            # Check to see if the protocol matches
            if origin[0] != secure_origin[0] and secure_origin[0] != "*":
                continue

            try:
                # We need to do this decode dance to ensure that we have a
                # unicode object, even on Python 2.x.
                addr = ipaddress.ip_address(
                    origin[1]
                    if (
                        isinstance(origin[1], six.text_type)
                        or origin[1] is None
                    )
                    else origin[1].decode("utf8")
                )
                network = ipaddress.ip_network(
                    secure_origin[1]
                    if isinstance(secure_origin[1], six.text_type)
                    else secure_origin[1].decode("utf8")
                )
            except ValueError:
                # We don't have both a valid address or a valid network, so
                # we'll check this origin against hostnames.
                if origin[1] != secure_origin[1] and secure_origin[1] != "*":
                    continue
            else:
                # We have a valid address and network, so see if the address
                # is contained within the network.
                if addr not in network:
                    continue

            # Check to see if the port patches
            if (origin[2] != secure_origin[2]
                    and secure_origin[2] != "*"
                    and secure_origin[2] is not None):
                continue

            # If we've gotten here, then this origin matches the current
            # secure origin and we should break out of the loop and continue
            # on.
            break
        else:
            # If the loop successfully completed without a break, that means
            # that the origin we are testing is not a secure origin.
            logger.warning(
                "This repository located at %s is not a trusted host, if "
                "this repository is available via HTTPS it is recommend to "
                "use HTTPS instead, otherwise you may silence this warning "
                "with '--trusted-host %s'.",
                parsed.hostname,
                parsed.hostname,
            )

            warnings.warn(
                "Implicitly allowing locations which are not hosted at a "
                "secure origin is deprecated and will require the use of "
                "--trusted-host in the future.",
                RemovedInPip7Warning,
            )

    def find_requirement(self, req, upgrade):

        def mkurl_pypi_url(url):
            loc = posixpath.join(url, url_name)
            # For maximum compatibility with easy_install, ensure the path
            # ends in a trailing slash.  Although this isn't in the spec
            # (and PyPI can handle it without the slash) some other index
            # implementations might break if they relied on easy_install's
            # behavior.
            if not loc.endswith('/'):
                loc = loc + '/'
            return loc

        url_name = req.url_name

        # Only check main index if index URL is given:
        main_index_url = None
        if self.index_urls:
            # Check that we have the url_name correctly spelled:
            main_index_url = Link(
                mkurl_pypi_url(self.index_urls[0]),
                trusted=True,
            )

            page = self._get_page(main_index_url, req)
            if page is None and PyPI.netloc not in str(main_index_url):
                warnings.warn(
                    "Failed to find %r at %s. It is suggested to upgrade "
                    "your index to support normalized names as the name in "
                    "/simple/{name}." % (req.name, main_index_url),
                    RemovedInPip8Warning,
                )

                url_name = self._find_url_name(
                    Link(self.index_urls[0], trusted=True),
                    url_name, req
                ) or req.url_name

        if url_name is not None:
            locations = [
                mkurl_pypi_url(url)
                for url in self.index_urls] + self.find_links
        else:
            locations = list(self.find_links)

        file_locations, url_locations = self._sort_locations(locations)
        _flocations, _ulocations = self._sort_locations(self.dependency_links)
        file_locations.extend(_flocations)

        # We trust every url that the user has given us whether it was given
        #   via --index-url or --find-links
        locations = [Link(url, trusted=True) for url in url_locations]

        # We explicitly do not trust links that came from dependency_links
        locations.extend([Link(url) for url in _ulocations])

        logger.debug('URLs to search for versions for %s:', req)
        for location in locations:
            logger.debug('* %s', location)
            self._validate_secure_origin(logger, location)

        found_versions = []
        found_versions.extend(
            self._package_versions(
                # We trust every directly linked archive in find_links
                [Link(url, '-f', trusted=True) for url in self.find_links],
                req.name.lower()
            )
        )
        page_versions = []
        for page in self._get_pages(locations, req):
            logger.debug('Analyzing links from page %s', page.url)
            with indent_log():
                page_versions.extend(
                    self._package_versions(page.links, req.name.lower())
                )
        dependency_versions = list(self._package_versions(
            [Link(url) for url in self.dependency_links], req.name.lower()))
        if dependency_versions:
            logger.debug(
                'dependency_links found: %s',
                ', '.join([
                    version.location.url for version in dependency_versions
                ])
            )
        file_versions = list(
            self._package_versions(
                [Link(url) for url in file_locations],
                req.name.lower()
            )
        )
        if (not found_versions
                and not page_versions
                and not dependency_versions
                and not file_versions):
            logger.critical(
                'Could not find any downloads that satisfy the requirement %s',
                req,
            )

            if self.need_warn_external:
                logger.warning(
                    "Some externally hosted files were ignored as access to "
                    "them may be unreliable (use --allow-external %s to "
                    "allow).",
                    req.name,
                )

            if self.need_warn_unverified:
                logger.warning(
                    "Some insecure and unverifiable files were ignored"
                    " (use --allow-unverified %s to allow).",
                    req.name,
                )

            raise DistributionNotFound(
                'No distributions at all found for %s' % req
            )
        installed_version = []
        if req.satisfied_by is not None:
            installed_version = [
                InstallationCandidate(
                    req.name,
                    req.satisfied_by.version,
                    INSTALLED_VERSION,
                ),
            ]
        if file_versions:
            file_versions.sort(reverse=True)
            logger.debug(
                'Local files found: %s',
                ', '.join([
                    url_to_path(candidate.location.url)
                    for candidate in file_versions
                ])
            )

        # This is an intentional priority ordering
        all_versions = (
            file_versions + found_versions + page_versions
            + dependency_versions
        )

        # Filter out anything which doesn't match our specifier
        _versions = set(
            req.specifier.filter(
                [x.version for x in all_versions],
                prereleases=(
                    self.allow_all_prereleases
                    if self.allow_all_prereleases else None
                ),
            )
        )
        applicable_versions = [
            x for x in all_versions if x.version in _versions
        ]

        # Finally add our existing versions to the front of our versions.
        applicable_versions = installed_version + applicable_versions

        applicable_versions = self._sort_versions(applicable_versions)
        existing_applicable = any(
            i.location is INSTALLED_VERSION
            for i in applicable_versions
        )

        if not upgrade and existing_applicable:
            if applicable_versions[0].location is INSTALLED_VERSION:
                logger.debug(
                    'Existing installed version (%s) is most up-to-date and '
                    'satisfies requirement',
                    req.satisfied_by.version,
                )
            else:
                logger.debug(
                    'Existing installed version (%s) satisfies requirement '
                    '(most up-to-date version is %s)',
                    req.satisfied_by.version,
                    applicable_versions[0][2],
                )
            return None

        if not applicable_versions:
            logger.critical(
                'Could not find a version that satisfies the requirement %s '
                '(from versions: %s)',
                req,
                ', '.join(
                    sorted(
                        set(str(i.version) for i in all_versions),
                        key=parse_version,
                    )
                )
            )

            if self.need_warn_external:
                logger.warning(
                    "Some externally hosted files were ignored as access to "
                    "them may be unreliable (use --allow-external to allow)."
                )

            if self.need_warn_unverified:
                logger.warning(
                    "Some insecure and unverifiable files were ignored"
                    " (use --allow-unverified %s to allow).",
                    req.name,
                )

            raise DistributionNotFound(
                'No distributions matching the version for %s' % req
            )

        if applicable_versions[0].location is INSTALLED_VERSION:
            # We have an existing version, and its the best version
            logger.debug(
                'Installed version (%s) is most up-to-date (past versions: '
                '%s)',
                req.satisfied_by.version,
                ', '.join(str(i.version) for i in applicable_versions[1:])
                or "none",
            )
            raise BestVersionAlreadyInstalled

        if len(applicable_versions) > 1:
            logger.debug(
                'Using version %s (newest of versions: %s)',
                applicable_versions[0].version,
                ', '.join(str(i.version) for i in applicable_versions)
            )

        selected_version = applicable_versions[0].location

        if (selected_version.verifiable is not None
                and not selected_version.verifiable):
            logger.warning(
                "%s is potentially insecure and unverifiable.", req.name,
            )

        if selected_version._deprecated_regex:
            warnings.warn(
                "%s discovered using a deprecated method of parsing, in the "
                "future it will no longer be discovered." % req.name,
                RemovedInPip7Warning,
            )

        return selected_version

    def _find_url_name(self, index_url, url_name, req):
        """
        Finds the true URL name of a package, when the given name isn't quite
        correct.
        This is usually used to implement case-insensitivity.
        """
        if not index_url.url.endswith('/'):
            # Vaguely part of the PyPI API... weird but true.
            # FIXME: bad to modify this?
            index_url.url += '/'
        page = self._get_page(index_url, req)
        if page is None:
            logger.critical('Cannot fetch index base URL %s', index_url)
            return
        norm_name = normalize_name(req.url_name)
        for link in page.links:
            base = posixpath.basename(link.path.rstrip('/'))
            if norm_name == normalize_name(base):
                logger.debug(
                    'Real name of requirement %s is %s', url_name, base,
                )
                return base
        return None

    def _get_pages(self, locations, req):
        """
        Yields (page, page_url) from the given locations, skipping
        locations that have errors, and adding download/homepage links
        """
        all_locations = list(locations)
        seen = set()

        while all_locations:
            location = all_locations.pop(0)
            if location in seen:
                continue
            seen.add(location)

            page = self._get_page(location, req)
            if page is None:
                continue

            yield page

            for link in page.rel_links():
                normalized = normalize_name(req.name).lower()

                if (normalized not in self.allow_external
                        and not self.allow_all_external):
                    self.need_warn_external = True
                    logger.debug(
                        "Not searching %s for files because external "
                        "urls are disallowed.",
                        link,
                    )
                    continue

                if (link.trusted is not None
                        and not link.trusted
                        and normalized not in self.allow_unverified):
                    logger.debug(
                        "Not searching %s for urls, it is an "
                        "untrusted link and cannot produce safe or "
                        "verifiable files.",
                        link,
                    )
                    self.need_warn_unverified = True
                    continue

                all_locations.append(link)

    _egg_fragment_re = re.compile(r'#egg=([^&]*)')
    _egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)
    _py_version_re = re.compile(r'-py([123]\.?[0-9]?)$')

    def _sort_links(self, links):
        """
        Returns elements of links in order, non-egg links first, egg links
        second, while eliminating duplicates
        """
        eggs, no_eggs = [], []
        seen = set()
        for link in links:
            if link not in seen:
                seen.add(link)
                if link.egg_fragment:
                    eggs.append(link)
                else:
                    no_eggs.append(link)
        return no_eggs + eggs

    def _package_versions(self, links, search_name):
        for link in self._sort_links(links):
            v = self._link_package_versions(link, search_name)
            if v is not None:
                yield v

    def _known_extensions(self):
        extensions = ('.tar.gz', '.tar.bz2', '.tar', '.tgz', '.zip')
        if self.use_wheel:
            return extensions + (wheel_ext,)
        return extensions

    def _link_package_versions(self, link, search_name):
        """
        Return an iterable of triples (pkg_resources_version_key,
        link, python_version) that can be extracted from the given
        link.

        Meant to be overridden by subclasses, not called by clients.
        """
        platform = get_platform()

        version = None
        if link.egg_fragment:
            egg_info = link.egg_fragment
        else:
            egg_info, ext = link.splitext()
            if not ext:
                if link not in self.logged_links:
                    logger.debug('Skipping link %s; not a file', link)
                    self.logged_links.add(link)
                return
            if egg_info.endswith('.tar'):
                # Special double-extension case:
                egg_info = egg_info[:-4]
                ext = '.tar' + ext
            if ext not in self._known_extensions():
                if link not in self.logged_links:
                    logger.debug(
                        'Skipping link %s; unknown archive format: %s',
                        link,
                        ext,
                    )
                    self.logged_links.add(link)
                return
            if "macosx10" in link.path and ext == '.zip':
                if link not in self.logged_links:
                    logger.debug('Skipping link %s; macosx10 one', link)
                    self.logged_links.add(link)
                return
            if ext == wheel_ext:
                try:
                    wheel = Wheel(link.filename)
                except InvalidWheelFilename:
                    logger.debug(
                        'Skipping %s because the wheel filename is invalid',
                        link
                    )
                    return
                if (pkg_resources.safe_name(wheel.name).lower()
                        != pkg_resources.safe_name(search_name).lower()):
                    logger.debug(
                        'Skipping link %s; wrong project name (not %s)',
                        link,
                        search_name,
                    )
                    return
                if not wheel.supported():
                    logger.debug(
                        'Skipping %s because it is not compatible with this '
                        'Python',
                        link,
                    )
                    return
                # This is a dirty hack to prevent installing Binary Wheels from
                # PyPI unless it is a Windows or Mac Binary Wheel. This is
                # paired with a change to PyPI disabling uploads for the
                # same. Once we have a mechanism for enabling support for
                # binary wheels on linux that deals with the inherent problems
                # of binary distribution this can be removed.
                comes_from = getattr(link, "comes_from", None)
                if (
                        (
                            not platform.startswith('win')
                            and not platform.startswith('macosx')
                            and not platform == 'cli'
                        )
                        and comes_from is not None
                        and urllib_parse.urlparse(
                            comes_from.url
                        ).netloc.endswith(PyPI.netloc)):
                    if not wheel.supported(tags=supported_tags_noarch):
                        logger.debug(
                            "Skipping %s because it is a pypi-hosted binary "
                            "Wheel on an unsupported platform",
                            link,
                        )
                        return
                version = wheel.version

        if not version:
            version = self._egg_info_matches(egg_info, search_name, link)
        if version is None:
            logger.debug(
                'Skipping link %s; wrong project name (not %s)',
                link,
                search_name,
            )
            return

        if (link.internal is not None
                and not link.internal
                and not normalize_name(search_name).lower()
                in self.allow_external
                and not self.allow_all_external):
            # We have a link that we are sure is external, so we should skip
            #   it unless we are allowing externals
            logger.debug("Skipping %s because it is externally hosted.", link)
            self.need_warn_external = True
            return

        if (link.verifiable is not None
                and not link.verifiable
                and not (normalize_name(search_name).lower()
                         in self.allow_unverified)):
            # We have a link that we are sure we cannot verify its integrity,
            #   so we should skip it unless we are allowing unsafe installs
            #   for this requirement.
            logger.debug(
                "Skipping %s because it is an insecure and unverifiable file.",
                link,
            )
            self.need_warn_unverified = True
            return

        match = self._py_version_re.search(version)
        if match:
            version = version[:match.start()]
            py_version = match.group(1)
            if py_version != sys.version[:3]:
                logger.debug(
                    'Skipping %s because Python version is incorrect', link
                )
                return
        logger.debug('Found link %s, version: %s', link, version)

        return InstallationCandidate(search_name, version, link)

    def _egg_info_matches(self, egg_info, search_name, link):
        match = self._egg_info_re.search(egg_info)
        if not match:
            logger.debug('Could not parse version from link: %s', link)
            return None
        name = match.group(0).lower()
        # To match the "safe" name that pkg_resources creates:
        name = name.replace('_', '-')
        # project name and version must be separated by a dash
        look_for = search_name.lower() + "-"
        if name.startswith(look_for):
            return match.group(0)[len(look_for):]
        else:
            return None

    def _get_page(self, link, req):
        return HTMLPage.get_page(link, req, session=self.session)

class HTMLPage(object):
    """Represents one page, along with its URL"""

    # FIXME: these regexes are horrible hacks:
    _homepage_re = re.compile(b'<th>\\s*home\\s*page', re.I)
    _download_re = re.compile(b'<th>\\s*download\\s+url', re.I)
    _href_re = re.compile(
        b'href=(?:"([^"]*)"|\'([^\']*)\'|([^>\\s\\n]*))',
        re.I | re.S
    )

    def __init__(self, content, url, headers=None, trusted=None):
        # Determine if we have any encoding information in our headers
        encoding = None
        if headers and "Content-Type" in headers:
            content_type, params = cgi.parse_header(headers["Content-Type"])

            if "charset" in params:
                encoding = params['charset']

        self.content = content
        self.parsed = html5lib.parse(
            self.content,
            encoding=encoding,
            namespaceHTMLElements=False,
        )
        self.url = url
        self.headers = headers
        self.trusted = trusted

    def __str__(self):
        return self.url

    @classmethod
    def get_page(cls, link, req, skip_archives=True, session=None):
        if session is None:
            raise TypeError(
                "get_page() missing 1 required keyword argument: 'session'"
            )

        url = link.url
        url = url.split('#', 1)[0]

        # Check for VCS schemes that do not support lookup as web pages.
        from pip.vcs import VcsSupport
        for scheme in VcsSupport.schemes:
            if url.lower().startswith(scheme) and url[len(scheme)] in '+:':
                logger.debug('Cannot look at %s URL %s', scheme, link)
                return None

        try:
            if skip_archives:
                filename = link.filename
                for bad_ext in ['.tar', '.tar.gz', '.tar.bz2', '.tgz', '.zip']:
                    if filename.endswith(bad_ext):
                        content_type = cls._get_content_type(
                            url, session=session,
                        )
                        if content_type.lower().startswith('text/html'):
                            break
                        else:
                            logger.debug(
                                'Skipping page %s because of Content-Type: %s',
                                link,
                                content_type,
                            )
                            return

            logger.debug('Getting page %s', url)

            # Tack index.html onto file:// URLs that point to directories
            (scheme, netloc, path, params, query, fragment) = \
                urllib_parse.urlparse(url)
            if (scheme == 'file'
                    and os.path.isdir(urllib_request.url2pathname(path))):
                # add trailing slash if not present so urljoin doesn't trim
                # final segment
                if not url.endswith('/'):
                    url += '/'
                url = urllib_parse.urljoin(url, 'index.html')
                logger.debug(' file: URL is directory, getting %s', url)

            resp = session.get(
                url,
                headers={
                    "Accept": "text/html",
                    "Cache-Control": "max-age=600",
                },
            )
            resp.raise_for_status()

            # The check for archives above only works if the url ends with
            #   something that looks like an archive. However that is not a
            #   requirement of an url. Unless we issue a HEAD request on every
            #   url we cannot know ahead of time for sure if something is HTML
            #   or not. However we can check after we've downloaded it.
            content_type = resp.headers.get('Content-Type', 'unknown')
            if not content_type.lower().startswith("text/html"):
                logger.debug(
                    'Skipping page %s because of Content-Type: %s',
                    link,
                    content_type,
                )
                return

            inst = cls(
                resp.content, resp.url, resp.headers,
                trusted=link.trusted,
            )
        except requests.HTTPError as exc:
            level = 2 if exc.response.status_code == 404 else 1
            cls._handle_fail(req, link, exc, url, level=level)
        except requests.ConnectionError as exc:
            cls._handle_fail(
                req, link, "connection error: %s" % exc, url,
            )
        except requests.Timeout:
            cls._handle_fail(req, link, "timed out", url)
        except SSLError as exc:
            reason = ("There was a problem confirming the ssl certificate: "
                      "%s" % exc)
            cls._handle_fail(
                req, link, reason, url,
                level=2,
                meth=logger.info,
            )
        else:
            return inst

    @staticmethod
    def _handle_fail(req, link, reason, url, level=1, meth=None):
        if meth is None:
            meth = logger.debug

        meth("Could not fetch URL %s: %s", link, reason)
        meth("Will skip URL %s when looking for download links for %s" %
             (link.url, req))

    @staticmethod
    def _get_content_type(url, session):
        """Get the Content-Type of the given url, using a HEAD request"""
        scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url)
        if scheme not in ('http', 'https'):
            # FIXME: some warning or something?
            # assertion error?
            return ''

        resp = session.head(url, allow_redirects=True)
        resp.raise_for_status()

        return resp.headers.get("Content-Type", "")

    @cached_property
    def api_version(self):
        metas = [
            x for x in self.parsed.findall(".//meta")
            if x.get("name", "").lower() == "api-version"
        ]
        if metas:
            try:
                return int(metas[0].get("value", None))
            except (TypeError, ValueError):
                pass

        return None

    @cached_property
    def base_url(self):
        bases = [
            x for x in self.parsed.findall(".//base")
            if x.get("href") is not None
        ]
        if bases and bases[0].get("href"):
            return bases[0].get("href")
        else:
            return self.url

    @property
    def links(self):
        """Yields all links in the page"""
        for anchor in self.parsed.findall(".//a"):
            if anchor.get("href"):
                href = anchor.get("href")
                url = self.clean_link(
                    urllib_parse.urljoin(self.base_url, href)
                )

                # Determine if this link is internal. If that distinction
                #   doesn't make sense in this context, then we don't make
                #   any distinction.
                internal = None
                if self.api_version and self.api_version >= 2:
                    # Only api_versions >= 2 have a distinction between
                    #   external and internal links
                    internal = bool(
                        anchor.get("rel")
                        and "internal" in anchor.get("rel").split()
                    )

                yield Link(url, self, internal=internal)

    def rel_links(self):
        for url in self.explicit_rel_links():
            yield url
        for url in self.scraped_rel_links():
            yield url

    def explicit_rel_links(self, rels=('homepage', 'download')):
        """Yields all links with the given relations"""
        rels = set(rels)

        for anchor in self.parsed.findall(".//a"):
            if anchor.get("rel") and anchor.get("href"):
                found_rels = set(anchor.get("rel").split())
                # Determine the intersection between what rels were found and
                #   what rels were being looked for
                if found_rels & rels:
                    href = anchor.get("href")
                    url = self.clean_link(
                        urllib_parse.urljoin(self.base_url, href)
                    )
                    yield Link(url, self, trusted=False)

    def scraped_rel_links(self):
        # Can we get rid of this horrible horrible method?
        for regex in (self._homepage_re, self._download_re):
            match = regex.search(self.content)
            if not match:
                continue
            href_match = self._href_re.search(self.content, pos=match.end())
            if not href_match:
                continue
            url = (
                href_match.group(1)
                or href_match.group(2)
                or href_match.group(3)
            )
            if not url:
                continue
            try:
                url = url.decode("ascii")
            except UnicodeDecodeError:
                continue
            url = self.clean_link(urllib_parse.urljoin(self.base_url, url))
            yield Link(url, self, trusted=False, _deprecated_regex=True)

    _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I)

    def clean_link(self, url):
        """Makes sure a link is fully encoded.  That is, if a ' ' shows up in
        the link, it will be rewritten to %20 (while not over-quoting
        % or other characters)."""
        return self._clean_re.sub(
            lambda match: '%%%2x' % ord(match.group(0)), url)

class Link(object):

    def __init__(self, url, comes_from=None, internal=None, trusted=None,
                 _deprecated_regex=False):

        # url can be a UNC windows share
        if url != Inf and url.startswith('\\\\'):
            url = path_to_url(url)

        self.url = url
        self.comes_from = comes_from
        self.internal = internal
        self.trusted = trusted
        self._deprecated_regex = _deprecated_regex

    def __str__(self):
        if self.comes_from:
            return '%s (from %s)' % (self.url, self.comes_from)
        else:
            return str(self.url)

    def __repr__(self):
        return '<Link %s>' % self

    def __eq__(self, other):
        if not isinstance(other, Link):
            return NotImplemented
        return self.url == other.url

    def __ne__(self, other):
        if not isinstance(other, Link):
            return NotImplemented
        return self.url != other.url

    def __lt__(self, other):
        if not isinstance(other, Link):
            return NotImplemented
        return self.url < other.url

    def __le__(self, other):
        if not isinstance(other, Link):
            return NotImplemented
        return self.url <= other.url

    def __gt__(self, other):
        if not isinstance(other, Link):
            return NotImplemented
        return self.url > other.url

    def __ge__(self, other):
        if not isinstance(other, Link):
            return NotImplemented
        return self.url >= other.url

    def __hash__(self):
        return hash(self.url)

    @property
    def filename(self):
        _, netloc, path, _, _ = urllib_parse.urlsplit(self.url)
        name = posixpath.basename(path.rstrip('/')) or netloc
        name = urllib_parse.unquote(name)
        assert name, ('URL %r produced no filename' % self.url)
        return name

    @property
    def scheme(self):
        return urllib_parse.urlsplit(self.url)[0]

    @property
    def netloc(self):
        return urllib_parse.urlsplit(self.url)[1]

    @property
    def path(self):
        return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2])

    def splitext(self):
        return splitext(posixpath.basename(self.path.rstrip('/')))

    @property
    def ext(self):
        return self.splitext()[1]

    @property
    def url_without_fragment(self):
        scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url)
        return urllib_parse.urlunsplit((scheme, netloc, path, query, None))

    _egg_fragment_re = re.compile(r'#egg=([^&]*)')

    @property
    def egg_fragment(self):
        match = self._egg_fragment_re.search(self.url)
        if not match:
            return None
        return match.group(1)

    _hash_re = re.compile(
        r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)'
    )

    @property
    def hash(self):
        match = self._hash_re.search(self.url)
        if match:
            return match.group(2)
        return None

    @property
    def hash_name(self):
        match = self._hash_re.search(self.url)
        if match:
            return match.group(1)
        return None

    @property
    def show_url(self):
        return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0])

    @property
    def verifiable(self):
        """
        Returns True if this link can be verified after download, False if it
        cannot, and None if we cannot determine.
        """
        trusted = self.trusted or getattr(self.comes_from, "trusted", None)
        if trusted is not None and trusted:
            # This link came from a trusted source. It *may* be verifiable but
            #   first we need to see if this page is operating under the new
            #   API version.
            try:
                api_version = getattr(self.comes_from, "api_version", None)
                api_version = int(api_version)
            except (ValueError, TypeError):
                api_version = None

            if api_version is None or api_version <= 1:
                # This link is either trusted, or it came from a trusted,
                #   however it is not operating under the API version 2 so
                #   we can't make any claims about if it's safe or not
                return

            if self.hash:
                # This link came from a trusted source and it has a hash, so we
                #   can consider it safe.
                return True
            else:
                # This link came from a trusted source, using the new API
                #   version, and it does not have a hash. It is NOT verifiable
                return False
        elif trusted is not None:
            # This link came from an untrusted source and we cannot trust it
            return False

# An object to represent the "link" for the installed version of a requirement.
# Using Inf as the url makes it sort higher.
INSTALLED_VERSION = Link(Inf)


from django.conf import settings
from django.core.mail import EmailMultiAlternatives
from django.core.mail import EmailMessage

DEFAULT_FROM_EMAIL = getattr(settings, 'DEFAULT_FROM_EMAIL')

def prepare_email_subject(content):
    """ Email subject must be text and a single line """
    subject = ''.join(content.splitlines())
    return subject

def send_multi_alt_email(
    subject,  # single line with no line-breaks
    text_content,
    to_emails,
    html_content=None,
    from_email=DEFAULT_FROM_EMAIL,
    fail_silently=True
    ):
    """
    Send a message to one more email address(s).
    With text content as primary and html content as alternative.
    """
    messenger = EmailMultiAlternatives(subject, text_content, from_email, to_emails)
    if html_content:
        messenger.attach_alternative(html_content, "text/html")
    try:
        messenger.send()
    except Exception as e:
        if not fail_silently:
            raise

def send_html_email(
    subject,  # single line with no line-breaks
    html_content,
    to_emails,
    from_email=DEFAULT_FROM_EMAIL,
    fail_silently=True
    ):
    """
    Send a message to one more email address(s).
    With html content as primary.
    """
    messenger = EmailMessage(subject, html_content, from_email, to_emails)
    messenger.content_subtype = "html"  # Main content is now text/html
    try:
        messenger.send()
    except Exception as e:
        if not fail_silently:
            raise


import copy
import operator
from django import template
from django import forms
from django.utils.datastructures import SortedDict

register = template.Library()

class FieldSetNode(template.Node):
    """ Returns a subset of given form """

    def __init__(self, opcode, fields, orig_form, new_form):
        self.opcode = opcode
        self.fields = fields
        self.orig_form = orig_form
        self.new_form = new_form

    def render(self, context):
        oform = template.Variable(self.orig_form).resolve(context)
        nform = copy.copy(oform)
        if "fields" in self.opcode:
            nform.fields = SortedDict([(key, oform.fields[key]) for key in oform.fields if key in self.fields])
        if "exclude" in self.opcode:
            nform.fields = SortedDict([(key, oform.fields[key]) for key in oform.fields if key not in self.fields])
        context[self.new_form] = nform
        return ''

@register.tag
def trim_form(parser, token):
    """
        Returns a form that only contains a subset of the original fields (opcode: incude/exclude fields)
        Exampel:
            <fieldset>
                <legend>Business Info</legend>

                {% trim_form orig_form fields biz_name,biz_city,biz_email,biz_phone as new_form %}
                {{ new_form.as_ul }}

            </fieldset>
            OR:
            <fieldset>
                <legend>Business Info</legend>

                {% trim_form orig_form exclude biz_country,biz_url as new_form %}
                {{ new_form.as_ul }}

            </fieldset>
    """
    try:
        trim_form, orig_form, opcode, fields, as_, new_form = token.split_contents()
    except ValueError:
        raise template.TemplateSyntaxError('Invalid arguments for %r' % token.split_contents()[0])

    return FieldSetNode(opcode, fields.split(','), orig_form, new_form)


"""Command-line support for Coverage."""

import optparse, os, sys, time, traceback

from coverage.backward import sorted                # pylint: disable=W0622
from coverage.execfile import run_python_file, run_python_module
from coverage.misc import CoverageException, ExceptionDuringRun, NoSource
from coverage.debug import info_formatter

class Opts(object):
    """A namespace class for individual options we'll build parsers from."""

    append = optparse.make_option(
        '-a', '--append', action='store_false', dest="erase_first",
        help="Append coverage data to .coverage, otherwise it is started "
                "clean with each run."
        )
    branch = optparse.make_option(
        '', '--branch', action='store_true',
        help="Measure branch coverage in addition to statement coverage."
        )
    debug = optparse.make_option(
        '', '--debug', action='store', metavar="OPTS",
        help="Debug options, separated by commas"
        )
    directory = optparse.make_option(
        '-d', '--directory', action='store', metavar="DIR",
        help="Write the output files to DIR."
        )
    fail_under = optparse.make_option(
        '', '--fail-under', action='store', metavar="MIN", type="int",
        help="Exit with a status of 2 if the total coverage is less than MIN."
        )
    help = optparse.make_option(
        '-h', '--help', action='store_true',
        help="Get help on this command."
        )
    ignore_errors = optparse.make_option(
        '-i', '--ignore-errors', action='store_true',
        help="Ignore errors while reading source files."
        )
    include = optparse.make_option(
        '', '--include', action='store',
        metavar="PAT1,PAT2,...",
        help="Include files only when their filename path matches one of "
                "these patterns.  Usually needs quoting on the command line."
        )
    pylib = optparse.make_option(
        '-L', '--pylib', action='store_true',
        help="Measure coverage even inside the Python installed library, "
                "which isn't done by default."
        )
    show_missing = optparse.make_option(
        '-m', '--show-missing', action='store_true',
        help="Show line numbers of statements in each module that weren't "
                "executed."
        )
    old_omit = optparse.make_option(
        '-o', '--omit', action='store',
        metavar="PAT1,PAT2,...",
        help="Omit files when their filename matches one of these patterns. "
                "Usually needs quoting on the command line."
        )
    omit = optparse.make_option(
        '', '--omit', action='store',
        metavar="PAT1,PAT2,...",
        help="Omit files when their filename matches one of these patterns. "
                "Usually needs quoting on the command line."
        )
    output_xml = optparse.make_option(
        '-o', '', action='store', dest="outfile",
        metavar="OUTFILE",
        help="Write the XML report to this file. Defaults to 'coverage.xml'"
        )
    parallel_mode = optparse.make_option(
        '-p', '--parallel-mode', action='store_true',
        help="Append the machine name, process id and random number to the "
                ".coverage data file name to simplify collecting data from "
                "many processes."
        )
    module = optparse.make_option(
        '-m', '--module', action='store_true',
        help="<pyfile> is an importable Python module, not a script path, "
                "to be run as 'python -m' would run it."
        )
    rcfile = optparse.make_option(
        '', '--rcfile', action='store',
        help="Specify configuration file.  Defaults to '.coveragerc'"
        )
    source = optparse.make_option(
        '', '--source', action='store', metavar="SRC1,SRC2,...",
        help="A list of packages or directories of code to be measured."
        )
    timid = optparse.make_option(
        '', '--timid', action='store_true',
        help="Use a simpler but slower trace method.  Try this if you get "
                "seemingly impossible results!"
        )
    title = optparse.make_option(
        '', '--title', action='store', metavar="TITLE",
        help="A text string to use as the title on the HTML."
        )
    version = optparse.make_option(
        '', '--version', action='store_true',
        help="Display version information and exit."
        )

class CoverageOptionParser(optparse.OptionParser, object):
    """Base OptionParser for coverage.

    Problems don't exit the program.
    Defaults are initialized for all options.

    """

    def __init__(self, *args, **kwargs):
        super(CoverageOptionParser, self).__init__(
            add_help_option=False, *args, **kwargs
            )
        self.set_defaults(
            actions=[],
            branch=None,
            debug=None,
            directory=None,
            fail_under=None,
            help=None,
            ignore_errors=None,
            include=None,
            omit=None,
            parallel_mode=None,
            module=None,
            pylib=None,
            rcfile=True,
            show_missing=None,
            source=None,
            timid=None,
            title=None,
            erase_first=None,
            version=None,
            )

        self.disable_interspersed_args()
        self.help_fn = self.help_noop

    def help_noop(self, error=None, topic=None, parser=None):
        """No-op help function."""
        pass

    class OptionParserError(Exception):
        """Used to stop the optparse error handler ending the process."""
        pass

    def parse_args(self, args=None, options=None):
        """Call optparse.parse_args, but return a triple:

        (ok, options, args)

        """
        try:
            options, args = \
                super(CoverageOptionParser, self).parse_args(args, options)
        except self.OptionParserError:
            return False, None, None
        return True, options, args

    def error(self, msg):
        """Override optparse.error so sys.exit doesn't get called."""
        self.help_fn(msg)
        raise self.OptionParserError

class ClassicOptionParser(CoverageOptionParser):
    """Command-line parser for coverage.py classic arguments."""

    def __init__(self):
        super(ClassicOptionParser, self).__init__()

        self.add_action('-a', '--annotate', 'annotate')
        self.add_action('-b', '--html', 'html')
        self.add_action('-c', '--combine', 'combine')
        self.add_action('-e', '--erase', 'erase')
        self.add_action('-r', '--report', 'report')
        self.add_action('-x', '--execute', 'execute')

        self.add_options([
            Opts.directory,
            Opts.help,
            Opts.ignore_errors,
            Opts.pylib,
            Opts.show_missing,
            Opts.old_omit,
            Opts.parallel_mode,
            Opts.timid,
            Opts.version,
        ])

    def add_action(self, dash, dashdash, action_code):
        """Add a specialized option that is the action to execute."""
        option = self.add_option(dash, dashdash, action='callback',
            callback=self._append_action
            )
        option.action_code = action_code

    def _append_action(self, option, opt_unused, value_unused, parser):
        """Callback for an option that adds to the `actions` list."""
        parser.values.actions.append(option.action_code)

class CmdOptionParser(CoverageOptionParser):
    """Parse one of the new-style commands for coverage.py."""

    def __init__(self, action, options=None, defaults=None, usage=None,
                cmd=None, description=None
                ):
        """Create an OptionParser for a coverage command.

        `action` is the slug to put into `options.actions`.
        `options` is a list of Option's for the command.
        `defaults` is a dict of default value for options.
        `usage` is the usage string to display in help.
        `cmd` is the command name, if different than `action`.
        `description` is the description of the command, for the help text.

        """
        if usage:
            usage = "%prog " + usage
        super(CmdOptionParser, self).__init__(
            prog="coverage %s" % (cmd or action),
            usage=usage,
            description=description,
        )
        self.set_defaults(actions=[action], **(defaults or {}))
        if options:
            self.add_options(options)
        self.cmd = cmd or action

    def __eq__(self, other):
        # A convenience equality, so that I can put strings in unit test
        # results, and they will compare equal to objects.
        return (other == "<CmdOptionParser:%s>" % self.cmd)

GLOBAL_ARGS = [
    Opts.rcfile,
    Opts.help,
    ]

CMDS = {
    'annotate': CmdOptionParser("annotate",
        [
            Opts.directory,
            Opts.ignore_errors,
            Opts.omit,
            Opts.include,
            ] + GLOBAL_ARGS,
        usage = "[options] [modules]",
        description = "Make annotated copies of the given files, marking "
            "statements that are executed with > and statements that are "
            "missed with !."
        ),

    'combine': CmdOptionParser("combine", GLOBAL_ARGS,
        usage = " ",
        description = "Combine data from multiple coverage files collected "
            "with 'run -p'.  The combined results are written to a single "
            "file representing the union of the data."
        ),

    'debug': CmdOptionParser("debug", GLOBAL_ARGS,
        usage = "<topic>",
        description = "Display information on the internals of coverage.py, "
            "for diagnosing problems. "
            "Topics are 'data' to show a summary of the collected data, "
            "or 'sys' to show installation information."
        ),

    'erase': CmdOptionParser("erase", GLOBAL_ARGS,
        usage = " ",
        description = "Erase previously collected coverage data."
        ),

    'help': CmdOptionParser("help", GLOBAL_ARGS,
        usage = "[command]",
        description = "Describe how to use coverage.py"
        ),

    'html': CmdOptionParser("html",
        [
            Opts.directory,
            Opts.fail_under,
            Opts.ignore_errors,
            Opts.omit,
            Opts.include,
            Opts.title,
            ] + GLOBAL_ARGS,
        usage = "[options] [modules]",
        description = "Create an HTML report of the coverage of the files.  "
            "Each file gets its own page, with the source decorated to show "
            "executed, excluded, and missed lines."
        ),

    'report': CmdOptionParser("report",
        [
            Opts.fail_under,
            Opts.ignore_errors,
            Opts.omit,
            Opts.include,
            Opts.show_missing,
            ] + GLOBAL_ARGS,
        usage = "[options] [modules]",
        description = "Report coverage statistics on modules."
        ),

    'run': CmdOptionParser("execute",
        [
            Opts.append,
            Opts.branch,
            Opts.debug,
            Opts.pylib,
            Opts.parallel_mode,
            Opts.module,
            Opts.timid,
            Opts.source,
            Opts.omit,
            Opts.include,
            ] + GLOBAL_ARGS,
        defaults = {'erase_first': True},
        cmd = "run",
        usage = "[options] <pyfile> [program options]",
        description = "Run a Python program, measuring code execution."
        ),

    'xml': CmdOptionParser("xml",
        [
            Opts.fail_under,
            Opts.ignore_errors,
            Opts.omit,
            Opts.include,
            Opts.output_xml,
            ] + GLOBAL_ARGS,
        cmd = "xml",
        usage = "[options] [modules]",
        description = "Generate an XML report of coverage results."
        ),
    }

OK, ERR, FAIL_UNDER = 0, 1, 2

class CoverageScript(object):
    """The command-line interface to Coverage."""

    def __init__(self, _covpkg=None, _run_python_file=None,
                 _run_python_module=None, _help_fn=None):
        # _covpkg is for dependency injection, so we can test this code.
        if _covpkg:
            self.covpkg = _covpkg
        else:
            import coverage
            self.covpkg = coverage

        # For dependency injection:
        self.run_python_file = _run_python_file or run_python_file
        self.run_python_module = _run_python_module or run_python_module
        self.help_fn = _help_fn or self.help
        self.classic = False

        self.coverage = None

    def command_line(self, argv):
        """The bulk of the command line interface to Coverage.

        `argv` is the argument list to process.

        Returns 0 if all is well, 1 if something went wrong.

        """
        # Collect the command-line options.
        if not argv:
            self.help_fn(topic='minimum_help')
            return OK

        # The command syntax we parse depends on the first argument.  Classic
        # syntax always starts with an option.
        self.classic = argv[0].startswith('-')
        if self.classic:
            parser = ClassicOptionParser()
        else:
            parser = CMDS.get(argv[0])
            if not parser:
                self.help_fn("Unknown command: '%s'" % argv[0])
                return ERR
            argv = argv[1:]

        parser.help_fn = self.help_fn
        ok, options, args = parser.parse_args(argv)
        if not ok:
            return ERR

        # Handle help and version.
        if self.do_help(options, args, parser):
            return OK

        # Check for conflicts and problems in the options.
        if not self.args_ok(options, args):
            return ERR

        # Listify the list options.
        source = unshell_list(options.source)
        omit = unshell_list(options.omit)
        include = unshell_list(options.include)
        debug = unshell_list(options.debug)

        # Do something.
        self.coverage = self.covpkg.coverage(
            data_suffix = options.parallel_mode,
            cover_pylib = options.pylib,
            timid = options.timid,
            branch = options.branch,
            config_file = options.rcfile,
            source = source,
            omit = omit,
            include = include,
            debug = debug,
            )

        if 'debug' in options.actions:
            return self.do_debug(args)

        if 'erase' in options.actions or options.erase_first:
            self.coverage.erase()
        else:
            self.coverage.load()

        if 'execute' in options.actions:
            self.do_execute(options, args)

        if 'combine' in options.actions:
            self.coverage.combine()
            self.coverage.save()

        # Remaining actions are reporting, with some common options.
        report_args = dict(
            morfs = args,
            ignore_errors = options.ignore_errors,
            omit = omit,
            include = include,
            )

        if 'report' in options.actions:
            total = self.coverage.report(
                show_missing=options.show_missing, **report_args)
        if 'annotate' in options.actions:
            self.coverage.annotate(
                directory=options.directory, **report_args)
        if 'html' in options.actions:
            total = self.coverage.html_report(
                directory=options.directory, title=options.title,
                **report_args)
        if 'xml' in options.actions:
            outfile = options.outfile
            total = self.coverage.xml_report(outfile=outfile, **report_args)

        if options.fail_under is not None:
            if total >= options.fail_under:
                return OK
            else:
                return FAIL_UNDER
        else:
            return OK

    def help(self, error=None, topic=None, parser=None):
        """Display an error message, or the named topic."""
        assert error or topic or parser
        if error:
            print(error)
            print("Use 'coverage help' for help.")
        elif parser:
            print(parser.format_help().strip())
        else:
            help_msg = HELP_TOPICS.get(topic, '').strip()
            if help_msg:
                print(help_msg % self.covpkg.__dict__)
            else:
                print("Don't know topic %r" % topic)

    def do_help(self, options, args, parser):
        """Deal with help requests.

        Return True if it handled the request, False if not.

        """
        # Handle help.
        if options.help:
            if self.classic:
                self.help_fn(topic='help')
            else:
                self.help_fn(parser=parser)
            return True

        if "help" in options.actions:
            if args:
                for a in args:
                    parser = CMDS.get(a)
                    if parser:
                        self.help_fn(parser=parser)
                    else:
                        self.help_fn(topic=a)
            else:
                self.help_fn(topic='help')
            return True

        # Handle version.
        if options.version:
            self.help_fn(topic='version')
            return True

        return False

    def args_ok(self, options, args):
        """Check for conflicts and problems in the options.

        Returns True if everything is ok, or False if not.

        """
        for i in ['erase', 'execute']:
            for j in ['annotate', 'html', 'report', 'combine']:
                if (i in options.actions) and (j in options.actions):
                    self.help_fn("You can't specify the '%s' and '%s' "
                              "options at the same time." % (i, j))
                    return False

        if not options.actions:
            self.help_fn(
                "You must specify at least one of -e, -x, -c, -r, -a, or -b."
                )
            return False
        args_allowed = (
            'execute' in options.actions or
            'annotate' in options.actions or
            'html' in options.actions or
            'debug' in options.actions or
            'report' in options.actions or
            'xml' in options.actions
            )
        if not args_allowed and args:
            self.help_fn("Unexpected arguments: %s" % " ".join(args))
            return False

        if 'execute' in options.actions and not args:
            self.help_fn("Nothing to do.")
            return False

        return True

    def do_execute(self, options, args):
        """Implementation of 'coverage run'."""

        # Set the first path element properly.
        old_path0 = sys.path[0]

        # Run the script.
        self.coverage.start()
        code_ran = True
        try:
            try:
                if options.module:
                    sys.path[0] = ''
                    self.run_python_module(args[0], args)
                else:
                    filename = args[0]
                    sys.path[0] = os.path.abspath(os.path.dirname(filename))
                    self.run_python_file(filename, args)
            except NoSource:
                code_ran = False
                raise
        finally:
            self.coverage.stop()
            if code_ran:
                self.coverage.save()

            # Restore the old path
            sys.path[0] = old_path0

    def do_debug(self, args):
        """Implementation of 'coverage debug'."""

        if not args:
            self.help_fn("What information would you like: data, sys?")
            return ERR
        for info in args:
            if info == 'sys':
                print("-- sys ----------------------------------------")
                for line in info_formatter(self.coverage.sysinfo()):
                    print(" %s" % line)
            elif info == 'data':
                print("-- data ---------------------------------------")
                self.coverage.load()
                print("path: %s" % self.coverage.data.filename)
                print("has_arcs: %r" % self.coverage.data.has_arcs())
                summary = self.coverage.data.summary(fullpath=True)
                if summary:
                    filenames = sorted(summary.keys())
                    print("\n%d files:" % len(filenames))
                    for f in filenames:
                        print("%s: %d lines" % (f, summary[f]))
                else:
                    print("No data collected")
            else:
                self.help_fn("Don't know what you mean by %r" % info)
                return ERR
        return OK

def unshell_list(s):
    """Turn a command-line argument into a list."""
    if not s:
        return None
    if sys.platform == 'win32':
        # When running coverage as coverage.exe, some of the behavior
        # of the shell is emulated: wildcards are expanded into a list of
        # filenames.  So you have to single-quote patterns on the command
        # line, but (not) helpfully, the single quotes are included in the
        # argument, so we have to strip them off here.
        s = s.strip("'")
    return s.split(',')

HELP_TOPICS = {
# -------------------------
'classic':
r"""Coverage.py version %(__version__)s
Measure, collect, and report on code coverage in Python programs.

Usage:

coverage -x [-p] [-L] [--timid] MODULE.py [ARG1 ARG2 ...]
    Execute the module, passing the given command-line arguments, collecting
    coverage data.  With the -p option, include the machine name and process
    id in the .coverage file name.  With -L, measure coverage even inside the
    Python installed library, which isn't done by default.  With --timid, use a
    simpler but slower trace method.

coverage -e
    Erase collected coverage data.

coverage -c
    Combine data from multiple coverage files (as created by -p option above)
    and store it into a single file representing the union of the coverage.

coverage -r [-m] [-i] [-o DIR,...] [FILE1 FILE2 ...]
    Report on the statement coverage for the given files.  With the -m
    option, show line numbers of the statements that weren't executed.

coverage -b -d DIR [-i] [-o DIR,...] [FILE1 FILE2 ...]
    Create an HTML report of the coverage of the given files.  Each file gets
    its own page, with the file listing decorated to show executed, excluded,
    and missed lines.

coverage -a [-d DIR] [-i] [-o DIR,...] [FILE1 FILE2 ...]
    Make annotated copies of the given files, marking statements that
    are executed with > and statements that are missed with !.

-d DIR
    Write output files for -b or -a to this directory.

-i  Ignore errors while reporting or annotating.

-o DIR,...
    Omit reporting or annotating files when their filename path starts with
    a directory listed in the omit list.
    e.g. coverage -i -r -o c:\python25,lib\enthought\traits

Coverage data is saved in the file .coverage by default.  Set the
COVERAGE_FILE environment variable to save it somewhere else.
""",
# -------------------------
'help': """\
Coverage.py, version %(__version__)s
Measure, collect, and report on code coverage in Python programs.

usage: coverage <command> [options] [args]

Commands:
    annotate    Annotate source files with execution information.
    combine     Combine a number of data files.
    erase       Erase previously collected coverage data.
    help        Get help on using coverage.py.
    html        Create an HTML report.
    report      Report coverage stats on modules.
    run         Run a Python program and measure code execution.
    xml         Create an XML report of coverage results.

Use "coverage help <command>" for detailed help on any command.
Use "coverage help classic" for help on older command syntax.
For more information, see %(__url__)s
""",
# -------------------------
'minimum_help': """\
Code coverage for Python.  Use 'coverage help' for help.
""",
# -------------------------
'version': """\
Coverage.py, version %(__version__)s.  %(__url__)s
""",
}

def main(argv=None):
    """The main entry point to Coverage.

    This is installed as the script entry point.

    """
    if argv is None:
        argv = sys.argv[1:]
    try:
        start = time.clock()
        status = CoverageScript().command_line(argv)
        end = time.clock()
        if 0:
            print("time: %.3fs" % (end - start))
    except ExceptionDuringRun:
        # An exception was caught while running the product code.  The
        # sys.exc_info() return tuple is packed into an ExceptionDuringRun
        # exception.
        _, err, _ = sys.exc_info()
        traceback.print_exception(*err.args)
        status = ERR
    except CoverageException:
        # A controlled error inside coverage.py: print the message to the user.
        _, err, _ = sys.exc_info()
        print(err)
        status = ERR
    except SystemExit:
        # The user called `sys.exit()`.  Exit with their argument, if any.
        _, err, _ = sys.exc_info()
        if err.args:
            status = err.args[0]
        else:
            status = None
    return status


"""serialization utilities for apply messages

Authors:

* Min RK
"""
#-----------------------------------------------------------------------------
#  Copyright (C) 2010-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

# Standard library imports
import logging
import os
import re
import socket
import sys

try:
    import cPickle
    pickle = cPickle
except:
    cPickle = None
    import pickle

# IPython imports
from IPython.utils import py3compat
from IPython.utils.pickleutil import can, uncan, canSequence, uncanSequence
from IPython.utils.newserialized import serialize, unserialize

if py3compat.PY3:
    buffer = memoryview

#-----------------------------------------------------------------------------
# Serialization Functions
#-----------------------------------------------------------------------------

def serialize_object(obj, threshold=64e-6):
    """Serialize an object into a list of sendable buffers.

    Parameters
    ----------

    obj : object
        The object to be serialized
    threshold : float
        The threshold for not double-pickling the content.

    Returns
    -------
    ('pmd', [bufs]) :
        where pmd is the pickled metadata wrapper,
        bufs is a list of data buffers
    """
    databuffers = []
    if isinstance(obj, (list, tuple)):
        clist = canSequence(obj)
        slist = map(serialize, clist)
        for s in slist:
            if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:
                databuffers.append(s.getData())
                s.data = None
        return pickle.dumps(slist,-1), databuffers
    elif isinstance(obj, dict):
        sobj = {}
        for k in sorted(obj.iterkeys()):
            s = serialize(can(obj[k]))
            if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:
                databuffers.append(s.getData())
                s.data = None
            sobj[k] = s
        return pickle.dumps(sobj,-1),databuffers
    else:
        s = serialize(can(obj))
        if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold:
            databuffers.append(s.getData())
            s.data = None
        return pickle.dumps(s,-1),databuffers

def unserialize_object(bufs):
    """reconstruct an object serialized by serialize_object from data buffers."""
    bufs = list(bufs)
    sobj = pickle.loads(bufs.pop(0))
    if isinstance(sobj, (list, tuple)):
        for s in sobj:
            if s.data is None:
                s.data = bufs.pop(0)
        return uncanSequence(map(unserialize, sobj)), bufs
    elif isinstance(sobj, dict):
        newobj = {}
        for k in sorted(sobj.iterkeys()):
            s = sobj[k]
            if s.data is None:
                s.data = bufs.pop(0)
            newobj[k] = uncan(unserialize(s))
        return newobj, bufs
    else:
        if sobj.data is None:
            sobj.data = bufs.pop(0)
        return uncan(unserialize(sobj)), bufs

def pack_apply_message(f, args, kwargs, threshold=64e-6):
    """pack up a function, args, and kwargs to be sent over the wire
    as a series of buffers. Any object whose data is larger than `threshold`
    will not have their data copied (currently only numpy arrays support zero-copy)"""
    msg = [pickle.dumps(can(f),-1)]
    databuffers = [] # for large objects
    sargs, bufs = serialize_object(args,threshold)
    msg.append(sargs)
    databuffers.extend(bufs)
    skwargs, bufs = serialize_object(kwargs,threshold)
    msg.append(skwargs)
    databuffers.extend(bufs)
    msg.extend(databuffers)
    return msg

def unpack_apply_message(bufs, g=None, copy=True):
    """unpack f,args,kwargs from buffers packed by pack_apply_message()
    Returns: original f,args,kwargs"""
    bufs = list(bufs) # allow us to pop
    assert len(bufs) >= 3, "not enough buffers!"
    if not copy:
        for i in range(3):
            bufs[i] = bufs[i].bytes
    cf = pickle.loads(bufs.pop(0))
    sargs = list(pickle.loads(bufs.pop(0)))
    skwargs = dict(pickle.loads(bufs.pop(0)))
    # print sargs, skwargs
    f = uncan(cf, g)
    for sa in sargs:
        if sa.data is None:
            m = bufs.pop(0)
            if sa.getTypeDescriptor() in ('buffer', 'ndarray'):
                # always use a buffer, until memoryviews get sorted out
                sa.data = buffer(m)
                # disable memoryview support
                # if copy:
                #     sa.data = buffer(m)
                # else:
                #     sa.data = m.buffer
            else:
                if copy:
                    sa.data = m
                else:
                    sa.data = m.bytes

    args = uncanSequence(map(unserialize, sargs), g)
    kwargs = {}
    for k in sorted(skwargs.iterkeys()):
        sa = skwargs[k]
        if sa.data is None:
            m = bufs.pop(0)
            if sa.getTypeDescriptor() in ('buffer', 'ndarray'):
                # always use a buffer, until memoryviews get sorted out
                sa.data = buffer(m)
                # disable memoryview support
                # if copy:
                #     sa.data = buffer(m)
                # else:
                #     sa.data = m.buffer
            else:
                if copy:
                    sa.data = m
                else:
                    sa.data = m.bytes

        kwargs[k] = uncan(unserialize(sa), g)

    return f,args,kwargs


# encoding: utf-8
"""
A context manager for handling sys.displayhook.

Authors:

* Robert Kern
* Brian Granger
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import sys

from IPython.config.configurable import Configurable
from IPython.utils.traitlets import Any

#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------

class DisplayTrap(Configurable):
    """Object to manage sys.displayhook.

    This came from IPython.core.kernel.display_hook, but is simplified
    (no callbacks or formatters) until more of the core is refactored.
    """

    hook = Any

    def __init__(self, hook=None):
        super(DisplayTrap, self).__init__(hook=hook, config=None)
        self.old_hook = None
        # We define this to track if a single BuiltinTrap is nested.
        # Only turn off the trap when the outermost call to __exit__ is made.
        self._nested_level = 0

    def __enter__(self):
        if self._nested_level == 0:
            self.set()
        self._nested_level += 1
        return self

    def __exit__(self, type, value, traceback):
        if self._nested_level == 1:
            self.unset()
        self._nested_level -= 1
        # Returning False will cause exceptions to propagate
        return False

    def set(self):
        """Set the hook."""
        if sys.displayhook is not self.hook:
            self.old_hook = sys.displayhook
            sys.displayhook = self.hook

    def unset(self):
        """Unset the hook."""
        sys.displayhook = self.old_hook


"""some generic utilities for dealing with classes, urls, and serialization

Authors:

* Min RK
"""
#-----------------------------------------------------------------------------
#  Copyright (C) 2010-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

# Standard library imports.
import logging
import os
import re
import stat
import socket
import sys
from signal import signal, SIGINT, SIGABRT, SIGTERM
try:
    from signal import SIGKILL
except ImportError:
    SIGKILL=None

try:
    import cPickle
    pickle = cPickle
except:
    cPickle = None
    import pickle

# System library imports
import zmq
from zmq.log import handlers

from IPython.external.decorator import decorator

# IPython imports
from IPython.config.application import Application
from IPython.utils import py3compat
from IPython.utils.pickleutil import can, uncan, canSequence, uncanSequence
from IPython.utils.newserialized import serialize, unserialize
from IPython.zmq.log import EnginePUBHandler
from IPython.zmq.serialize import (
    unserialize_object, serialize_object, pack_apply_message, unpack_apply_message
)

if py3compat.PY3:
    buffer = memoryview

#-----------------------------------------------------------------------------
# Classes
#-----------------------------------------------------------------------------

class Namespace(dict):
    """Subclass of dict for attribute access to keys."""

    def __getattr__(self, key):
        """getattr aliased to getitem"""
        if key in self.iterkeys():
            return self[key]
        else:
            raise NameError(key)

    def __setattr__(self, key, value):
        """setattr aliased to setitem, with strict"""
        if hasattr(dict, key):
            raise KeyError("Cannot override dict keys %r"%key)
        self[key] = value

class ReverseDict(dict):
    """simple double-keyed subset of dict methods."""

    def __init__(self, *args, **kwargs):
        dict.__init__(self, *args, **kwargs)
        self._reverse = dict()
        for key, value in self.iteritems():
            self._reverse[value] = key

    def __getitem__(self, key):
        try:
            return dict.__getitem__(self, key)
        except KeyError:
            return self._reverse[key]

    def __setitem__(self, key, value):
        if key in self._reverse:
            raise KeyError("Can't have key %r on both sides!"%key)
        dict.__setitem__(self, key, value)
        self._reverse[value] = key

    def pop(self, key):
        value = dict.pop(self, key)
        self._reverse.pop(value)
        return value

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

#-----------------------------------------------------------------------------
# Functions
#-----------------------------------------------------------------------------

@decorator
def log_errors(f, self, *args, **kwargs):
    """decorator to log unhandled exceptions raised in a method.

    For use wrapping on_recv callbacks, so that exceptions
    do not cause the stream to be closed.
    """
    try:
        return f(self, *args, **kwargs)
    except Exception:
        self.log.error("Uncaught exception in %r" % f, exc_info=True)

def is_url(url):
    """boolean check for whether a string is a zmq url"""
    if '://' not in url:
        return False
    proto, addr = url.split('://', 1)
    if proto.lower() not in ['tcp','pgm','epgm','ipc','inproc']:
        return False
    return True

def validate_url(url):
    """validate a url for zeromq"""
    if not isinstance(url, basestring):
        raise TypeError("url must be a string, not %r"%type(url))
    url = url.lower()

    proto_addr = url.split('://')
    assert len(proto_addr) == 2, 'Invalid url: %r'%url
    proto, addr = proto_addr
    assert proto in ['tcp','pgm','epgm','ipc','inproc'], "Invalid protocol: %r"%proto

    # domain pattern adapted from http://www.regexlib.com/REDetails.aspx?regexp_id=391
    # author: Remi Sabourin
    pat = re.compile(r'^([\w\d]([\w\d\-]{0,61}[\w\d])?\.)*[\w\d]([\w\d\-]{0,61}[\w\d])?$')

    if proto == 'tcp':
        lis = addr.split(':')
        assert len(lis) == 2, 'Invalid url: %r'%url
        addr,s_port = lis
        try:
            port = int(s_port)
        except ValueError:
            raise AssertionError("Invalid port %r in url: %r"%(port, url))

        assert addr == '*' or pat.match(addr) is not None, 'Invalid url: %r'%url

    else:
        # only validate tcp urls currently
        pass

    return True

def validate_url_container(container):
    """validate a potentially nested collection of urls."""
    if isinstance(container, basestring):
        url = container
        return validate_url(url)
    elif isinstance(container, dict):
        container = container.itervalues()

    for element in container:
        validate_url_container(element)

def split_url(url):
    """split a zmq url (tcp://ip:port) into ('tcp','ip','port')."""
    proto_addr = url.split('://')
    assert len(proto_addr) == 2, 'Invalid url: %r'%url
    proto, addr = proto_addr
    lis = addr.split(':')
    assert len(lis) == 2, 'Invalid url: %r'%url
    addr,s_port = lis
    return proto,addr,s_port

def disambiguate_ip_address(ip, location=None):
    """turn multi-ip interfaces '0.0.0.0' and '*' into connectable
    ones, based on the location (default interpretation of location is localhost)."""
    if ip in ('0.0.0.0', '*'):
        try:
            external_ips = socket.gethostbyname_ex(socket.gethostname())[2]
        except (socket.gaierror, IndexError):
            # couldn't identify this machine, assume localhost
            external_ips = []
        if location is None or location in external_ips or not external_ips:
            # If location is unspecified or cannot be determined, assume local
            ip='127.0.0.1'
        elif location:
            return location
    return ip

def disambiguate_url(url, location=None):
    """turn multi-ip interfaces '0.0.0.0' and '*' into connectable
    ones, based on the location (default interpretation is localhost).

    This is for zeromq urls, such as tcp://*:10101."""
    try:
        proto,ip,port = split_url(url)
    except AssertionError:
        # probably not tcp url; could be ipc, etc.
        return url

    ip = disambiguate_ip_address(ip,location)

    return "%s://%s:%s"%(proto,ip,port)

#--------------------------------------------------------------------------
# helpers for implementing old MEC API via view.apply
#--------------------------------------------------------------------------

def interactive(f):
    """decorator for making functions appear as interactively defined.
    This results in the function being linked to the user_ns as globals()
    instead of the module globals().
    """
    f.__module__ = '__main__'
    return f

@interactive
def _push(**ns):
    """helper method for implementing `client.push` via `client.apply`"""
    globals().update(ns)

@interactive
def _pull(keys):
    """helper method for implementing `client.pull` via `client.apply`"""
    user_ns = globals()
    if isinstance(keys, (list,tuple, set)):
        for key in keys:
            if not user_ns.has_key(key):
                raise NameError("name '%s' is not defined"%key)
        return map(user_ns.get, keys)
    else:
        if not user_ns.has_key(keys):
            raise NameError("name '%s' is not defined"%keys)
        return user_ns.get(keys)

@interactive
def _execute(code):
    """helper method for implementing `client.execute` via `client.apply`"""
    exec code in globals()

#--------------------------------------------------------------------------
# extra process management utilities
#--------------------------------------------------------------------------

_random_ports = set()

def select_random_ports(n):
    """Selects and return n random ports that are available."""
    ports = []
    for i in xrange(n):
        sock = socket.socket()
        sock.bind(('', 0))
        while sock.getsockname()[1] in _random_ports:
            sock.close()
            sock = socket.socket()
            sock.bind(('', 0))
        ports.append(sock)
    for i, sock in enumerate(ports):
        port = sock.getsockname()[1]
        sock.close()
        ports[i] = port
        _random_ports.add(port)
    return ports

def signal_children(children):
    """Relay interupt/term signals to children, for more solid process cleanup."""
    def terminate_children(sig, frame):
        log = Application.instance().log
        log.critical("Got signal %i, terminating children..."%sig)
        for child in children:
            child.terminate()

        sys.exit(sig != SIGINT)
        # sys.exit(sig)
    for sig in (SIGINT, SIGABRT, SIGTERM):
        signal(sig, terminate_children)

def generate_exec_key(keyfile):
    import uuid
    newkey = str(uuid.uuid4())
    with open(keyfile, 'w') as f:
        # f.write('ipython-key ')
        f.write(newkey+'\n')
    # set user-only RW permissions (0600)
    # this will have no effect on Windows
    os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)

def integer_loglevel(loglevel):
    try:
        loglevel = int(loglevel)
    except ValueError:
        if isinstance(loglevel, str):
            loglevel = getattr(logging, loglevel)
    return loglevel

def connect_logger(logname, context, iface, root="ip", loglevel=logging.DEBUG):
    logger = logging.getLogger(logname)
    if any([isinstance(h, handlers.PUBHandler) for h in logger.handlers]):
        # don't add a second PUBHandler
        return
    loglevel = integer_loglevel(loglevel)
    lsock = context.socket(zmq.PUB)
    lsock.connect(iface)
    handler = handlers.PUBHandler(lsock)
    handler.setLevel(loglevel)
    handler.root_topic = root
    logger.addHandler(handler)
    logger.setLevel(loglevel)

def connect_engine_logger(context, iface, engine, loglevel=logging.DEBUG):
    logger = logging.getLogger()
    if any([isinstance(h, handlers.PUBHandler) for h in logger.handlers]):
        # don't add a second PUBHandler
        return
    loglevel = integer_loglevel(loglevel)
    lsock = context.socket(zmq.PUB)
    lsock.connect(iface)
    handler = EnginePUBHandler(engine, lsock)
    handler.setLevel(loglevel)
    logger.addHandler(handler)
    logger.setLevel(loglevel)
    return logger

def local_logger(logname, loglevel=logging.DEBUG):
    loglevel = integer_loglevel(loglevel)
    logger = logging.getLogger(logname)
    if any([isinstance(h, logging.StreamHandler) for h in logger.handlers]):
        # don't add a second StreamHandler
        return
    handler = logging.StreamHandler()
    handler.setLevel(loglevel)
    formatter = logging.Formatter("%(asctime)s.%(msecs).03d [%(name)s] %(message)s",
                datefmt="%Y-%m-%d %H:%M:%S")
    handler.setFormatter(formatter)

    logger.addHandler(handler)
    logger.setLevel(loglevel)
    return logger


"""Remote Functions and decorators for Views.

Authors:

* Brian Granger
* Min RK
"""
#-----------------------------------------------------------------------------
#  Copyright (C) 2010-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from __future__ import division

import sys
import warnings

from IPython.testing.skipdoctest import skip_doctest

from . import map as Map
from .asyncresult import AsyncMapResult

#-----------------------------------------------------------------------------
# Functions and Decorators
#-----------------------------------------------------------------------------

@skip_doctest
def remote(view, block=None, **flags):
    """Turn a function into a remote function.

    This method can be used for map:

    In [1]: @remote(view,block=True)
       ...: def func(a):
       ...:    pass
    """

    def remote_function(f):
        return RemoteFunction(view, f, block=block, **flags)
    return remote_function

@skip_doctest
def parallel(view, dist='b', block=None, ordered=True, **flags):
    """Turn a function into a parallel remote function.

    This method can be used for map:

    In [1]: @parallel(view, block=True)
       ...: def func(a):
       ...:    pass
    """

    def parallel_function(f):
        return ParallelFunction(view, f, dist=dist, block=block, ordered=ordered, **flags)
    return parallel_function

def getname(f):
    """Get the name of an object.

    For use in case of callables that are not functions, and
    thus may not have __name__ defined.

    Order: f.__name__ >  f.name > str(f)
    """
    try:
        return f.__name__
    except:
        pass
    try:
        return f.name
    except:
        pass

    return str(f)

#--------------------------------------------------------------------------
# Classes
#--------------------------------------------------------------------------

class RemoteFunction(object):
    """Turn an existing function into a remote function.

    Parameters
    ----------

    view : View instance
        The view to be used for execution
    f : callable
        The function to be wrapped into a remote function
    block : bool [default: None]
        Whether to wait for results or not.  The default behavior is
        to use the current `block` attribute of `view`

    **flags : remaining kwargs are passed to View.temp_flags
    """

    view = None # the remote connection
    func = None # the wrapped function
    block = None # whether to block
    flags = None # dict of extra kwargs for temp_flags

    def __init__(self, view, f, block=None, **flags):
        self.view = view
        self.func = f
        self.block=block
        self.flags=flags

    def __call__(self, *args, **kwargs):
        block = self.view.block if self.block is None else self.block
        with self.view.temp_flags(block=block, **self.flags):
            return self.view.apply(self.func, *args, **kwargs)

class ParallelFunction(RemoteFunction):
    """Class for mapping a function to sequences.

    This will distribute the sequences according the a mapper, and call
    the function on each sub-sequence.  If called via map, then the function
    will be called once on each element, rather that each sub-sequence.

    Parameters
    ----------

    view : View instance
        The view to be used for execution
    f : callable
        The function to be wrapped into a remote function
    dist : str [default: 'b']
        The key for which mapObject to use to distribute sequences
        options are:
          * 'b' : use contiguous chunks in order
          * 'r' : use round-robin striping
    block : bool [default: None]
        Whether to wait for results or not.  The default behavior is
        to use the current `block` attribute of `view`
    chunksize : int or None
        The size of chunk to use when breaking up sequences in a load-balanced manner
    ordered : bool [default: True]
        Whether
    **flags : remaining kwargs are passed to View.temp_flags
    """

    chunksize=None
    ordered=None
    mapObject=None

    def __init__(self, view, f, dist='b', block=None, chunksize=None, ordered=True, **flags):
        super(ParallelFunction, self).__init__(view, f, block=block, **flags)
        self.chunksize = chunksize
        self.ordered = ordered

        mapClass = Map.dists[dist]
        self.mapObject = mapClass()

    def __call__(self, *sequences):
        client = self.view.client

        # check that the length of sequences match
        len_0 = len(sequences[0])
        for s in sequences:
            if len(s)!=len_0:
                msg = 'all sequences must have equal length, but %i!=%i'%(len_0,len(s))
                raise ValueError(msg)
        balanced = 'Balanced' in self.view.__class__.__name__
        if balanced:
            if self.chunksize:
                nparts = len_0//self.chunksize + int(len_0%self.chunksize > 0)
            else:
                nparts = len_0
            targets = [None]*nparts
        else:
            if self.chunksize:
                warnings.warn("`chunksize` is ignored unless load balancing", UserWarning)
            # multiplexed:
            targets = self.view.targets
            # 'all' is lazily evaluated at execution time, which is now:
            if targets == 'all':
                targets = client._build_targets(targets)[1]
            elif isinstance(targets, int):
                # single-engine view, targets must be iterable
                targets = [targets]
            nparts = len(targets)

        msg_ids = []
        for index, t in enumerate(targets):
            args = []
            for seq in sequences:
                part = self.mapObject.getPartition(seq, index, nparts)
                if len(part) == 0:
                    continue
                else:
                    args.append(part)
            if not args:
                continue

            # print (args)
            if hasattr(self, '_map'):
                if sys.version_info[0] >= 3:
                    f = lambda f, *sequences: list(map(f, *sequences))
                else:
                    f = map
                args = [self.func]+args
            else:
                f=self.func

            view = self.view if balanced else client[t]
            with view.temp_flags(block=False, **self.flags):
                ar = view.apply(f, *args)

            msg_ids.append(ar.msg_ids[0])

        r = AsyncMapResult(self.view.client, msg_ids, self.mapObject,
                            fname=getname(self.func),
                            ordered=self.ordered
                        )

        if self.block:
            try:
                return r.get()
            except KeyboardInterrupt:
                return r
        else:
            return r

    def map(self, *sequences):
        """call a function on each element of a sequence remotely.
        This should behave very much like the builtin map, but return an AsyncMapResult
        if self.block is False.
        """
        # set _map as a flag for use inside self.__call__
        self._map = True
        try:
            ret = self.__call__(*sequences)
        finally:
            del self._map
        return ret

__all__ = ['remote', 'parallel', 'RemoteFunction', 'ParallelFunction']