def pmap(func, *args, **kwargs):
    """
    Run a function in parallel.

    This function behaves pretty much like the built-in
    function map(), but it runs in parallel using threads:

        >>> from golismero.api.parallel import pmap
        >>> def triple(x):
        ...   return x * 3
        ...
        >>> pmap(triple, [1, 2, 3, 4, 5])
        [3, 6, 9, 12, 15]
        >>> def addition(x, y):
        ...   return x + y
        ...
        >>> pmap(addition, [1, 2, 3, 4, 5], [2, 4, 6, 8, 10])
        [3, 6, 9, 12, 15]
        >>> pmap((lambda x: x + 1), [1, 2, 3, 4, 5])
        [2, 3, 4, 5, 6]
        >>> def printer(x, y):
        ...    print "%s - %s" (x, y)
        ...
        >>> from functools import partial
        >>> pmap(partial(printer, "fixed_text"), [1, 2])
        fixed_text - 1
        fixed_text - 2
        >>> class Point(object):
        ...   def __init__(self, x, y):
        ...     self.x = x
        ...     self.y = y
        ...   def __repr__(self):
        ...     return "Point(%d, %d)" % (self.x, self.y)
        ...
        >>> pmap(Point, [1, 2, 3], [4, 5, 6])
        [Point(1, 4), Point(2, 5), Point(3, 6)]

    .. warning: Unlike the built-in map() function, exceptions raised
                by the callback function will be silently ignored.

    :param func: A function or any other executable object.
    :type func: callable

    :param args: One or more iterables or simple params containing the parameters for each call.
    :type args: simple_param, simple_param, iterable, iterable...

    :keyword pool_size: Maximum number of concurrent threads. Defaults to 4.
    :type pool_size: int

    :return: List with returned results, in the same order as the input data.
    :rtype: list
    """

    # Check for missing arguments.
    if len(args) <= 0:
        raise TypeError("Expected at least one positional argument")

    # Get the pool size.
    pool_size = kwargs.pop("pool_size", 10)

    # Check for extraneous keyword arguments.
    if kwargs:
        if len(kwargs) > 1:
            msg = "Unknown keyword arguments: "
        else:
            msg = "Unknown keyword argument: "
        raise TypeError(msg + ", ".join(kwargs))

    # Undocumented behavior of the built-in map() function:
    # passing None as the callback works as the identity function.
    # (This behavior was removed in Python 3).
    if func is None:
        return map(None, *args)

    # Interpolate the arguments.
    if len(args) == 1:
        data = [ (x,) for x in args[0] ]  # force to 1-tuples
    else:
        data = map(None, *args)

    # If there are no tasks to run, return an empty list.
    # This is exactly how the built-in map() behaves.
    if not data:
        return []

    # If there's only one task to run, just do it now.
    # No need to run in parallel.
    if len(data) == 1:
        return [ func( *data[0] ) ]

    # Create the task group.
    m_task_group = TaskGroup(func, data)

    # Create the pool.
    m_pool = WorkerPool(pool_size)

    try:

        # Add the task and get the returned values.
        m_pool.add_task_group(m_task_group)

        # Wait for the tasks to end.
        m_pool.join_tasks()

    finally:

        # Stop the pool.
        m_pool.stop()

    # Return the list of results.
    return m_task_group.pack_output() class URL(_AbstractURL):
    """
    Universal Resource Locator (URL).

    You can get the URL in canonical form:

    - url

    In deconstructed form:

    - parsed_url

    The current crawling depth level:

    - depth

    And some extra information needed to build an HTTP request:

    - method
    - url_params
    - post_params
    - referer
    """

    data_subtype = "url"


    #--------------------------------------------------------------------------
    def __init__(self, url, method = "GET", post_params = None, referer = None, **kwargs):
        """
        :param url: Absolute URL.
        :type url: str

        :param method: HTTP method.
        :type method: str

        :param post_params: POST parameters or raw data.
        :type post_params: dict(str -> str) | str

        :param referer: Referrer URL.
        :type referer: str

        :raises ValueError: Currently, relative URLs are not allowed.
        """

        # Validate the arguments.
        if method:
            method = to_utf8(method)
        else:
            method = "GET"
        if referer:
            referer = to_utf8(referer)
        else:
            referer = None
        if not isinstance(method, str):
            raise TypeError("Expected string, got %r instead" % type(method))
        if post_params is not None and not isinstance(post_params, dict):
            raise TypeError("Expected dict, got %r instead" % type(post_params))
        if referer is not None and not isinstance(referer, str):
            raise TypeError("Expected string, got %r instead" % type(referer))
        if post_params:
            if hasattr(post_params, "iteritems"):
                post_params = {
                    to_utf8(k): to_utf8(v) for k,v in post_params.iteritems()
                }
                post_data = '&'.join(
                    '%s=%s' % ( quote(k, safe=''), quote(v, safe='') )
                    for (k, v) in sorted(post_params.iteritems())
                )
            else:
                post_data   = to_utf8(post_params)
                post_params = None
        else:
            post_data   = None
            post_params = None

            # Checks for get param
            url_params = kwargs.get('url_params', None)

            if url_params is not None:
                if hasattr(url_params, "iteritems"):
                    url_params = {
                        to_utf8(k): to_utf8(v) for k, v in url_params.iteritems()
                    }
                    query_param = '&'.join(
                        '%s=%s' % ( quote(k, safe=''), quote(v, safe='') )
                        for (k, v) in sorted(url_params.iteritems())
                    )
                    url = url + '?' + query_param
                    print url

        # Save the properties.
        self.__method      = method
        self.__post_data   = post_data
        self.__post_params = post_params
        self.__referer     = parse_url(referer).url if referer else None

        # Call the parent constructor.
        super(URL, self).__init__(url)

        # Increment the crawling depth by one.
        self.depth += 1


    #--------------------------------------------------------------------------
    def __repr__(self):
        s = "<URL url=%r, method=%r, params=%r, referer=%r, depth=%r>"
        s %= (self.url, self.method, self.post_params, self.referer, self.depth)
        return s


    #--------------------------------------------------------------------------
    @property
    def display_name(self):
        return "URL"


    #--------------------------------------------------------------------------

    @identity
    def method(self):
        """
        :return: HTTP method.
        :rtype: str
        """
        return self.__method

    @identity
    def post_data(self):
        """
        :return: POST data.
        :rtype: str
        """
        return self.__post_data


    #--------------------------------------------------------------------------

    @property
    def url_params(self):
        """
        :return: GET parameters.
        :rtype: dict(str -> str)
        """
        query_params = self.parsed_url.query_params
        if query_params:
            return query_params
        return {}

    @property
    def has_url_params(self):
        """
        :return: True if there are GET parameters, False otherwise.
        :rtype: bool
        """
        return bool(self.url_params)

    @property
    def post_params(self):
        """
        :return: POST parameters.
        :rtype: dict(str -> str)
        """
        if self.__post_params:
            return self.__post_params.copy()
        return {}

    @property
    def has_post_params(self):
        """
        :return: True if there are POST parameters, False otherwise.
        :rtype: bool
        """
        return bool(self.post_params)

    @property
    def referer(self):
        """
        :return: Referer for this URL.
        :rtype: str
        """
        return self.__referer


    #--------------------------------------------------------------------------
    @property
    def discovered(self):
        if self.is_in_scope():
            result = FolderURL.from_url(self.url)
            result.append( BaseURL(self.url) )
            try:
                result.append( IP(self.hostname) )
            except ValueError:
                result.append( Domain(self.hostname) )
            return result
        return [] def extract_from_html(raw_html, base_url, only_links = True):
    """
    Extract URLs from HTML.

    Implementation notes:

    - The current implementation is fault tolerant, meaning it will try
      to extract URLs even if the HTML is malformed and browsers wouldn't
      normally see those links. This may therefore result in some false
      positives.

    - HTML5 tags are supported, including tags not currently supported by
      any major browser.

    :param raw_html: Raw HTML data.
    :type raw_html: str

    :param base_url: Base URL for the current document.
    :type base_url: str

    :param only_links: If True, only extract links to other resources.
        If False, extract all URLs.
    :type only_links: bool

    :returns: Extracted URLs.
    :rtype: set(str)
    """

    # Set where the URLs will be collected.
    result = set()
    add_result = result.add

    # Remove the fragment from the base URL.
    base_url = urldefrag(base_url)[0]

    # Parse the raw HTML.
    bs = BeautifulSoup(decode(raw_html, detect(raw_html)["encoding"]),
                       convertEntities = BeautifulSoup.ALL_ENTITIES)

    # Some sets of tags and attributes to look for.
    href_tags = {"a", "link", "area"}
    src_tags = {"script", "img", "iframe", "frame", "embed", "source", "track"}
    param_names = {"movie", "href", "link", "src", "url", "uri"}

    # Iterate once through all tags...
    for tag in bs.findAll():

        # Get the tag name, case insensitive.
        name = tag.name.lower()

        # Extract the URL from each tag that has one.
        url = None
        if name in href_tags:
            url = tag.get("href", None)
        elif name in src_tags:
            url = tag.get("src", None)
        elif name == "param":
            name = tag.get("name", "").lower().strip()
            if name in param_names:
                url = tag.get("value", None)
        ##elif name == "form":
        ##    url = tag.get("action", None)
        elif name == "object":
            url = tag.get("data", None)
        elif name == "applet":
            url = tag.get("code", None)
        elif name == "meta":
            name = tag.get("name", "").lower().strip()
            if name == "http-equiv":
                content = tag.get("content", "")
                p = content.find(";")
                if p >= 0:
                    url = content[ p + 1 : ]
        elif name == "base":
            url = tag.get("href", None)
            if url is not None:

                # XXX FIXME
                # Unicode URLs are not supported.
                try:
                    url = str(url)
                except Exception:
                    warn("Unicode URLs not yet supported: %r" % url)
                    continue

                # Update the base URL.
                try:
                    base_url = urljoin(base_url, url.strip(),
                                       allow_fragments = False)
                except Exception:
                    continue

        # If we found an URL in this tag...
        if url is not None:

            # XXX FIXME
            # Unicode URLs are not supported.
            try:
                url = str(url)
            except Exception:
                warn("Unicode URLs not yet supported: %r" % url)
                continue

            # Canonicalize the URL.
            try:
                url = urljoin(base_url, url.strip())
            except Exception:
                continue

            # Discard URLs that are not links to other pages or resources.
            if not only_links or is_link(url, base_url = base_url):

                # Add the URL to the set.
                add_result(url)

    # Return the set of collected URLs.
    return result def extract_from_text(text, base_url = None, only_links = True):
    """
    Extract URLs from text.

    Implementation notes:

    - Unicode URLs are currently not supported.

    :param text: Text.
    :type text: str

    :param base_url: Base URL for the current document.
        If not specified, relative URLs are ignored.
    :type base_url: str

    :param only_links: If True, only extract links to other resources.
        If False, extract all URLs.
    :type only_links: bool

    :returns: Extracted URLs.
    :rtype: set(str)
    """

    # Trivial case.
    if not text:
        return set()

    # Check the type.
    if not isinstance(text, basestring):
        raise TypeError("Expected string, got %r instead" % type(text))

    # Set where the URLs will be collected.
    result = set()

    # Remove the fragment from the base URL.
    if base_url:
        base_url = urldefrag(base_url)[0]

    # Look for URLs using regular expressions.
    for regex in (_re_url_rfc, _re_url_readable):
        for url in regex.findall(text):
            url = url[0]

            # Skip if we've already seen it.
            if url in result:
                continue

            # XXX FIXME
            # Make sure the text is really ASCII text.
            # We don't support Unicode yet.
            try:
                url = str(url)
            except Exception:
                warn("Unicode URLs not yet supported: %r" % url)
                continue

            # If a base URL was given...
            if base_url:

                # Canonicalize the URL.
                # Discard it on parse error.
                try:
                    url = urljoin(base_url, url.strip())
                except Exception:
                    continue

                # Skip if we've already seen it.
                if url in result:
                    continue

                # Discard URLs that are not links to other pages or resources,
                # and URLs we've already seen.
                if only_links and (url in result or
                                       not is_link(url, base_url = base_url)):
                    continue

            # If a base URL was NOT given...
            else:

                # Discard relative URLs.
                # Also discard them on parse error.
                try:
                    parsed = parse_url(url)
                    if not parsed.scheme or not parsed.netloc:
                        continue
                except Exception:
                    raise
                    continue

            # Add the URL to the set.
            result.add(url)

    # Return the set of collected URLs.
    return result def extract_forms_from_html(raw_html, base_url):
    """
    Extract forms info from HTML.

    :param raw_html: Raw HTML data.
    :type raw_html: str

    :param base_url: Base URL for the current document.
    :type base_url: str

    :returns: Extracted form info.
    :rtype: list((URL, METHOD, list({ "name" : PARAM_NAME, "value" : PARAM_VALUE, "type" : PARAM_TYPE})))
    """

    # Set where the URLs will be collected.
    result = list()
    result_append = result.append

    # Remove the fragment from the base URL.
    base_url = urldefrag(base_url)[0]

    # Parse the raw HTML.
    bs = BeautifulSoup(decode(raw_html, detect(raw_html)["encoding"]))

    for form in bs.findAll("form"):
        target = form.get("action", None)
        method = form.get("method", "POST").upper()

        if not target:
            continue

        try:
            target = str(target)
        except Exception:
            warn("Unicode URLs not yet supported: %r" % target)
            continue

        # Canonicalize the URL.
        try:
            target = urljoin(base_url, target.strip())
        except Exception:
            continue

        form_params = []
        form_params_append = form_params.append
        for params in form.findAll("input"):
            if params.get("type") == "submit":
                continue

            form_params_append({
                "name": params.get("name", "NAME"),
                "value": params.get("value", "VALUE"),
                "type": params.get("type", "TYPE")})

        # Add to results
        result_append((target, method, form_params))

    return  result def download(url, callback = None, timeout = 10.0, allow_redirects = True,
             allow_out_of_scope = False):
    """
    Download the file pointed to by the given URL.

    An optional callback function may be given. It will be called just
    before downloading the file, and receives the file size. If it
    returns True the download proceeds, if it returns False it's
    cancelled.

    Example:

        >>> from golismero.api.data.resource.url import URL
        >>> from golismero.api.net.web_utils import download
        >>> def decide(url, name, size, type):
        ...     # 'url' is the URL for the download
        ...     if url is not None:
        ...         print "URL: %s" % url
        ...     # 'name' is the suggested filename (None if not available)
        ...     if name is not None:
        ...         print "Name: %s" % name
        ...     # 'size' is the file size (None if not available)
        ...     if size is not None:
        ...         print "Size: %d" % size
        ...     # 'type' is the MIME type (None if not available)
        ...     if type is not None:
        ...         print "Type: %s" % type
        ...     # Cancel download if not a web page
        ...     if type != "text/html":
        ...         return False
        ...     # Cancel download if it's too large
        ...     if size > 1000000:
        ...         return False
        ...     # Continue downloading
        ...     return True
        ...
        >>> download(URL("http://www.example.com/index.html"), callback=decide)
        URL: http://www.example.com/index.html
        Name: index.html
        Size: 1234
        Type: text/html
        <HTML identity=606489619590839a1c0ad662bcdc0189>
        >>> download(URL("http://www.example.com/"), callback=decide)
        URL: http://www.example.com/
        Size: 1234
        Type: text/html
        <HTML identity=606489619590839a1c0ad662bcdc0189>
        >>> print download(URL("http://www.example.com/big_file.iso"), callback=decide)
        URL: http://www.example.com/big_file.iso
        Name: big_file.iso
        Size: 1234567890
        Type: application/octet-stream
        None

    :param url: URL to download.
    :type url: Url

    :param callback: Callback function.
    :type callback: callable

    :param timeout: Timeout in seconds.
            The minimum value is 0.5 and the maximum is 100.0. Any other values
            will be silently converted to either one of them.
    :type timeout: int | float

    :param allow_redirects: True to follow redirections, False otherwise.
    :type allow_redirects: bool

    :param allow_out_of_scope: True to allow download of URLs out of scope,
                               False otherwise.
    :type allow_out_of_scope: bool

    :returns: Downloaded data as an object of the GoLismero data model,
              or None if cancelled.
    :rtype: File | None

    :raises NetworkOutOfScope: The resource is out of the audit scope.
    :raises NetworkException: A network error occurred during download.
    :raises NotImplementedError: The network protocol is not supported.
    """

    # Validate the callback type.
    if callback is not None and not callable(callback):
        raise TypeError(
            "Expected callable (function, class, instance with __call__),"
            " got %r instead" % type(callback)
        )

    # Autogenerate an URL object if a string is given (common mistake).
    from ..data.resource.url import URL
    if not isinstance(url, URL):
        url = URL(url)
        LocalDataCache.on_autogeneration(url)
        parsed = url.parsed_url
        if not parsed.hostname or not parsed.scheme:
            raise ValueError("Only absolute URLs must be used!")

    # Validate the protocol.
    # TODO: add support for FTP
    scheme = url.parsed_url.scheme
    if scheme not in ("http", "https"):
        raise NotImplementedError("Protocol not supported: %s" % scheme)

    # Validate the scope.
    if not url.is_in_scope() and allow_out_of_scope is False:
        raise NetworkOutOfScope("URL out of scope: %s" % url.url)

    # Autogenerate the HTTP request object.
    from ..config import Config
    from ..data.information.http import HTTP_Request
    request = HTTP_Request( url         = url.url,
                            method      = url.method,
                            post_data   = url.post_params,
                            referer     = url.referer,
                            user_agent  = Config.audit_config.user_agent)
    LocalDataCache.on_autogeneration(request)

    # Prepare the callback.
    if callback is None:
        temp_callback = None
    else:
        def temp_callback(
                    request, url, status_code, content_length, content_type):

            # Abort if not successful.
            if status_code != "200":
                return False

            # Get the name.
            # TODO: parse the Content-Disposition header.
            name = ParsedURL(url).filename
            if not name:
                name = request.parsed_url.filename
                if not name:
                    name = None

            # Call the user-defined callback.
            return callback(url, name, content_length, content_type)

    # Send the request and get the response.

    from .http import HTTP
    response = HTTP.make_request(request,
                                 callback = temp_callback,
                                 timeout = timeout,
                                 allow_redirects = allow_redirects,
                                 allow_out_of_scope = allow_out_of_scope)

    # If not aborted...
    if response:

        # The response object is autogenerated.
        LocalDataCache.on_autogeneration(response)

        # Associate the URL to the request and the response.
        request.add_resource(url)
        response.add_resource(url)

        # Extract the data from the response.
        data = data_from_http_response(response)

        # Associate the data to the URL.
        data.add_resource(url)

        # Return the data.
        return data def parse_url(url, base_url = None):
    """
    Parse an URL and return a mutable object with all its parts.

    For more details see: ParsedURL

    :param url: URL to parse.
    :type url: str

    :param base_url: Optional base URL.
    :type base_url: str

    :returns: Mutable object with access to the URL parts.
    :rtype: ParsedURL
    """
    return ParsedURL(url, base_url) def argument_query(query_str):
    if query_str and query_str.startswith('?'):
        warn("You don't need to use a leading '?' when setting the query"
         " string, this may be an error!", stacklevel=3)
    if not query_str:
        query_params = {}
    else:
        try:
            # much faster than parse_qsl()
            query_params = dict(( map(unquote_plus, (to_utf8(token) + '=').split('=', 2)[:2])
                                  for token in query_str.split('&') ))
            if len(query_params) == 1 and not query_params.values()[0]:
                query_params = {}
            else:
                query = None
        except Exception:
            ##raise   # XXX DEBUG
            query_params = {}
    return query_params def download(url, callback = None, timeout = 10.0, allow_redirects = True,
             allow_out_of_scope = False):
    """
    Download the file pointed to by the given URL.

    An optional callback function may be given. It will be called just
    before downloading the file, and receives the file size. If it
    returns True the download proceeds, if it returns False it's
    cancelled.

    Example:

        >>> from golismero.api.data.resource.url import URL
        >>> from golismero.api.net.web_utils import download
        >>> def decide(url, name, size, type):
        ...     # 'url' is the URL for the download
        ...     if url is not None:
        ...         print "URL: %s" % url
        ...     # 'name' is the suggested filename (None if not available)
        ...     if name is not None:
        ...         print "Name: %s" % name
        ...     # 'size' is the file size (None if not available)
        ...     if size is not None:
        ...         print "Size: %d" % size
        ...     # 'type' is the MIME type (None if not available)
        ...     if type is not None:
        ...         print "Type: %s" % type
        ...     # Cancel download if not a web page
        ...     if type != "text/html":
        ...         return False
        ...     # Cancel download if it's too large
        ...     if size > 1000000:
        ...         return False
        ...     # Continue downloading
        ...     return True
        ...
        >>> download(URL("http://www.example.com/index.html"), callback=decide)
        URL: http://www.example.com/index.html
        Name: index.html
        Size: 1234
        Type: text/html
        <HTML identity=606489619590839a1c0ad662bcdc0189>
        >>> download(URL("http://www.example.com/"), callback=decide)
        URL: http://www.example.com/
        Size: 1234
        Type: text/html
        <HTML identity=606489619590839a1c0ad662bcdc0189>
        >>> print download(URL("http://www.example.com/big_file.iso"), callback=decide)
        URL: http://www.example.com/big_file.iso
        Name: big_file.iso
        Size: 1234567890
        Type: application/octet-stream
        None

    :param url: URL to download.
    :type url: Url

    :param callback: Callback function.
    :type callback: callable

    :param timeout: Timeout in seconds.
            The minimum value is 0.5 and the maximum is 100.0. Any other values
            will be silently converted to either one of them.
    :type timeout: int | float

    :param allow_redirects: True to follow redirections, False otherwise.
    :type allow_redirects: bool

    :param allow_out_of_scope: True to allow download of URLs out of scope,
                               False otherwise.
    :type allow_out_of_scope: bool

    :returns: Downloaded data as an object of the GoLismero data model,
              or None if cancelled.
    :rtype: File | None

    :raises NetworkOutOfScope: The resource is out of the audit scope.
    :raises NetworkException: A network error occurred during download.
    :raises NotImplementedError: The network protocol is not supported.
    """

    # Validate the callback type.
    if callback is not None and not callable(callback):
        raise TypeError(
            "Expected callable (function, class, instance with __call__),"
            " got %r instead" % type(callback)
        )

    # Autogenerate an URL object if a string is given (common mistake).
    from ..data.resource.url import URL
    if not isinstance(url, URL):
        url = URL(url)
        LocalDataCache.on_autogeneration(url)
        parsed = url.parsed_url
        if not parsed.hostname or not parsed.scheme:
            raise ValueError("Only absolute URLs must be used!")

    # Validate the protocol.
    # TODO: add support for FTP
    scheme = url.parsed_url.scheme
    if scheme not in ("http", "https"):
        raise NotImplementedError("Protocol not supported: %s" % scheme)

    # Validate the scope.
    if not url.is_in_scope() and allow_out_of_scope is False:
        raise NetworkOutOfScope("URL out of scope: %s" % url.url)

    # Autogenerate the HTTP request object.
    from ..config import Config
    from ..data.information.http import HTTP_Request
    request = HTTP_Request( url         = url.url,
                            method      = url.method,
                            post_data   = url.post_params,
                            referer     = url.referer,
                            user_agent  = Config.audit_config.user_agent)
    LocalDataCache.on_autogeneration(request)

    # Prepare the callback.
    if callback is None:
        temp_callback = None
    else:
        def temp_callback(
                    request, url, status_code, content_length, content_type):

            # Abort if not successful.
            if status_code != "200":
                return False

            # Get the name.
            # TODO: parse the Content-Disposition header.
            name = ParsedURL(url).filename
            if not name:
                name = request.parsed_url.filename
                if not name:
                    name = None

            # Call the user-defined callback.
            return callback(url, name, content_length, content_type)

    # Send the request and get the response.

    from .http import HTTP
    response = HTTP.make_request(request,
                                 callback = temp_callback,
                                 timeout = timeout,
                                 allow_redirects = allow_redirects,
                                 allow_out_of_scope = allow_out_of_scope)

    # If not aborted...
    if response:

        # The response object is autogenerated.
        LocalDataCache.on_autogeneration(response)

        # Associate the URL to the request and the response.
        request.add_resource(url)
        response.add_resource(url)

        # Extract the data from the response.
        data = data_from_http_response(response)

        # Associate the data to the URL.
        data.add_resource(url)

        # Return the data.
        return data def get_request(url, timeout = 10.0, allow_redirects = True,
             allow_out_of_scope = False, use_cache = None):
    """
    request the given URL.

    :param url: URL to download.
    :type url: Url

    :param timeout: Timeout in seconds.
            The minimum value is 0.5 and the maximum is 100.0. Any other values
            will be silently converted to either one of them.
    :type timeout: int | float

    :param allow_redirects: True to follow redirections, False otherwise.
    :type allow_redirects: bool

    :param allow_out_of_scope: True to allow download of URLs out of scope,
                               False otherwise.
    :type allow_out_of_scope: bool

    :returns: Downloaded data as an object of the GoLismero data model,
              or None if cancelled.
    :rtype: File | None

    :raises NetworkOutOfScope: The resource is out of the audit scope.
    :raises NetworkException: A network error occurred during download.
    :raises NotImplementedError: The network protocol is not supported.
    """

    # Autogenerate an URL object if a string is given (common mistake).
    from ..data.resource.url import URL
    if not isinstance(url, URL):
        url = URL(url)
        LocalDataCache.on_autogeneration(url)
        parsed = url.parsed_url
        if not parsed.hostname or not parsed.scheme:
            raise ValueError("Only absolute URLs must be used!")

    # Validate the protocol.
    # TODO: add support for FTP
    scheme = url.parsed_url.scheme
    if scheme not in ("http", "https"):
        raise NotImplementedError("Protocol not supported: %s" % scheme)

    # Validate the scope.
    if not url.is_in_scope() and allow_out_of_scope is False:
        raise NetworkOutOfScope("URL out of scope: %s" % url.url)

    # Autogenerate the HTTP request object.
    from ..config import Config
    from ..data.information.http import HTTP_Request
    request = HTTP_Request( url         = url.url,
                            method      = url.method,
                            post_data   = url.post_params,
                            referer     = url.referer,
                            user_agent  = Config.audit_config.user_agent)
    LocalDataCache.on_autogeneration(request)

    # Send the request and get the response.

    from .http import HTTP
    response = HTTP.make_request(request,
                                 callback = None,
                                 timeout = timeout,
                                 use_cache = use_cache,
                                 allow_redirects = allow_redirects,
                                 allow_out_of_scope = allow_out_of_scope)

    return response def payload_muntants(url_info, payload = {}, bmethod = 'GET', exclude_cgi_suffix = ['css', 'js', 'jpeg', 'jpg', 'png', 'gif', 'svg', 'txt'],
                     use_cache = None, timeout = 10.0 , bcheck_use_orig_body = True, req_header = {},
                     resp_code = '200', resp_header = {}, **kwargs):

    '''

    :param url_info:
    :param payload: {'k':'id', 'pos': 1, 'payload':str, 'type': 0}  (pos:0 key, pos:1 value) (type:0 append, type:1 replace)
    :param exclude_cgi_suffix:
    :param depth:
    :param bcheck_use_orig_body:
    :param req_header:
    :param resp_code:
    :param resp_header:
    :param kwargs:
    :return:
    '''
    if not isinstance(url_info , URL):
        raise TypeError("Expected url object, type:%s" % type(url_info))

    if not isinstance(payload, dict):
        raise TypeError("Excepted payload object, type:%s" % type(payload))

    if url_info.parsed_url.extension[1:] in exclude_cgi_suffix:
        Logger.log_verbose("Skipping URL: %s" % url_info.url)

    m_url_info = copy(url_info)
    if  bmethod == "GET":
        param_dict = copy(m_url_info.url_params)
    elif bmethod == "POST":
        param_dict = copy(m_url_info.post_params)

    if len(param_dict) == None and len(param_dict) == 0:
        return None

    __ = parse_url(m_url_info.url)

    k = payload['k']
    if payload['pos'] == 1:
        #value
        if payload['type'] == 0:  #append
            param_dict[k] = param_dict[k] + payload['payload']
        elif payload['type'] == 1:  #replace
            param_dict[k] = payload['payload']
    else:
        #key 先不考虑key值
        if payload['type'] == 0:
            param_dict.update(k = param_dict.pop(k))

        # TODO GET/POST param key need deal
        raise ValueError("GET/POST param key payload is not support!")

    retry_cnt = 0

    while retry_cnt < 3:
        if bmethod == "GET":
            m_resource_url_payload = URL(url = __.request_cgi, method = m_url_info.method, referer = m_url_info.referer, url_params= param_dict)

        elif bmethod == "POST":
            m_resource_url_payload = URL(url = __.request_cgi, method = m_url_info.method, referer = m_url_info.referer, post_params= param_dict)

        try:
            p = get_request(url = m_resource_url_payload, allow_redirects=False, use_cache = use_cache, timeout = timeout)
            return p

        except NetworkException, e:
            retry_cnt += 1
            time.sleep(0.5)
            Logger.log_error_verbose("Error while processing %r: %s" % (m_resource_url_payload.url, str(e)))

    return None
from golismero.api.config import Config
from golismero.api.data.information.html import HTML
from golismero.api.data.information.text import Text
from golismero.api.data.resource.email import Email
from golismero.api.parallel import pmap
from golismero.api.data.resource.url import URL
from golismero.api.logger import Logger
from golismero.api.net import NetworkException
from golismero.api.net.scraper import extract_from_html, extract_from_text, extract_forms_from_html
from golismero.api.net.web_utils import download, parse_url, argument_query, download, get_request
from golismero.api.plugin import TestingPlugin
from golismero.api.text.wordlist import WordListLoader
from golismero.api.text.text_utils import to_utf8
from golismero.api.net.web_mutants import payload_muntants
from golismero.api.data.vulnerability.injection.xss import XSS
from scan_policy import xss_reflection_detect_test_cases
from random import randint
from bs4 import BeautifulSoup
from types import NoneType as NT
    import re2 as re
    import re
#!/usr/bin/env/python
#-*- coding:utf-8 -*-

__author__ = 'BlackYe.'



try:
except ImportError:
else:
    re.set_fallback_notification(re.FALLBACK_WARNING)

class ReflectionXSSPlugin(TestingPlugin):

    '''
    this plugin is a any file read plugin
    '''

    #--------------------------------------------------------------------------
    def get_accepted_types(self):
        return [URL]


    #--------------------------------------------------------------------------
    def run(self, info):
        #if not info.has_url_params and not info.has_post_params:
        #    return

        m_return = []

        #TODO 30X redirect


        #TODO Content-Type
        p = get_request(url = info, allow_redirects=False)
        if (p.status == '301' or p.status == '302') and not p.headers.get('Location'):

            return m_return

        if p.content_type is not None and re.search('(application\/json)|(application\/javascript)|(text\/json)|(text\/javascript)|'
                       '(application\/x-javascript)|(application\/octet-stream)|(text\/xml)|(application\/xml)', p.content_type) is not None:

            return m_return

        m_url = info

        if info.has_url_params:

            for k,v in m_url.url_params.iteritems():
                key = to_utf8(k)
                value = to_utf8(v)

                if self.xss_detect(m_url, method = 'GET', k = key, v = value):

                    url = URL(url = m_url.url,
                          method = 'GET',
                          post_params = None,
                          referer = m_url.referer)

                    vul = XSS(url, vulnerable_params = {"injection":"xxxxxx"}, injection_point = XSS.INJECTION_POINT_URL, injection_type = "XSS")

                    vul.description += "fuck"

                    m_return.append(vul)
                    break

                #return m_return

        if info.has_post_params:
            print 'POST'

        # Send the results
        return m_return

    def xss_detect(self, url, method = 'GET',  **kwargs):

        if not isinstance(url, URL):
            raise TypeError("Expected url type, type:%s" % type(url))

        k = kwargs.get("k", None)
        if k is None or not isinstance(k, str):
            raise ValueError("Except param has not key!")

        v = kwargs.get("v", None)

        for xss_test_case_dict in xss_reflection_detect_test_cases:
            #xss_test_case_dict = xss_reflection_detect_test_cases[12]

            rand_num = 900000000 + randint(1, 9999999)
            xss_payload = xss_test_case_dict['input'].replace('rndstr', str(rand_num))

            xss_resp = payload_muntants(url, payload = {'k': k , 'pos': 1, 'payload':xss_payload, 'type': 1}, bmethod = method)