def extract_urls(url, data, unescape=HTMLParser.HTMLParser().unescape):
    """Extracts the URLs from an HTML document."""
    parts = urlparse.urlparse(url)
    prefix = '%s://%s' % (parts.scheme, parts.netloc)

    accessed_dir = os.path.dirname(parts.path)
    if not accessed_dir.endswith('/'):
        accessed_dir += '/'

    for pattern, replacement in REPLACEMENT_REGEXES:
        fixed = replacement % {
            'base': prefix,
            'accessed_dir': accessed_dir,
        }
        data = re.sub(pattern, fixed, data)

    result = set()
    for match in re.finditer(MAYBE_HTML_URL_REGEX, data):
        found_url = unescape(match.groupdict()['absurl'])
        found_url = clean_url(
            found_url,
            force_scheme=parts[0])  # Use the main page's scheme
        result.add(found_url)

    return result
def extract_urls ( url , data , unescape = HTMLParser . HTMLParser ( ) . unescape ) : parts = urlparse . urlparse ( url ) prefix = '%s://%s' % ( parts . scheme , parts . netloc ) accessed_dir = os . path . dirname ( parts . path ) if not accessed_dir . endswith ( '/' ) : accessed_dir += '/' for pattern , replacement in REPLACEMENT_REGEXES : fixed = replacement % { 'base' : prefix , 'accessed_dir' : accessed_dir , } data = re . sub ( pattern , fixed , data ) result = set ( ) for match in re . finditer ( MAYBE_HTML_URL_REGEX , data ) : found_url = unescape ( match . groupdict ( ) [ 'absurl' ] ) found_url = clean_url ( found_url , force_scheme = parts [ 0 ] ) result . add ( found_url ) return result
Extracts the URLs from an HTML document.