def get_url(url, data=None, cached=True, cache_key=None, crawler='urllib'):
    """Retrieves the HTML code for a given URL.
     If a cached version is not available, uses phantom_retrieve to fetch the page.

    data - Additional data that gets passed onto the crawler.
    cached - If True, retrieves the URL from the cache if it is available. If False, will still store the page in cache.
    cache_key - If set, will be used instead of the URL to lookup the cached version of the page.
    crawler - A string referencing one of the builtin crawlers.

    Returns the HTML as a unicode string.
    Raises a HttpNotFound exception if the page could not be found.
    """
    if cache_key is None:
        cache_key = url
    cache_path = cache_path_for_url(cache_key)
    if cached and os.path.exists(cache_path):
        with open(cache_path) as f:
            html = f.read().decode('utf-8')
    else:
        if FAIL_IF_NOT_CACHED:
            raise BaseException("URL is not in cache and FAIL_IF_NOT_CACHED is True: %s" % url)
        crawler_fn = CRAWLERS[crawler]
        status, html = crawler_fn(url, data)
        if status != 200: 
            raise HttpNotFound(url)
        _ensure_directory(CACHE_DIRECTORY)
        with open(cache_path, 'w') as f:
            f.write(html.encode('utf-8'))
    return html
def get_url ( url , data = None , cached = True , cache_key = None , crawler = 'urllib' ) : if cache_key is None : cache_key = url cache_path = cache_path_for_url ( cache_key ) if cached and os . path . exists ( cache_path ) : with open ( cache_path ) as f : html = f . read ( ) . decode ( 'utf-8' ) else : if FAIL_IF_NOT_CACHED : raise BaseException ( "URL is not in cache and FAIL_IF_NOT_CACHED is True: %s" % url ) crawler_fn = CRAWLERS [ crawler ] status , html = crawler_fn ( url , data ) if status != 200 : raise HttpNotFound ( url ) _ensure_directory ( CACHE_DIRECTORY ) with open ( cache_path , 'w' ) as f : f . write ( html . encode ( 'utf-8' ) ) return html
Retrieves the HTML code for a given URL.
     If a cached version is not available, uses phantom_retrieve to fetch the page.

    data - Additional data that gets passed onto the crawler.
    cached - If True, retrieves the URL from the cache if it is available. If False, will still store the page in cache.
    cache_key - If set, will be used instead of the URL to lookup the cached version of the page.
    crawler - A string referencing one of the builtin crawlers.

    Returns the HTML as a unicode string.
    Raises a HttpNotFound exception if the page could not be found.