<s> """<STR_LIT>""" <EOL> from json import dumps , loads <EOL> from os . path import abspath , basename , join <EOL> from urllib import quote <EOL> from urlparse import urlsplit , urlunsplit <EOL> from BeautifulSoup import BeautifulSoup <EOL> from requests import codes , ConnectionError , get , Timeout <EOL> from cheddar . exceptions import NotFoundError <EOL> from cheddar . index . index import Index <EOL> from cheddar . model . versions import guess_name_and_version <EOL> class RemoteIndex ( Index ) : <EOL> """<STR_LIT>""" <EOL> MAX_DEPTH = <NUM_LIT:2> <EOL> def __init__ ( self , app ) : <EOL> self . index_url = app . config [ "<STR_LIT>" ] <EOL> self . get_timeout = app . config [ "<STR_LIT>" ] <EOL> self . logger = app . logger <EOL> def get_projects ( self ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> def get_versions ( self , name ) : <EOL> """<STR_LIT>""" <EOL> url = "<STR_LIT>" . format ( self . index_url , name ) <EOL> versions = { name : build_remote_path ( href , location ) <EOL> for name , href , location in self . _iter_listings ( url , name ) } <EOL> self . logger . debug ( "<STR_LIT>" . format ( name , versions ) ) <EOL> return versions <EOL> def get_metadata ( self , name , version ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def get_distribution ( self , location , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . logger . info ( "<STR_LIT>" . format ( location ) ) <EOL> response = fetch_url ( location , self . get_timeout , self . logger ) <EOL> return response . content , response . headers [ "<STR_LIT:Content-Type>" ] <EOL> def remove_version ( self , name , version ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> def validate_metadata ( self , metadata ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> def upload_distribution ( self , file_ ) : <EOL> """<STR_LIT>""" <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> def _iter_listings ( self , url , name , depth = <NUM_LIT:0> ) : <EOL> """<STR_LIT>""" <EOL> self . logger . info ( "<STR_LIT>" . format ( name ) ) <EOL> response = fetch_url ( url , self . get_timeout , self . logger ) <EOL> location = get_request_location ( response , url ) <EOL> self . logger . debug ( "<STR_LIT>" . format ( location ) ) <EOL> for link in iter_version_links ( response . text , name ) : <EOL> if isinstance ( link , tuple ) : <EOL> name , href = link <EOL> yield name , href , location <EOL> else : <EOL> if depth <= RemoteIndex . MAX_DEPTH : <EOL> try : <EOL> self . logger . info ( "<STR_LIT>" . format ( link ) ) <EOL> for listing in self . _iter_listings ( link , name , depth + <NUM_LIT:1> ) : <EOL> yield listing <EOL> except NotFoundError : <EOL> self . logger . debug ( "<STR_LIT>" . format ( link ) ) <EOL> else : <EOL> self . logger . info ( "<STR_LIT>" . format ( link ) ) <EOL> class CachedRemoteIndex ( RemoteIndex ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , app ) : <EOL> super ( CachedRemoteIndex , self ) . __init__ ( app ) <EOL> self . redis = app . redis <EOL> self . storage = app . remote_storage <EOL> self . versions_short_ttl = app . config [ "<STR_LIT>" ] <EOL> self . versions_long_ttl = app . config [ "<STR_LIT>" ] <EOL> self . logger = app . logger <EOL> def _key ( self , name ) : <EOL> return "<STR_LIT>" . format ( name ) <EOL> def _is_expired ( self , ttl ) : <EOL> """<STR_LIT>""" <EOL> if ttl == - <NUM_LIT:1> : <EOL> return False <EOL> if ttl == - <NUM_LIT:2> : <EOL> return False <EOL> age = max ( <NUM_LIT:0> , self . versions_long_ttl - ttl ) <EOL> return age >= self . versions_short_ttl <EOL> def _get_cached_index ( self , name ) : <EOL> """<STR_LIT>""" <EOL> versions = self . redis . get ( self . _key ( name ) ) <EOL> if versions is None : <EOL> self . logger . debug ( "<STR_LIT>" . format ( name ) ) <EOL> return None , False <EOL> ttl = self . redis . ttl ( self . _key ( name ) ) <EOL> self . logger . debug ( "<STR_LIT>" . format ( name , ttl ) ) <EOL> expired = self . _is_expired ( ttl ) <EOL> self . logger . debug ( "<STR_LIT>" . format ( name , expired ) ) <EOL> return loads ( versions ) , expired <EOL> def _save_negative_index ( self , name ) : <EOL> """<STR_LIT>""" <EOL> self . logger . debug ( "<STR_LIT>" . format ( name ) ) <EOL> self . redis . setex ( self . _key ( name ) , time = int ( self . versions_long_ttl ) , value = dumps ( { } ) ) <EOL> def _save_index ( self , name , versions ) : <EOL> self . logger . debug ( "<STR_LIT>" . format ( name ) ) <EOL> self . redis . setex ( self . _key ( name ) , time = int ( self . versions_long_ttl ) , value = dumps ( versions ) ) <EOL> def get_versions ( self , name ) : <EOL> """<STR_LIT>""" <EOL> self . logger . info ( "<STR_LIT>" . format ( name ) ) <EOL> cached_versions , cached_expired = self . _get_cached_index ( name ) <EOL> if cached_versions is not None and not cached_expired : <EOL> self . logger . debug ( "<STR_LIT>" . format ( name ) ) <EOL> return cached_versions <EOL> try : <EOL> computed_versions = super ( CachedRemoteIndex , self ) . get_versions ( name ) <EOL> except NotFoundError as error : <EOL> if error . status_code == codes . not_found : <EOL> self . _save_negative_index ( name ) <EOL> raise <EOL> elif cached_versions is None : <EOL> raise <EOL> else : <EOL> self . logger . debug ( "<STR_LIT>" . format ( cached_versions ) ) <EOL> return cached_versions <EOL> else : <EOL> self . _save_index ( name , computed_versions ) <EOL> self . logger . debug ( "<STR_LIT>" . format ( computed_versions ) ) <EOL> return computed_versions <EOL> def get_distribution ( self , location , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> cached = self . storage . read ( location ) <EOL> if cached is not None : <EOL> self . logger . debug ( "<STR_LIT>" . format ( location ) ) <EOL> return cached <EOL> content_data , content_type = super ( CachedRemoteIndex , self ) . get_distribution ( location , <EOL> ** kwargs ) <EOL> self . logger . debug ( "<STR_LIT>" . format ( location ) ) <EOL> self . storage . write ( location , content_data ) <EOL> return content_data , content_type <EOL> def get_absolute_path ( url , path ) : <EOL> """<STR_LIT>""" <EOL> url_parts = list ( urlsplit ( url ) ) <EOL> return abspath ( join ( url_parts [ <NUM_LIT:2> ] , path ) ) <EOL> def get_base_url ( url ) : <EOL> """<STR_LIT>""" <EOL> url_parts = list ( urlsplit ( url ) ) <EOL> return urlunsplit ( url_parts [ : <NUM_LIT:2> ] + [ "<STR_LIT>" ] * <NUM_LIT:3> ) <EOL> def get_request_location ( response , url ) : <EOL> """<STR_LIT>""" <EOL> url_parts = urlsplit ( url ) <EOL> scheme , netloc , path = url_parts . scheme , url_parts . netloc , url_parts . path <EOL> for redirect in response . history or [ response ] : <EOL> if "<STR_LIT:location>" not in redirect . headers : <EOL> continue <EOL> redirect_parts = urlsplit ( redirect . headers [ "<STR_LIT:location>" ] ) <EOL> if redirect_parts . scheme : <EOL> scheme = redirect_parts . scheme <EOL> if redirect_parts . netloc : <EOL> netloc = redirect_parts . netloc <EOL> if redirect_parts . path : <EOL> path = redirect_parts . path <EOL> return urlunsplit ( [ scheme , netloc , path ] + [ "<STR_LIT>" ] * <NUM_LIT:2> ) <EOL> def build_remote_path ( href , location ) : <EOL> """<STR_LIT>""" <EOL> if has_scheme ( href ) : <EOL> path = get_absolute_path ( href , "<STR_LIT>" ) <EOL> base_url = get_base_url ( href ) <EOL> else : <EOL> path = get_absolute_path ( location , href ) <EOL> base_url = get_base_url ( location ) <EOL> if "<STR_LIT:#>" in path : <EOL> base , fragment = path . split ( "<STR_LIT:#>" , <NUM_LIT:1> ) <EOL> return "<STR_LIT>" . format ( base , quote ( base_url , "<STR_LIT>" ) , fragment ) <EOL> else : <EOL> return "<STR_LIT>" . format ( path , quote ( base_url , "<STR_LIT>" ) ) <EOL> def has_scheme ( url ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> url_parts = urlsplit ( url ) <EOL> return bool ( url_parts . scheme ) <EOL> except : <EOL> return False <EOL> def iter_version_links ( html , name ) : <EOL> """<STR_LIT>""" <EOL> soup = BeautifulSoup ( html ) <EOL> for node in soup . findAll ( "<STR_LIT:a>" ) : <EOL> if node . get ( "<STR_LIT>" ) is None : <EOL> continue <EOL> try : <EOL> guessed_name , _ = guess_name_and_version ( node . text ) <EOL> except ValueError : <EOL> href = node [ "<STR_LIT>" ] <EOL> for extension in [ "<STR_LIT>" , "<STR_LIT>" ] : <EOL> if href . endswith ( extension ) : <EOL> yield basename ( href ) , href <EOL> break <EOL> else : <EOL> if node . get ( "<STR_LIT>" ) == "<STR_LIT>" : <EOL> yield href <EOL> else : <EOL> if guessed_name . replace ( "<STR_LIT:_>" , "<STR_LIT:->" ) . lower ( ) != name . replace ( "<STR_LIT:_>" , "<STR_LIT:->" ) . lower ( ) : <EOL> continue <EOL> yield node . text , node [ "<STR_LIT>" ] <EOL> def fetch_url ( url , timeout , logger ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> response = get ( url , timeout = timeout ) <EOL> except Timeout : <EOL> logger . info ( "<STR_LIT>" . format ( url ) ) <EOL> raise NotFoundError ( ) <EOL> except ConnectionError : <EOL> logger . info ( "<STR_LIT>" . format ( url ) ) <EOL> raise NotFoundError ( ) <EOL> if response . status_code != codes . ok : <EOL> logger . info ( "<STR_LIT>" . format ( response . status_code , url ) ) <EOL> raise NotFoundError ( response . status_code ) <EOL> return response </s>
