<s> """<STR_LIT>""" <EOL> import urlparse <EOL> import urllib <EOL> __all__ = [ "<STR_LIT>" ] <EOL> class RobotFileParser : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , url = '<STR_LIT>' ) : <EOL> self . entries = [ ] <EOL> self . default_entry = None <EOL> self . disallow_all = False <EOL> self . allow_all = False <EOL> self . set_url ( url ) <EOL> self . last_checked = <NUM_LIT:0> <EOL> def mtime ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . last_checked <EOL> def modified ( self ) : <EOL> """<STR_LIT>""" <EOL> import time <EOL> self . last_checked = time . time ( ) <EOL> def set_url ( self , url ) : <EOL> """<STR_LIT>""" <EOL> self . url = url <EOL> self . host , self . path = urlparse . urlparse ( url ) [ <NUM_LIT:1> : <NUM_LIT:3> ] <EOL> def read ( self ) : <EOL> """<STR_LIT>""" <EOL> opener = URLopener ( ) <EOL> f = opener . open ( self . url ) <EOL> lines = [ line . strip ( ) for line in f ] <EOL> f . close ( ) <EOL> self . errcode = opener . errcode <EOL> if self . errcode in ( <NUM_LIT> , <NUM_LIT> ) : <EOL> self . disallow_all = True <EOL> elif self . errcode >= <NUM_LIT> : <EOL> self . allow_all = True <EOL> elif self . errcode == <NUM_LIT:200> and lines : <EOL> self . parse ( lines ) <EOL> def _add_entry ( self , entry ) : <EOL> if "<STR_LIT:*>" in entry . useragents : <EOL> if self . default_entry is None : <EOL> self . default_entry = entry <EOL> else : <EOL> self . entries . append ( entry ) <EOL> def parse ( self , lines ) : <EOL> """<STR_LIT>""" <EOL> state = <NUM_LIT:0> <EOL> linenumber = <NUM_LIT:0> <EOL> entry = Entry ( ) <EOL> for line in lines : <EOL> linenumber += <NUM_LIT:1> <EOL> if not line : <EOL> if state == <NUM_LIT:1> : <EOL> entry = Entry ( ) <EOL> state = <NUM_LIT:0> <EOL> elif state == <NUM_LIT:2> : <EOL> self . _add_entry ( entry ) <EOL> entry = Entry ( ) <EOL> state = <NUM_LIT:0> <EOL> i = line . find ( '<STR_LIT:#>' ) <EOL> if i >= <NUM_LIT:0> : <EOL> line = line [ : i ] <EOL> line = line . strip ( ) <EOL> if not line : <EOL> continue <EOL> line = line . split ( '<STR_LIT::>' , <NUM_LIT:1> ) <EOL> if len ( line ) == <NUM_LIT:2> : <EOL> line [ <NUM_LIT:0> ] = line [ <NUM_LIT:0> ] . strip ( ) . lower ( ) <EOL> line [ <NUM_LIT:1> ] = urllib . unquote ( line [ <NUM_LIT:1> ] . strip ( ) ) <EOL> if line [ <NUM_LIT:0> ] == "<STR_LIT>" : <EOL> if state == <NUM_LIT:2> : <EOL> self . _add_entry ( entry ) <EOL> entry = Entry ( ) <EOL> entry . useragents . append ( line [ <NUM_LIT:1> ] ) <EOL> state = <NUM_LIT:1> <EOL> elif line [ <NUM_LIT:0> ] == "<STR_LIT>" : <EOL> if state != <NUM_LIT:0> : <EOL> entry . rulelines . append ( RuleLine ( line [ <NUM_LIT:1> ] , False ) ) <EOL> state = <NUM_LIT:2> <EOL> elif line [ <NUM_LIT:0> ] == "<STR_LIT>" : <EOL> if state != <NUM_LIT:0> : <EOL> entry . rulelines . append ( RuleLine ( line [ <NUM_LIT:1> ] , True ) ) <EOL> state = <NUM_LIT:2> <EOL> if state == <NUM_LIT:2> : <EOL> self . _add_entry ( entry ) <EOL> def can_fetch ( self , useragent , url ) : <EOL> """<STR_LIT>""" <EOL> if self . disallow_all : <EOL> return False <EOL> if self . allow_all : <EOL> return True <EOL> parsed_url = urlparse . urlparse ( urllib . unquote ( url ) ) <EOL> url = urlparse . urlunparse ( ( '<STR_LIT>' , '<STR_LIT>' , parsed_url . path , <EOL> parsed_url . params , parsed_url . query , parsed_url . fragment ) ) <EOL> url = urllib . quote ( url ) <EOL> if not url : <EOL> url = "<STR_LIT:/>" <EOL> for entry in self . entries : <EOL> if entry . applies_to ( useragent ) : <EOL> return entry . allowance ( url ) <EOL> if self . default_entry : <EOL> return self . default_entry . allowance ( url ) <EOL> return True <EOL> def __str__ ( self ) : <EOL> return '<STR_LIT>' . join ( [ str ( entry ) + "<STR_LIT:\n>" for entry in self . entries ] ) <EOL> class RuleLine : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , path , allowance ) : <EOL> if path == '<STR_LIT>' and not allowance : <EOL> allowance = True <EOL> self . path = urllib . quote ( path ) <EOL> self . allowance = allowance <EOL> def applies_to ( self , filename ) : <EOL> return self . path == "<STR_LIT:*>" or filename . startswith ( self . path ) <EOL> def __str__ ( self ) : <EOL> return ( self . allowance and "<STR_LIT>" or "<STR_LIT>" ) + "<STR_LIT>" + self . path <EOL> class Entry : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . useragents = [ ] <EOL> self . rulelines = [ ] <EOL> def __str__ ( self ) : <EOL> ret = [ ] <EOL> for agent in self . useragents : <EOL> ret . extend ( [ "<STR_LIT>" , agent , "<STR_LIT:\n>" ] ) <EOL> for line in self . rulelines : <EOL> ret . extend ( [ str ( line ) , "<STR_LIT:\n>" ] ) <EOL> return '<STR_LIT>' . join ( ret ) <EOL> def applies_to ( self , useragent ) : <EOL> """<STR_LIT>""" <EOL> useragent = useragent . split ( "<STR_LIT:/>" ) [ <NUM_LIT:0> ] . lower ( ) <EOL> for agent in self . useragents : <EOL> if agent == '<STR_LIT:*>' : <EOL> return True <EOL> agent = agent . lower ( ) <EOL> if agent in useragent : <EOL> return True <EOL> return False <EOL> def allowance ( self , filename ) : <EOL> """<STR_LIT>""" <EOL> for line in self . rulelines : <EOL> if line . applies_to ( filename ) : <EOL> return line . allowance <EOL> return True <EOL> class URLopener ( urllib . FancyURLopener ) : <EOL> def __init__ ( self , * args ) : <EOL> urllib . FancyURLopener . __init__ ( self , * args ) <EOL> self . errcode = <NUM_LIT:200> <EOL> def prompt_user_passwd ( self , host , realm ) : <EOL> return None , None <EOL> def http_error_default ( self , url , fp , errcode , errmsg , headers ) : <EOL> self . errcode = errcode <EOL> return urllib . FancyURLopener . http_error_default ( self , url , fp , errcode , <EOL> errmsg , headers ) </s>
