<s> __usage__ = """<STR_LIT>""" <EOL> import sys <EOL> if sys . hexversion < <NUM_LIT> : <EOL> print '<STR_LIT>' <EOL> print '<STR_LIT>' % sys . version <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> import fnmatch <EOL> import glob <EOL> import gzip <EOL> import hashlib <EOL> import os <EOL> import re <EOL> import stat <EOL> import time <EOL> import types <EOL> import urllib <EOL> import urlparse <EOL> import xml . sax <EOL> try : <EOL> testTrue = True <EOL> del testTrue <EOL> except NameError : <EOL> True = <NUM_LIT:1> <EOL> False = <NUM_LIT:0> <EOL> ENC_ASCII = '<STR_LIT>' <EOL> ENC_UTF8 = '<STR_LIT>' <EOL> ENC_IDNA = '<STR_LIT>' <EOL> ENC_ASCII_LIST = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ] <EOL> ENC_DEFAULT_LIST = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> MAXURLS_PER_SITEMAP = <NUM_LIT> <EOL> SITEINDEX_SUFFIX = '<STR_LIT>' <EOL> ACCESSLOG_CLF_PATTERN = re . compile ( <EOL> r'<STR_LIT>' <EOL> ) <EOL> LASTMOD_PATTERNS = map ( re . compile , [ <EOL> r'<STR_LIT>' , <EOL> r'<STR_LIT>' , <EOL> r'<STR_LIT>' , <EOL> r'<STR_LIT>' , <EOL> r'<STR_LIT>' , <EOL> r'<STR_LIT>' , <EOL> r'<STR_LIT>' , <EOL> ] ) <EOL> CHANGEFREQ_PATTERNS = [ <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' <EOL> ] <EOL> SITEINDEX_HEADER = '<STR_LIT>' '<STR_LIT>' '<STR_LIT>' '<STR_LIT>' '<STR_LIT>' '<STR_LIT>' '<STR_LIT>' <EOL> SITEINDEX_FOOTER = '<STR_LIT>' <EOL> SITEINDEX_ENTRY = '<STR_LIT>' '<STR_LIT>' '<STR_LIT>' '<STR_LIT>' <EOL> SITEMAP_HEADER = '<STR_LIT>' '<STR_LIT>' '<STR_LIT>' '<STR_LIT>' '<STR_LIT>' '<STR_LIT>' '<STR_LIT>' <EOL> SITEMAP_FOOTER = '<STR_LIT>' <EOL> SITEURL_XML_PREFIX = '<STR_LIT>' <EOL> SITEURL_XML_SUFFIX = '<STR_LIT>' <EOL> NOTIFICATION_SITES = [ <EOL> ( '<STR_LIT:http>' , '<STR_LIT>' , '<STR_LIT>' , { } , '<STR_LIT>' , '<STR_LIT>' ) <EOL> ] <EOL> class Error ( Exception ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> class SchemaError ( Error ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> class Encoder : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . _user = None <EOL> self . _learned = [ ] <EOL> self . _widefiles = False <EOL> try : <EOL> self . _widefiles = os . path . supports_unicode_filenames <EOL> except AttributeError : <EOL> try : <EOL> self . _widefiles = sys . getwindowsversion ( ) == os . VER_PLATFORM_WIN32_NT <EOL> except AttributeError : <EOL> pass <EOL> try : <EOL> encoding = sys . getfilesystemencoding ( ) <EOL> if encoding and not ( encoding . upper ( ) in ENC_ASCII_LIST ) : <EOL> self . _learned = [ encoding ] <EOL> except AttributeError : <EOL> pass <EOL> if not self . _learned : <EOL> encoding = sys . getdefaultencoding ( ) <EOL> if encoding and not ( encoding . upper ( ) in ENC_ASCII_LIST ) : <EOL> self . _learned = [ encoding ] <EOL> if not self . _learned : <EOL> self . _learned = ENC_DEFAULT_LIST <EOL> def SetUserEncoding ( self , encoding ) : <EOL> self . _user = encoding <EOL> def NarrowText ( self , text , encoding ) : <EOL> """<STR_LIT>""" <EOL> if type ( text ) != types . UnicodeType : <EOL> return text <EOL> if encoding : <EOL> try : <EOL> result = text . encode ( encoding ) <EOL> if not encoding in self . _learned : <EOL> self . _learned . append ( encoding ) <EOL> return result <EOL> except UnicodeError : <EOL> pass <EOL> except LookupError : <EOL> output . Warn ( '<STR_LIT>' % encoding ) <EOL> if self . _user : <EOL> try : <EOL> return text . encode ( self . _user ) <EOL> except UnicodeError : <EOL> pass <EOL> except LookupError : <EOL> temp = self . _user <EOL> self . _user = None <EOL> output . Warn ( '<STR_LIT>' % temp ) <EOL> while self . _learned : <EOL> try : <EOL> return text . encode ( self . _learned [ <NUM_LIT:0> ] ) <EOL> except : <EOL> del self . _learned [ <NUM_LIT:0> ] <EOL> try : <EOL> return text . encode ( ENC_UTF8 ) <EOL> except UnicodeError : <EOL> pass <EOL> return text . encode ( ENC_ASCII , '<STR_LIT:ignore>' ) <EOL> def MaybeNarrowPath ( self , text ) : <EOL> """<STR_LIT>""" <EOL> if self . _widefiles : <EOL> return text <EOL> return self . NarrowText ( text , None ) <EOL> def WidenText ( self , text , encoding ) : <EOL> """<STR_LIT>""" <EOL> if type ( text ) != types . StringType : <EOL> return text <EOL> if encoding : <EOL> try : <EOL> result = unicode ( text , encoding ) <EOL> if not encoding in self . _learned : <EOL> self . _learned . append ( encoding ) <EOL> return result <EOL> except UnicodeError : <EOL> pass <EOL> except LookupError : <EOL> output . Warn ( '<STR_LIT>' % encoding ) <EOL> if self . _user : <EOL> try : <EOL> return unicode ( text , self . _user ) <EOL> except UnicodeError : <EOL> pass <EOL> except LookupError : <EOL> temp = self . _user <EOL> self . _user = None <EOL> output . Warn ( '<STR_LIT>' % temp ) <EOL> while self . _learned : <EOL> try : <EOL> return unicode ( text , self . _learned [ <NUM_LIT:0> ] ) <EOL> except : <EOL> del self . _learned [ <NUM_LIT:0> ] <EOL> try : <EOL> return unicode ( text , ENC_UTF8 ) <EOL> except UnicodeError : <EOL> pass <EOL> output . Warn ( '<STR_LIT>' % text ) <EOL> if not self . _user : <EOL> output . Warn ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> return text . decode ( ENC_ASCII , '<STR_LIT:ignore>' ) <EOL> encoder = Encoder ( ) <EOL> class Output : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . num_errors = <NUM_LIT:0> <EOL> self . num_warns = <NUM_LIT:0> <EOL> self . _errors_shown = { } <EOL> self . _warns_shown = { } <EOL> self . _verbose = <NUM_LIT:0> <EOL> def Log ( self , text , level ) : <EOL> """<STR_LIT>""" <EOL> if text : <EOL> text = encoder . NarrowText ( text , None ) <EOL> if self . _verbose >= level : <EOL> print text <EOL> def Warn ( self , text ) : <EOL> """<STR_LIT>""" <EOL> if text : <EOL> text = encoder . NarrowText ( text , None ) <EOL> hash = hashlib . md5 ( text ) . hexdigest ( ) <EOL> if not self . _warns_shown . has_key ( hash ) : <EOL> self . _warns_shown [ hash ] = <NUM_LIT:1> <EOL> print '<STR_LIT>' + text <EOL> else : <EOL> self . Log ( '<STR_LIT>' + text , <NUM_LIT:3> ) <EOL> self . num_warns = self . num_warns + <NUM_LIT:1> <EOL> def Error ( self , text ) : <EOL> """<STR_LIT>""" <EOL> if text : <EOL> text = encoder . NarrowText ( text , None ) <EOL> hash = hashlib . md5 ( text ) . hexdigest ( ) <EOL> if not self . _errors_shown . has_key ( hash ) : <EOL> self . _errors_shown [ hash ] = <NUM_LIT:1> <EOL> print '<STR_LIT>' + text <EOL> else : <EOL> self . Log ( '<STR_LIT>' + text , <NUM_LIT:3> ) <EOL> self . num_errors = self . num_errors + <NUM_LIT:1> <EOL> def Fatal ( self , text ) : <EOL> """<STR_LIT>""" <EOL> if text : <EOL> text = encoder . NarrowText ( text , None ) <EOL> print '<STR_LIT>' + text <EOL> else : <EOL> print '<STR_LIT>' <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> def SetVerbose ( self , level ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> if type ( level ) != types . IntType : <EOL> level = int ( level ) <EOL> if ( level >= <NUM_LIT:0> ) and ( level <= <NUM_LIT:3> ) : <EOL> self . _verbose = level <EOL> return <EOL> except ValueError : <EOL> pass <EOL> self . Error ( '<STR_LIT>' % level ) <EOL> output = Output ( ) <EOL> class URL ( object ) : <EOL> """<STR_LIT>""" <EOL> __slots__ = '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' <EOL> def __init__ ( self ) : <EOL> self . loc = None <EOL> self . lastmod = None <EOL> self . changefreq = None <EOL> self . priority = None <EOL> def __cmp__ ( self , other ) : <EOL> if self . loc < other . loc : <EOL> return - <NUM_LIT:1> <EOL> if self . loc > other . loc : <EOL> return <NUM_LIT:1> <EOL> return <NUM_LIT:0> <EOL> def TrySetAttribute ( self , attribute , value ) : <EOL> """<STR_LIT>""" <EOL> if attribute == '<STR_LIT>' : <EOL> self . loc = self . Canonicalize ( value ) <EOL> else : <EOL> try : <EOL> setattr ( self , attribute , value ) <EOL> except AttributeError : <EOL> output . Warn ( '<STR_LIT>' % attribute ) <EOL> def IsAbsolute ( loc ) : <EOL> """<STR_LIT>""" <EOL> if not loc : <EOL> return False <EOL> narrow = encoder . NarrowText ( loc , None ) <EOL> ( scheme , netloc , path , query , frag ) = urlparse . urlsplit ( narrow ) <EOL> if ( not scheme ) or ( not netloc ) : <EOL> return False <EOL> return True <EOL> IsAbsolute = staticmethod ( IsAbsolute ) <EOL> def Canonicalize ( loc ) : <EOL> """<STR_LIT>""" <EOL> if not loc : <EOL> return loc <EOL> narrow = encoder . NarrowText ( loc , None ) <EOL> ( scheme , netloc , path , query , frag ) = urlparse . urlsplit ( narrow ) <EOL> unr = '<STR_LIT>' <EOL> sub = '<STR_LIT>' <EOL> netloc = urllib . quote ( netloc , unr + sub + '<STR_LIT>' ) <EOL> path = urllib . quote ( path , unr + sub + '<STR_LIT>' ) <EOL> query = urllib . quote ( query , unr + sub + '<STR_LIT>' ) <EOL> frag = urllib . quote ( frag , unr + sub + '<STR_LIT>' ) <EOL> try : <EOL> ( ignore , widenetloc , ignore , ignore , ignore ) = urlparse . urlsplit ( loc ) <EOL> for c in widenetloc : <EOL> if c >= unichr ( <NUM_LIT> ) : <EOL> netloc = widenetloc . encode ( ENC_IDNA ) <EOL> netloc = urllib . quote ( netloc , unr + sub + '<STR_LIT>' ) <EOL> break <EOL> except UnicodeError : <EOL> pass <EOL> except LookupError : <EOL> output . Warn ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> bad_netloc = False <EOL> if '<STR_LIT:%>' in netloc : <EOL> bad_netloc = True <EOL> narrow = urlparse . urlunsplit ( ( scheme , netloc , path , query , frag ) ) <EOL> HEXDIG = '<STR_LIT>' <EOL> list = narrow . split ( '<STR_LIT:%>' ) <EOL> narrow = list [ <NUM_LIT:0> ] <EOL> del list [ <NUM_LIT:0> ] <EOL> for item in list : <EOL> if ( len ( item ) >= <NUM_LIT:2> ) and ( item [ <NUM_LIT:0> ] in HEXDIG ) and ( item [ <NUM_LIT:1> ] in HEXDIG ) : <EOL> narrow = narrow + '<STR_LIT:%>' + item <EOL> else : <EOL> narrow = narrow + '<STR_LIT>' + item <EOL> if bad_netloc : <EOL> output . Warn ( '<STR_LIT>' <EOL> + narrow ) <EOL> return narrow <EOL> Canonicalize = staticmethod ( Canonicalize ) <EOL> def Validate ( self , base_url , allow_fragment ) : <EOL> """<STR_LIT>""" <EOL> assert type ( base_url ) == types . StringType <EOL> if not self . loc : <EOL> output . Warn ( '<STR_LIT>' ) <EOL> return False <EOL> if allow_fragment : <EOL> self . loc = urlparse . urljoin ( base_url , self . loc ) <EOL> if not self . loc . startswith ( base_url ) : <EOL> output . Warn ( '<STR_LIT>' % <EOL> self . loc ) <EOL> self . loc = None <EOL> return False <EOL> if self . lastmod : <EOL> match = False <EOL> self . lastmod = self . lastmod . upper ( ) <EOL> for pattern in LASTMOD_PATTERNS : <EOL> match = pattern . match ( self . lastmod ) <EOL> if match : <EOL> break <EOL> if not match : <EOL> output . Warn ( '<STR_LIT>' <EOL> '<STR_LIT>' % ( self . lastmod , self . loc ) ) <EOL> self . lastmod = None <EOL> if self . changefreq : <EOL> match = False <EOL> self . changefreq = self . changefreq . lower ( ) <EOL> for pattern in CHANGEFREQ_PATTERNS : <EOL> if self . changefreq == pattern : <EOL> match = True <EOL> break <EOL> if not match : <EOL> output . Warn ( '<STR_LIT>' <EOL> '<STR_LIT>' % ( self . changefreq , self . loc ) ) <EOL> self . changefreq = None <EOL> if self . priority : <EOL> priority = - <NUM_LIT:1.0> <EOL> try : <EOL> priority = float ( self . priority ) <EOL> except ValueError : <EOL> pass <EOL> if ( priority < <NUM_LIT:0.0> ) or ( priority > <NUM_LIT:1.0> ) : <EOL> output . Warn ( '<STR_LIT>' <EOL> '<STR_LIT>' % ( self . priority , self . loc ) ) <EOL> self . priority = None <EOL> return True <EOL> def MakeHash ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . loc : <EOL> return None <EOL> if self . loc . endswith ( '<STR_LIT:/>' ) : <EOL> return hashlib . md5 ( self . loc [ : - <NUM_LIT:1> ] ) . hexdigest ( ) <EOL> return hashlib . md5 ( self . loc ) . hexdigest ( ) <EOL> def Log ( self , prefix = '<STR_LIT>' , level = <NUM_LIT:3> ) : <EOL> """<STR_LIT>""" <EOL> out = prefix + '<STR_LIT::>' <EOL> for attribute in self . __slots__ : <EOL> value = getattr ( self , attribute ) <EOL> if not value : <EOL> value = '<STR_LIT>' <EOL> out = out + ( '<STR_LIT>' % ( attribute , value ) ) <EOL> output . Log ( '<STR_LIT:%s>' % encoder . NarrowText ( out , None ) , level ) <EOL> def WriteXML ( self , file ) : <EOL> """<STR_LIT>""" <EOL> if not self . loc : <EOL> return <EOL> out = SITEURL_XML_PREFIX <EOL> for attribute in self . __slots__ : <EOL> value = getattr ( self , attribute ) <EOL> if value : <EOL> if type ( value ) == types . UnicodeType : <EOL> value = encoder . NarrowText ( value , None ) <EOL> elif type ( value ) != types . StringType : <EOL> value = str ( value ) <EOL> value = xml . sax . saxutils . escape ( value ) <EOL> out = out + ( '<STR_LIT>' % ( attribute , value , attribute ) ) <EOL> out = out + SITEURL_XML_SUFFIX <EOL> file . write ( out ) <EOL> class Filter : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , attributes ) : <EOL> self . _wildcard = None <EOL> self . _regexp = None <EOL> self . _pass = False <EOL> if not ValidateAttributes ( '<STR_LIT>' , attributes , <EOL> ( '<STR_LIT>' , '<STR_LIT:type>' , '<STR_LIT:action>' ) ) : <EOL> return <EOL> num_errors = output . num_errors <EOL> pattern = attributes . get ( '<STR_LIT>' ) <EOL> type = attributes . get ( '<STR_LIT:type>' , '<STR_LIT>' ) <EOL> action = attributes . get ( '<STR_LIT:action>' , '<STR_LIT>' ) <EOL> if type : <EOL> type = type . lower ( ) <EOL> if action : <EOL> action = action . lower ( ) <EOL> if not pattern : <EOL> output . Error ( '<STR_LIT>' ) <EOL> elif ( not type ) or ( ( type != '<STR_LIT>' ) and ( type != '<STR_LIT>' ) ) : <EOL> output . Error ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> elif ( action != '<STR_LIT>' ) and ( action != '<STR_LIT>' ) : <EOL> output . Error ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if action == '<STR_LIT>' : <EOL> self . _pass = False <EOL> elif action == '<STR_LIT>' : <EOL> self . _pass = True <EOL> if type == '<STR_LIT>' : <EOL> self . _wildcard = pattern <EOL> elif type == '<STR_LIT>' : <EOL> try : <EOL> self . _regexp = re . compile ( pattern ) <EOL> except re . error : <EOL> output . Error ( '<STR_LIT>' % pattern ) <EOL> if num_errors == output . num_errors : <EOL> output . Log ( '<STR_LIT>' % <EOL> ( action , type , pattern ) , <NUM_LIT:2> ) <EOL> def Apply ( self , url ) : <EOL> """<STR_LIT>""" <EOL> if ( not url ) or ( not url . loc ) : <EOL> return None <EOL> if self . _wildcard : <EOL> if fnmatch . fnmatchcase ( url . loc , self . _wildcard ) : <EOL> return self . _pass <EOL> return None <EOL> if self . _regexp : <EOL> if self . _regexp . search ( url . loc ) : <EOL> return self . _pass <EOL> return None <EOL> assert False <EOL> class InputURL : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , attributes ) : <EOL> self . _url = None <EOL> if not ValidateAttributes ( '<STR_LIT>' , attributes , <EOL> ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) ) : <EOL> return <EOL> url = URL ( ) <EOL> for attr in attributes . keys ( ) : <EOL> if attr == '<STR_LIT>' : <EOL> url . TrySetAttribute ( '<STR_LIT>' , attributes [ attr ] ) <EOL> else : <EOL> url . TrySetAttribute ( attr , attributes [ attr ] ) <EOL> if not url . loc : <EOL> output . Error ( '<STR_LIT>' ) <EOL> return <EOL> self . _url = url <EOL> output . Log ( '<STR_LIT>' % self . _url . loc , <NUM_LIT:2> ) <EOL> def ProduceURLs ( self , consumer ) : <EOL> """<STR_LIT>""" <EOL> if self . _url : <EOL> consumer ( self . _url , True ) <EOL> class InputURLList : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , attributes ) : <EOL> self . _path = None <EOL> self . _encoding = None <EOL> if not ValidateAttributes ( '<STR_LIT>' , attributes , ( '<STR_LIT:path>' , '<STR_LIT>' ) ) : <EOL> return <EOL> self . _path = attributes . get ( '<STR_LIT:path>' ) <EOL> self . _encoding = attributes . get ( '<STR_LIT>' , ENC_UTF8 ) <EOL> if self . _path : <EOL> self . _path = encoder . MaybeNarrowPath ( self . _path ) <EOL> if os . path . isfile ( self . _path ) : <EOL> output . Log ( '<STR_LIT>' % self . _path , <NUM_LIT:2> ) <EOL> else : <EOL> output . Error ( '<STR_LIT>' % self . _path ) <EOL> self . _path = None <EOL> else : <EOL> output . Error ( '<STR_LIT>' ) <EOL> def ProduceURLs ( self , consumer ) : <EOL> """<STR_LIT>""" <EOL> ( frame , file ) = OpenFileForRead ( self . _path , '<STR_LIT>' ) <EOL> if not file : <EOL> return <EOL> linenum = <NUM_LIT:0> <EOL> for line in file . readlines ( ) : <EOL> linenum = linenum + <NUM_LIT:1> <EOL> if self . _encoding : <EOL> line = encoder . WidenText ( line , self . _encoding ) <EOL> line = line . strip ( ) <EOL> if ( not line ) or line [ <NUM_LIT:0> ] == '<STR_LIT:#>' : <EOL> continue <EOL> url = URL ( ) <EOL> cols = line . split ( '<STR_LIT:U+0020>' ) <EOL> for i in range ( <NUM_LIT:0> , len ( cols ) ) : <EOL> cols [ i ] = cols [ i ] . strip ( ) <EOL> url . TrySetAttribute ( '<STR_LIT>' , cols [ <NUM_LIT:0> ] ) <EOL> for i in range ( <NUM_LIT:1> , len ( cols ) ) : <EOL> if cols [ i ] : <EOL> try : <EOL> ( attr_name , attr_val ) = cols [ i ] . split ( '<STR_LIT:=>' , <NUM_LIT:1> ) <EOL> url . TrySetAttribute ( attr_name , attr_val ) <EOL> except ValueError : <EOL> output . Warn ( '<STR_LIT>' % <EOL> ( linenum , cols [ i ] ) ) <EOL> consumer ( url , False ) <EOL> file . close ( ) <EOL> if frame : <EOL> frame . close ( ) <EOL> class InputDirectory : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , attributes , base_url ) : <EOL> self . _path = None <EOL> self . _url = None <EOL> self . _default_file = None <EOL> if not ValidateAttributes ( '<STR_LIT>' , attributes , ( '<STR_LIT:path>' , '<STR_LIT:url>' , <EOL> '<STR_LIT>' ) ) : <EOL> return <EOL> path = attributes . get ( '<STR_LIT:path>' ) <EOL> if not path : <EOL> output . Error ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> return <EOL> path = encoder . MaybeNarrowPath ( path ) <EOL> if not path . endswith ( os . sep ) : <EOL> path = path + os . sep <EOL> if not os . path . isdir ( path ) : <EOL> output . Error ( '<STR_LIT>' % path ) <EOL> return <EOL> url = attributes . get ( '<STR_LIT:url>' ) <EOL> if not url : <EOL> output . Error ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> return <EOL> url = URL . Canonicalize ( url ) <EOL> if not url . endswith ( '<STR_LIT:/>' ) : <EOL> url = url + '<STR_LIT:/>' <EOL> if not url . startswith ( base_url ) : <EOL> url = urlparse . urljoin ( base_url , url ) <EOL> if not url . startswith ( base_url ) : <EOL> output . Error ( '<STR_LIT>' <EOL> '<STR_LIT>' % ( url , base_url ) ) <EOL> return <EOL> file = attributes . get ( '<STR_LIT>' ) <EOL> if file : <EOL> file = encoder . MaybeNarrowPath ( file ) <EOL> if os . sep in file : <EOL> output . Error ( '<STR_LIT>' <EOL> % file ) <EOL> file = None <EOL> self . _path = path <EOL> self . _url = url <EOL> self . _default_file = file <EOL> if file : <EOL> output . Log ( '<STR_LIT>' <EOL> % ( path , url , file ) , <NUM_LIT:2> ) <EOL> else : <EOL> output . Log ( '<STR_LIT>' <EOL> % ( path , url ) , <NUM_LIT:2> ) <EOL> def ProduceURLs ( self , consumer ) : <EOL> """<STR_LIT>""" <EOL> if not self . _path : <EOL> return <EOL> root_path = self . _path <EOL> root_URL = self . _url <EOL> root_file = self . _default_file <EOL> def PerFile ( dirpath , name ) : <EOL> """<STR_LIT>""" <EOL> url = URL ( ) <EOL> isdir = False <EOL> try : <EOL> if name : <EOL> path = os . path . join ( dirpath , name ) <EOL> else : <EOL> path = dirpath <EOL> isdir = os . path . isdir ( path ) <EOL> time = None <EOL> if isdir and root_file : <EOL> file = os . path . join ( path , root_file ) <EOL> try : <EOL> time = os . stat ( file ) [ stat . ST_MTIME ] ; <EOL> except OSError : <EOL> pass <EOL> if not time : <EOL> time = os . stat ( path ) [ stat . ST_MTIME ] ; <EOL> url . lastmod = TimestampISO8601 ( time ) <EOL> except OSError : <EOL> pass <EOL> except ValueError : <EOL> pass <EOL> middle = dirpath [ len ( root_path ) : ] <EOL> if os . sep != '<STR_LIT:/>' : <EOL> middle = middle . replace ( os . sep , '<STR_LIT:/>' ) <EOL> if middle : <EOL> middle = middle + '<STR_LIT:/>' <EOL> if name : <EOL> middle = middle + name <EOL> if isdir : <EOL> middle = middle + '<STR_LIT:/>' <EOL> url . TrySetAttribute ( '<STR_LIT>' , root_URL + encoder . WidenText ( middle , None ) ) <EOL> if name and ( root_file == name ) : <EOL> url . Log ( prefix = '<STR_LIT>' , level = <NUM_LIT:2> ) <EOL> return <EOL> consumer ( url , False ) <EOL> def PerDirectory ( ignore , dirpath , namelist ) : <EOL> """<STR_LIT>""" <EOL> ignore = ignore <EOL> if not dirpath . startswith ( root_path ) : <EOL> output . Warn ( '<STR_LIT>' <EOL> '<STR_LIT:%s>' % dirpath ) <EOL> return <EOL> for name in namelist : <EOL> PerFile ( dirpath , name ) <EOL> output . Log ( '<STR_LIT>' % self . _path , <NUM_LIT:1> ) <EOL> PerFile ( self . _path , None ) <EOL> os . path . walk ( self . _path , PerDirectory , None ) <EOL> class InputAccessLog : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , attributes ) : <EOL> self . _path = None <EOL> self . _encoding = None <EOL> self . _is_elf = False <EOL> self . _is_clf = False <EOL> self . _elf_status = - <NUM_LIT:1> <EOL> self . _elf_method = - <NUM_LIT:1> <EOL> self . _elf_uri = - <NUM_LIT:1> <EOL> self . _elf_urifrag1 = - <NUM_LIT:1> <EOL> self . _elf_urifrag2 = - <NUM_LIT:1> <EOL> if not ValidateAttributes ( '<STR_LIT>' , attributes , ( '<STR_LIT:path>' , '<STR_LIT>' ) ) : <EOL> return <EOL> self . _path = attributes . get ( '<STR_LIT:path>' ) <EOL> self . _encoding = attributes . get ( '<STR_LIT>' , ENC_UTF8 ) <EOL> if self . _path : <EOL> self . _path = encoder . MaybeNarrowPath ( self . _path ) <EOL> if os . path . isfile ( self . _path ) : <EOL> output . Log ( '<STR_LIT>' % self . _path , <NUM_LIT:2> ) <EOL> else : <EOL> output . Error ( '<STR_LIT>' % self . _path ) <EOL> self . _path = None <EOL> else : <EOL> output . Error ( '<STR_LIT>' ) <EOL> def RecognizeELFLine ( self , line ) : <EOL> """<STR_LIT>""" <EOL> if not line . startswith ( '<STR_LIT>' ) : <EOL> return False <EOL> fields = line . split ( '<STR_LIT:U+0020>' ) <EOL> del fields [ <NUM_LIT:0> ] <EOL> for i in range ( <NUM_LIT:0> , len ( fields ) ) : <EOL> field = fields [ i ] . strip ( ) <EOL> if field == '<STR_LIT>' : <EOL> self . _elf_status = i <EOL> elif field == '<STR_LIT>' : <EOL> self . _elf_method = i <EOL> elif field == '<STR_LIT>' : <EOL> self . _elf_uri = i <EOL> elif field == '<STR_LIT>' : <EOL> self . _elf_urifrag1 = i <EOL> elif field == '<STR_LIT>' : <EOL> self . _elf_urifrag2 = i <EOL> output . Log ( '<STR_LIT>' , <NUM_LIT:2> ) <EOL> return True <EOL> def GetELFLine ( self , line ) : <EOL> """<STR_LIT>""" <EOL> fields = line . split ( '<STR_LIT:U+0020>' ) <EOL> count = len ( fields ) <EOL> if self . _elf_status >= <NUM_LIT:0> : <EOL> if self . _elf_status >= count : <EOL> return None <EOL> if not fields [ self . _elf_status ] . strip ( ) == '<STR_LIT>' : <EOL> return None <EOL> if self . _elf_method >= <NUM_LIT:0> : <EOL> if self . _elf_method >= count : <EOL> return None <EOL> if not fields [ self . _elf_method ] . strip ( ) in ( '<STR_LIT>' , '<STR_LIT:GET>' ) : <EOL> return None <EOL> if self . _elf_uri >= <NUM_LIT:0> : <EOL> if self . _elf_uri >= count : <EOL> return None <EOL> url = fields [ self . _elf_uri ] . strip ( ) <EOL> if url != '<STR_LIT:->' : <EOL> return url <EOL> if self . _elf_urifrag1 >= <NUM_LIT:0> : <EOL> if self . _elf_urifrag1 >= count or self . _elf_urifrag2 >= count : <EOL> return None <EOL> urlfrag1 = fields [ self . _elf_urifrag1 ] . strip ( ) <EOL> urlfrag2 = None <EOL> if self . _elf_urifrag2 >= <NUM_LIT:0> : <EOL> urlfrag2 = fields [ self . _elf_urifrag2 ] <EOL> if urlfrag1 and ( urlfrag1 != '<STR_LIT:->' ) : <EOL> if urlfrag2 and ( urlfrag2 != '<STR_LIT:->' ) : <EOL> urlfrag1 = urlfrag1 + '<STR_LIT:?>' + urlfrag2 <EOL> return urlfrag1 <EOL> return None <EOL> def RecognizeCLFLine ( self , line ) : <EOL> """<STR_LIT>""" <EOL> match = ACCESSLOG_CLF_PATTERN . match ( line ) <EOL> recognize = match and ( match . group ( <NUM_LIT:1> ) in ( '<STR_LIT>' , '<STR_LIT:GET>' ) ) <EOL> if recognize : <EOL> output . Log ( '<STR_LIT>' , <NUM_LIT:2> ) <EOL> return recognize <EOL> def GetCLFLine ( self , line ) : <EOL> """<STR_LIT>""" <EOL> match = ACCESSLOG_CLF_PATTERN . match ( line ) <EOL> if match : <EOL> request = match . group ( <NUM_LIT:1> ) <EOL> if request in ( '<STR_LIT>' , '<STR_LIT:GET>' ) : <EOL> return match . group ( <NUM_LIT:2> ) <EOL> return None <EOL> def ProduceURLs ( self , consumer ) : <EOL> """<STR_LIT>""" <EOL> ( frame , file ) = OpenFileForRead ( self . _path , '<STR_LIT>' ) <EOL> if not file : <EOL> return <EOL> for line in file . readlines ( ) : <EOL> if self . _encoding : <EOL> line = encoder . WidenText ( line , self . _encoding ) <EOL> line = line . strip ( ) <EOL> if ( not self . _is_clf ) and ( not self . _is_elf ) : <EOL> self . _is_elf = self . RecognizeELFLine ( line ) <EOL> self . _is_clf = self . RecognizeCLFLine ( line ) <EOL> match = None <EOL> if self . _is_elf : <EOL> match = self . GetELFLine ( line ) <EOL> elif self . _is_clf : <EOL> match = self . GetCLFLine ( line ) <EOL> if not match : <EOL> continue <EOL> url = URL ( ) <EOL> url . TrySetAttribute ( '<STR_LIT>' , match ) <EOL> consumer ( url , True ) <EOL> file . close ( ) <EOL> if frame : <EOL> frame . close ( ) <EOL> class InputSitemap ( xml . sax . handler . ContentHandler ) : <EOL> """<STR_LIT>""" <EOL> class _ContextBase ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , subtags ) : <EOL> """<STR_LIT>""" <EOL> self . _allowed_tags = subtags <EOL> self . _last_tag = None <EOL> def AcceptTag ( self , tag ) : <EOL> """<STR_LIT>""" <EOL> valid = tag in self . _allowed_tags <EOL> if valid : <EOL> self . _last_tag = tag <EOL> else : <EOL> self . _last_tag = None <EOL> return valid <EOL> def AcceptText ( self , text ) : <EOL> """<STR_LIT>""" <EOL> return False <EOL> def Open ( self ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def Close ( self ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def Return ( self , result ) : <EOL> """<STR_LIT>""" <EOL> if result : <EOL> raise NotImplementedError <EOL> class _ContextUrlSet ( _ContextBase ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> InputSitemap . _ContextBase . __init__ ( self , ( '<STR_LIT:url>' , ) ) <EOL> class _ContextUrl ( _ContextBase ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , consumer ) : <EOL> """<STR_LIT>""" <EOL> InputSitemap . _ContextBase . __init__ ( self , URL . __slots__ ) <EOL> self . _url = None <EOL> self . _consumer = consumer <EOL> def Open ( self ) : <EOL> """<STR_LIT>""" <EOL> assert not self . _url <EOL> self . _url = URL ( ) <EOL> def Close ( self ) : <EOL> """<STR_LIT>""" <EOL> assert self . _url <EOL> self . _consumer ( self . _url , False ) <EOL> self . _url = None <EOL> def Return ( self , result ) : <EOL> """<STR_LIT>""" <EOL> assert self . _url <EOL> if result : <EOL> self . _url . TrySetAttribute ( self . _last_tag , result ) <EOL> class _ContextSitemapIndex ( _ContextBase ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> InputSitemap . _ContextBase . __init__ ( self , ( '<STR_LIT>' , ) ) <EOL> self . _loclist = [ ] <EOL> def Open ( self ) : <EOL> """<STR_LIT>""" <EOL> assert not self . _loclist <EOL> def Close ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . _loclist : <EOL> temp = self . _loclist <EOL> self . _loclist = [ ] <EOL> return temp <EOL> def Return ( self , result ) : <EOL> """<STR_LIT>""" <EOL> if result : <EOL> self . _loclist . append ( result ) <EOL> class _ContextSitemap ( _ContextBase ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> InputSitemap . _ContextBase . __init__ ( self , ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> self . _loc = None <EOL> def Open ( self ) : <EOL> """<STR_LIT>""" <EOL> assert not self . _loc <EOL> def Close ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . _loc : <EOL> temp = self . _loc <EOL> self . _loc = None <EOL> return temp <EOL> output . Warn ( '<STR_LIT>' ) <EOL> def Return ( self , result ) : <EOL> """<STR_LIT>""" <EOL> if result and ( self . _last_tag == '<STR_LIT>' ) : <EOL> self . _loc = result <EOL> class _ContextValue ( _ContextBase ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> InputSitemap . _ContextBase . __init__ ( self , ( ) ) <EOL> self . _text = None <EOL> def AcceptText ( self , text ) : <EOL> """<STR_LIT>""" <EOL> if self . _text : <EOL> self . _text = self . _text + text <EOL> else : <EOL> self . _text = text <EOL> return True <EOL> def Open ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _text = None <EOL> def Close ( self ) : <EOL> """<STR_LIT>""" <EOL> text = self . _text <EOL> self . _text = None <EOL> if text : <EOL> text = text . strip ( ) <EOL> return text <EOL> def __init__ ( self , attributes ) : <EOL> """<STR_LIT>""" <EOL> xml . sax . handler . ContentHandler . __init__ ( self ) <EOL> self . _pathlist = None <EOL> self . _current = - <NUM_LIT:1> <EOL> self . _contexts = None <EOL> self . _contexts_idx = None <EOL> self . _contexts_stm = None <EOL> if not ValidateAttributes ( '<STR_LIT>' , attributes , [ '<STR_LIT:path>' ] ) : <EOL> return <EOL> path = attributes . get ( '<STR_LIT:path>' ) <EOL> if path : <EOL> path = encoder . MaybeNarrowPath ( path ) <EOL> if os . path . isfile ( path ) : <EOL> output . Log ( '<STR_LIT>' % path , <NUM_LIT:2> ) <EOL> self . _pathlist = [ path ] <EOL> else : <EOL> output . Error ( '<STR_LIT>' % path ) <EOL> else : <EOL> output . Error ( '<STR_LIT>' ) <EOL> def ProduceURLs ( self , consumer ) : <EOL> """<STR_LIT>""" <EOL> self . _contexts_idx = [ InputSitemap . _ContextSitemapIndex ( ) , <EOL> InputSitemap . _ContextSitemap ( ) , <EOL> InputSitemap . _ContextValue ( ) ] <EOL> self . _contexts_stm = [ InputSitemap . _ContextUrlSet ( ) , <EOL> InputSitemap . _ContextUrl ( consumer ) , <EOL> InputSitemap . _ContextValue ( ) ] <EOL> assert self . _pathlist <EOL> path = self . _pathlist [ <NUM_LIT:0> ] <EOL> self . _contexts = None <EOL> self . _ProcessFile ( path ) <EOL> self . _contexts = self . _contexts_stm <EOL> for path in self . _pathlist [ <NUM_LIT:1> : ] : <EOL> self . _ProcessFile ( path ) <EOL> def _ProcessFile ( self , path ) : <EOL> """<STR_LIT>""" <EOL> assert path <EOL> ( frame , file ) = OpenFileForRead ( path , '<STR_LIT>' ) <EOL> if not file : <EOL> return <EOL> try : <EOL> self . _current = - <NUM_LIT:1> <EOL> xml . sax . parse ( file , self ) <EOL> except SchemaError : <EOL> output . Error ( '<STR_LIT>' <EOL> % path ) <EOL> except IOError : <EOL> output . Error ( '<STR_LIT>' % path ) <EOL> except xml . sax . _exceptions . SAXParseException , e : <EOL> output . Error ( '<STR_LIT>' % <EOL> ( path , e . _linenum , e . _colnum , e . getMessage ( ) ) ) <EOL> file . close ( ) <EOL> if frame : <EOL> frame . close ( ) <EOL> def _MungeLocationListIntoFiles ( self , urllist ) : <EOL> """<STR_LIT>""" <EOL> assert self . _pathlist <EOL> path = self . _pathlist [ <NUM_LIT:0> ] <EOL> path = os . path . normpath ( path ) <EOL> dir = os . path . dirname ( path ) <EOL> wide = False <EOL> if type ( path ) == types . UnicodeType : <EOL> wide = True <EOL> for url in urllist : <EOL> url = URL . Canonicalize ( url ) <EOL> output . Log ( '<STR_LIT>' % url , <NUM_LIT:2> ) <EOL> ( scheme , netloc , path , query , frag ) = urlparse . urlsplit ( url ) <EOL> file = os . path . basename ( path ) <EOL> file = urllib . unquote ( file ) <EOL> if wide : <EOL> file = encoder . WidenText ( file ) <EOL> if dir : <EOL> file = dir + os . sep + file <EOL> if file : <EOL> self . _pathlist . append ( file ) <EOL> output . Log ( '<STR_LIT>' % file , <NUM_LIT:1> ) <EOL> def startElement ( self , tag , attributes ) : <EOL> """<STR_LIT>""" <EOL> if ( self . _current < <NUM_LIT:0> ) and not self . _contexts : <EOL> assert self . _contexts_idx and self . _contexts_stm <EOL> if tag == '<STR_LIT>' : <EOL> self . _contexts = self . _contexts_stm <EOL> elif tag == '<STR_LIT>' : <EOL> self . _contexts = self . _contexts_idx <EOL> output . Log ( '<STR_LIT>' , <NUM_LIT:2> ) <EOL> else : <EOL> output . Error ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> raise SchemaError <EOL> if ( self . _current < <NUM_LIT:0> ) and ( self . _contexts == self . _contexts_stm ) and ( <EOL> tag == '<STR_LIT>' ) : <EOL> output . Error ( '<STR_LIT>' ) <EOL> raise SchemaError <EOL> if attributes : <EOL> text = '<STR_LIT>' <EOL> for attr in attributes . keys ( ) : <EOL> if self . _current < <NUM_LIT:0> : <EOL> if attr . find ( '<STR_LIT>' ) >= <NUM_LIT:0> : <EOL> continue <EOL> if attr . find ( '<STR_LIT>' ) >= <NUM_LIT:0> : <EOL> continue <EOL> if text : <EOL> text = text + '<STR_LIT:U+002CU+0020>' <EOL> text = text + attr <EOL> if text : <EOL> output . Warn ( '<STR_LIT>' <EOL> '<STR_LIT>' % ( tag , text ) ) <EOL> if ( self . _current < <NUM_LIT:0> ) or ( self . _contexts [ self . _current ] . AcceptTag ( tag ) ) : <EOL> self . _current = self . _current + <NUM_LIT:1> <EOL> assert self . _current < len ( self . _contexts ) <EOL> self . _contexts [ self . _current ] . Open ( ) <EOL> else : <EOL> output . Error ( '<STR_LIT>' % tag ) <EOL> raise SchemaError <EOL> def endElement ( self , tag ) : <EOL> """<STR_LIT>""" <EOL> tag = tag <EOL> assert self . _current >= <NUM_LIT:0> <EOL> retval = self . _contexts [ self . _current ] . Close ( ) <EOL> self . _current = self . _current - <NUM_LIT:1> <EOL> if self . _current >= <NUM_LIT:0> : <EOL> self . _contexts [ self . _current ] . Return ( retval ) <EOL> elif retval and ( self . _contexts == self . _contexts_idx ) : <EOL> self . _MungeLocationListIntoFiles ( retval ) <EOL> def characters ( self , text ) : <EOL> """<STR_LIT>""" <EOL> if ( self . _current < <NUM_LIT:0> ) or ( <EOL> not self . _contexts [ self . _current ] . AcceptText ( text ) ) : <EOL> if text . strip ( ) : <EOL> output . Error ( '<STR_LIT>' % text ) <EOL> raise SchemaError <EOL> class FilePathGenerator : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . is_gzip = False <EOL> self . _path = None <EOL> self . _prefix = None <EOL> self . _suffix = None <EOL> def Preload ( self , path ) : <EOL> """<STR_LIT>""" <EOL> path = encoder . MaybeNarrowPath ( path ) <EOL> path = os . path . normpath ( path ) <EOL> base = os . path . basename ( path ) . lower ( ) <EOL> if not base : <EOL> output . Error ( '<STR_LIT>' % path ) <EOL> return False <EOL> lenbase = len ( base ) <EOL> lensuffix = <NUM_LIT:0> <EOL> compare_suffix = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> for suffix in compare_suffix : <EOL> if base . endswith ( suffix ) : <EOL> lensuffix = len ( suffix ) <EOL> break <EOL> if not lensuffix : <EOL> output . Error ( '<STR_LIT>' <EOL> '<STR_LIT>' % path ) <EOL> return False <EOL> self . is_gzip = suffix . endswith ( '<STR_LIT>' ) <EOL> lenpath = len ( path ) <EOL> self . _path = path [ : lenpath - lenbase ] <EOL> self . _prefix = path [ lenpath - lenbase : lenpath - lensuffix ] <EOL> self . _suffix = path [ lenpath - lensuffix : ] <EOL> return True <EOL> def GeneratePath ( self , instance ) : <EOL> """<STR_LIT>""" <EOL> prefix = self . _path + self . _prefix <EOL> if type ( instance ) == types . IntType : <EOL> if instance : <EOL> return '<STR_LIT>' % ( prefix , instance , self . _suffix ) <EOL> return prefix + self . _suffix <EOL> return prefix + instance <EOL> def GenerateURL ( self , instance , root_url ) : <EOL> """<STR_LIT>""" <EOL> prefix = root_url + self . _prefix <EOL> retval = None <EOL> if type ( instance ) == types . IntType : <EOL> if instance : <EOL> retval = '<STR_LIT>' % ( prefix , instance , self . _suffix ) <EOL> else : <EOL> retval = prefix + self . _suffix <EOL> else : <EOL> retval = prefix + instance <EOL> return URL . Canonicalize ( retval ) <EOL> def GenerateWildURL ( self , root_url ) : <EOL> """<STR_LIT>""" <EOL> prefix = URL . Canonicalize ( root_url + self . _prefix ) <EOL> temp = URL . Canonicalize ( prefix + self . _suffix ) <EOL> suffix = temp [ len ( prefix ) : ] <EOL> return prefix + '<STR_LIT:*>' + suffix <EOL> class PerURLStatistics : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . _extensions = { } <EOL> def Consume ( self , url ) : <EOL> """<STR_LIT>""" <EOL> if url and url . loc : <EOL> ( scheme , netloc , path , query , frag ) = urlparse . urlsplit ( url . loc ) <EOL> if not path : <EOL> return <EOL> if path . endswith ( '<STR_LIT:/>' ) : <EOL> if self . _extensions . has_key ( '<STR_LIT:/>' ) : <EOL> self . _extensions [ '<STR_LIT:/>' ] = self . _extensions [ '<STR_LIT:/>' ] + <NUM_LIT:1> <EOL> else : <EOL> self . _extensions [ '<STR_LIT:/>' ] = <NUM_LIT:1> <EOL> return <EOL> i = path . rfind ( '<STR_LIT:/>' ) <EOL> if i >= <NUM_LIT:0> : <EOL> assert i < len ( path ) <EOL> path = path [ i : ] <EOL> i = path . rfind ( '<STR_LIT:.>' ) <EOL> if i > <NUM_LIT:0> : <EOL> assert i < len ( path ) <EOL> ext = path [ i : ] . lower ( ) <EOL> if self . _extensions . has_key ( ext ) : <EOL> self . _extensions [ ext ] = self . _extensions [ ext ] + <NUM_LIT:1> <EOL> else : <EOL> self . _extensions [ ext ] = <NUM_LIT:1> <EOL> else : <EOL> if self . _extensions . has_key ( '<STR_LIT>' ) : <EOL> self . _extensions [ '<STR_LIT>' ] = self . _extensions [ <EOL> '<STR_LIT>' ] + <NUM_LIT:1> <EOL> else : <EOL> self . _extensions [ '<STR_LIT>' ] = <NUM_LIT:1> <EOL> def Log ( self ) : <EOL> """<STR_LIT>""" <EOL> if len ( self . _extensions ) : <EOL> output . Log ( '<STR_LIT>' , <NUM_LIT:1> ) <EOL> set = self . _extensions . keys ( ) <EOL> set . sort ( ) <EOL> for ext in set : <EOL> output . Log ( '<STR_LIT>' % ( self . _extensions [ ext ] , ext ) , <NUM_LIT:1> ) <EOL> class Sitemap ( xml . sax . handler . ContentHandler ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , suppress_notify ) : <EOL> xml . sax . handler . ContentHandler . __init__ ( self ) <EOL> self . _filters = [ ] <EOL> self . _inputs = [ ] <EOL> self . _urls = { } <EOL> self . _set = [ ] <EOL> self . _filegen = None <EOL> self . _wildurl1 = None <EOL> self . _wildurl2 = None <EOL> self . _sitemaps = <NUM_LIT:0> <EOL> self . _dup_max = <NUM_LIT:2> <EOL> self . _stat = PerURLStatistics ( ) <EOL> self . _in_site = False <EOL> self . _in_Site_ever = False <EOL> self . _default_enc = None <EOL> self . _base_url = None <EOL> self . _store_into = None <EOL> self . _suppress = suppress_notify <EOL> def ValidateBasicConfig ( self ) : <EOL> """<STR_LIT>""" <EOL> all_good = True <EOL> if self . _default_enc : <EOL> encoder . SetUserEncoding ( self . _default_enc ) <EOL> if all_good and not self . _base_url : <EOL> output . Error ( '<STR_LIT>' ) <EOL> all_good = False <EOL> if all_good and not URL . IsAbsolute ( self . _base_url ) : <EOL> output . Error ( '<STR_LIT>' % <EOL> self . _base_url ) <EOL> all_good = False <EOL> if all_good : <EOL> self . _base_url = URL . Canonicalize ( self . _base_url ) <EOL> if not self . _base_url . endswith ( '<STR_LIT:/>' ) : <EOL> self . _base_url = self . _base_url + '<STR_LIT:/>' <EOL> output . Log ( '<STR_LIT>' % self . _base_url , <NUM_LIT:2> ) <EOL> if all_good : <EOL> if self . _store_into : <EOL> self . _filegen = FilePathGenerator ( ) <EOL> if not self . _filegen . Preload ( self . _store_into ) : <EOL> all_good = False <EOL> else : <EOL> output . Error ( '<STR_LIT>' ) <EOL> all_good = False <EOL> if all_good : <EOL> self . _wildurl1 = self . _filegen . GenerateWildURL ( self . _base_url ) <EOL> self . _wildurl2 = self . _filegen . GenerateURL ( SITEINDEX_SUFFIX , <EOL> self . _base_url ) <EOL> if all_good : <EOL> if self . _suppress : <EOL> if ( type ( self . _suppress ) == types . StringType ) or ( type ( self . _suppress ) <EOL> == types . UnicodeType ) : <EOL> if ( self . _suppress == '<STR_LIT:0>' ) or ( self . _suppress . lower ( ) == '<STR_LIT:false>' ) : <EOL> self . _suppress = False <EOL> if not all_good : <EOL> output . Log ( '<STR_LIT>' , <NUM_LIT:0> ) <EOL> return all_good <EOL> def Generate ( self ) : <EOL> """<STR_LIT>""" <EOL> for input in self . _inputs : <EOL> input . ProduceURLs ( self . ConsumeURL ) <EOL> if len ( self . _set ) : <EOL> self . FlushSet ( ) <EOL> if not self . _sitemaps : <EOL> output . Warn ( '<STR_LIT>' ) <EOL> self . FlushSet ( ) <EOL> if self . _sitemaps > <NUM_LIT:1> : <EOL> self . WriteIndex ( ) <EOL> self . NotifySearch ( ) <EOL> self . _stat . Log ( ) <EOL> def ConsumeURL ( self , url , allow_fragment ) : <EOL> """<STR_LIT>""" <EOL> if not url : <EOL> return <EOL> if not url . Validate ( self . _base_url , allow_fragment ) : <EOL> return <EOL> accept = None <EOL> for filter in self . _filters : <EOL> accept = filter . Apply ( url ) <EOL> if accept != None : <EOL> break <EOL> if not ( accept or ( accept == None ) ) : <EOL> url . Log ( prefix = '<STR_LIT>' , level = <NUM_LIT:2> ) <EOL> return <EOL> if fnmatch . fnmatchcase ( url . loc , self . _wildurl1 ) or fnmatch . fnmatchcase ( <EOL> url . loc , self . _wildurl2 ) : <EOL> url . Log ( prefix = '<STR_LIT>' , level = <NUM_LIT:2> ) <EOL> return <EOL> hash = url . MakeHash ( ) <EOL> if self . _urls . has_key ( hash ) : <EOL> dup = self . _urls [ hash ] <EOL> if dup > <NUM_LIT:0> : <EOL> dup = dup + <NUM_LIT:1> <EOL> self . _urls [ hash ] = dup <EOL> if self . _dup_max < dup : <EOL> self . _dup_max = dup <EOL> url . Log ( prefix = '<STR_LIT>' ) <EOL> return <EOL> self . _urls [ hash ] = <NUM_LIT:1> <EOL> self . _set . append ( url ) <EOL> self . _stat . Consume ( url ) <EOL> url . Log ( ) <EOL> if len ( self . _set ) >= MAXURLS_PER_SITEMAP : <EOL> self . FlushSet ( ) <EOL> def FlushSet ( self ) : <EOL> """<STR_LIT>""" <EOL> output . Log ( '<STR_LIT>' , <NUM_LIT:1> ) <EOL> self . _set . sort ( ) <EOL> for url in self . _set : <EOL> hash = url . MakeHash ( ) <EOL> dup = self . _urls [ hash ] <EOL> if dup > <NUM_LIT:0> : <EOL> self . _urls [ hash ] = - <NUM_LIT:1> <EOL> if not url . priority : <EOL> url . priority = '<STR_LIT>' % ( float ( dup ) / float ( self . _dup_max ) ) <EOL> filename = self . _filegen . GeneratePath ( self . _sitemaps ) <EOL> if not filename : <EOL> output . Fatal ( '<STR_LIT>' ) <EOL> self . _sitemaps = self . _sitemaps + <NUM_LIT:1> <EOL> output . Log ( '<STR_LIT>' % <EOL> ( filename , len ( self . _set ) ) , <NUM_LIT:1> ) <EOL> frame = None <EOL> file = None <EOL> try : <EOL> if self . _filegen . is_gzip : <EOL> basename = os . path . basename ( filename ) ; <EOL> frame = open ( filename , '<STR_LIT:wb>' ) <EOL> file = gzip . GzipFile ( fileobj = frame , filename = basename , mode = '<STR_LIT>' ) <EOL> else : <EOL> file = open ( filename , '<STR_LIT>' ) <EOL> file . write ( SITEMAP_HEADER ) <EOL> for url in self . _set : <EOL> url . WriteXML ( file ) <EOL> file . write ( SITEMAP_FOOTER ) <EOL> file . close ( ) <EOL> if frame : <EOL> frame . close ( ) <EOL> frame = None <EOL> file = None <EOL> except IOError : <EOL> output . Fatal ( '<STR_LIT>' % filename ) <EOL> os . chmod ( filename , <NUM_LIT:0> <NUM_LIT> ) <EOL> self . _set = [ ] <EOL> def WriteIndex ( self ) : <EOL> """<STR_LIT>""" <EOL> filename = self . _filegen . GeneratePath ( SITEINDEX_SUFFIX ) <EOL> if not filename : <EOL> output . Fatal ( '<STR_LIT>' ) <EOL> output . Log ( '<STR_LIT>' % <EOL> ( filename , self . _sitemaps ) , <NUM_LIT:1> ) <EOL> lastmod = TimestampISO8601 ( time . time ( ) ) <EOL> try : <EOL> fd = open ( filename , '<STR_LIT>' ) <EOL> fd . write ( SITEINDEX_HEADER ) <EOL> for mapnumber in range ( <NUM_LIT:0> , self . _sitemaps ) : <EOL> mapurl = self . _filegen . GenerateURL ( mapnumber , self . _base_url ) <EOL> mapattributes = { '<STR_LIT>' : mapurl , '<STR_LIT>' : lastmod } <EOL> fd . write ( SITEINDEX_ENTRY % mapattributes ) <EOL> fd . write ( SITEINDEX_FOOTER ) <EOL> fd . close ( ) <EOL> fd = None <EOL> except IOError : <EOL> output . Fatal ( '<STR_LIT>' % filename ) <EOL> os . chmod ( filename , <NUM_LIT:0> <NUM_LIT> ) <EOL> def NotifySearch ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . _suppress : <EOL> output . Log ( '<STR_LIT>' , <NUM_LIT:1> ) <EOL> return <EOL> output . Log ( '<STR_LIT>' , <NUM_LIT:1> ) <EOL> class ExceptionURLopener ( urllib . FancyURLopener ) : <EOL> def http_error_default ( self , url , fp , errcode , errmsg , headers ) : <EOL> output . Log ( '<STR_LIT>' % ( errcode , errmsg ) , <NUM_LIT:2> ) <EOL> raise IOError <EOL> old_opener = urllib . _urlopener <EOL> urllib . _urlopener = ExceptionURLopener ( ) <EOL> if self . _sitemaps > <NUM_LIT:1> : <EOL> url = self . _filegen . GenerateURL ( SITEINDEX_SUFFIX , self . _base_url ) <EOL> else : <EOL> url = self . _filegen . GenerateURL ( <NUM_LIT:0> , self . _base_url ) <EOL> try : <EOL> u = urllib . urlopen ( url ) <EOL> u . close ( ) <EOL> except IOError : <EOL> output . Error ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % url ) <EOL> output . Warn ( '<STR_LIT>' ) <EOL> for ping in NOTIFICATION_SITES : <EOL> query_map = ping [ <NUM_LIT:3> ] <EOL> query_attr = ping [ <NUM_LIT:5> ] <EOL> query_map [ query_attr ] = url <EOL> query = urllib . urlencode ( query_map ) <EOL> notify = urlparse . urlunsplit ( ( ping [ <NUM_LIT:0> ] , ping [ <NUM_LIT:1> ] , ping [ <NUM_LIT:2> ] , query , ping [ <NUM_LIT:4> ] ) ) <EOL> output . Log ( '<STR_LIT>' % ping [ <NUM_LIT:1> ] , <NUM_LIT:1> ) <EOL> output . Log ( '<STR_LIT>' % notify , <NUM_LIT:2> ) <EOL> try : <EOL> u = urllib . urlopen ( notify ) <EOL> u . read ( ) <EOL> u . close ( ) <EOL> except IOError : <EOL> output . Warn ( '<STR_LIT>' % ping [ <NUM_LIT:1> ] ) <EOL> if old_opener : <EOL> urllib . _urlopener = old_opener <EOL> def startElement ( self , tag , attributes ) : <EOL> """<STR_LIT>""" <EOL> if tag == '<STR_LIT>' : <EOL> if self . _in_site : <EOL> output . Error ( '<STR_LIT>' ) <EOL> else : <EOL> self . _in_site = True <EOL> if not ValidateAttributes ( '<STR_LIT>' , attributes , <EOL> ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ) ) : <EOL> return <EOL> verbose = attributes . get ( '<STR_LIT>' , <NUM_LIT:0> ) <EOL> if verbose : <EOL> output . SetVerbose ( verbose ) <EOL> self . _default_enc = attributes . get ( '<STR_LIT>' ) <EOL> self . _base_url = attributes . get ( '<STR_LIT>' ) <EOL> self . _store_into = attributes . get ( '<STR_LIT>' ) <EOL> if not self . _suppress : <EOL> self . _suppress = attributes . get ( '<STR_LIT>' , <EOL> False ) <EOL> self . ValidateBasicConfig ( ) <EOL> elif tag == '<STR_LIT>' : <EOL> self . _filters . append ( Filter ( attributes ) ) <EOL> elif tag == '<STR_LIT:url>' : <EOL> self . _inputs . append ( InputURL ( attributes ) ) <EOL> elif tag == '<STR_LIT>' : <EOL> for attributeset in ExpandPathAttribute ( attributes , '<STR_LIT:path>' ) : <EOL> self . _inputs . append ( InputURLList ( attributeset ) ) <EOL> elif tag == '<STR_LIT>' : <EOL> self . _inputs . append ( InputDirectory ( attributes , self . _base_url ) ) <EOL> elif tag == '<STR_LIT>' : <EOL> for attributeset in ExpandPathAttribute ( attributes , '<STR_LIT:path>' ) : <EOL> self . _inputs . append ( InputAccessLog ( attributeset ) ) <EOL> elif tag == '<STR_LIT>' : <EOL> for attributeset in ExpandPathAttribute ( attributes , '<STR_LIT:path>' ) : <EOL> self . _inputs . append ( InputSitemap ( attributeset ) ) <EOL> else : <EOL> output . Error ( '<STR_LIT>' % tag ) <EOL> def endElement ( self , tag ) : <EOL> """<STR_LIT>""" <EOL> if tag == '<STR_LIT>' : <EOL> assert self . _in_site <EOL> self . _in_site = False <EOL> self . _in_site_ever = True <EOL> def endDocument ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . _in_site_ever : <EOL> output . Error ( '<STR_LIT>' ) <EOL> else : <EOL> if not self . _inputs : <EOL> output . Warn ( '<STR_LIT>' ) <EOL> def ValidateAttributes ( tag , attributes , goodattributes ) : <EOL> """<STR_LIT>""" <EOL> all_good = True <EOL> for attr in attributes . keys ( ) : <EOL> if not attr in goodattributes : <EOL> output . Error ( '<STR_LIT>' % ( tag , attr ) ) <EOL> all_good = False <EOL> return all_good <EOL> def ExpandPathAttribute ( src , attrib ) : <EOL> """<STR_LIT>""" <EOL> path = src . get ( attrib ) <EOL> if not path : <EOL> return [ src ] <EOL> path = encoder . MaybeNarrowPath ( path ) ; <EOL> pathlist = glob . glob ( path ) <EOL> if not pathlist : <EOL> return [ src ] <EOL> if type ( src ) != types . DictionaryType : <EOL> tmp = { } <EOL> for key in src . keys ( ) : <EOL> tmp [ key ] = src [ key ] <EOL> src = tmp <EOL> retval = [ ] <EOL> for path in pathlist : <EOL> dst = src . copy ( ) <EOL> dst [ attrib ] = path <EOL> retval . append ( dst ) <EOL> return retval <EOL> def OpenFileForRead ( path , logtext ) : <EOL> """<STR_LIT>""" <EOL> frame = None <EOL> file = None <EOL> if not path : <EOL> return ( frame , file ) <EOL> try : <EOL> if path . endswith ( '<STR_LIT>' ) : <EOL> frame = open ( path , '<STR_LIT:rb>' ) <EOL> file = gzip . GzipFile ( fileobj = frame , mode = '<STR_LIT>' ) <EOL> else : <EOL> file = open ( path , '<STR_LIT>' ) <EOL> if logtext : <EOL> output . Log ( '<STR_LIT>' % ( logtext , path ) , <NUM_LIT:1> ) <EOL> else : <EOL> output . Log ( '<STR_LIT>' % path , <NUM_LIT:1> ) <EOL> except IOError : <EOL> output . Error ( '<STR_LIT>' % path ) <EOL> return ( frame , file ) <EOL> def TimestampISO8601 ( t ) : <EOL> """<STR_LIT>""" <EOL> return time . strftime ( '<STR_LIT>' , time . gmtime ( t ) ) <EOL> def CreateSitemapFromFile ( configpath , suppress_notify ) : <EOL> """<STR_LIT>""" <EOL> num_errors = output . num_errors <EOL> sitemap = Sitemap ( suppress_notify ) <EOL> try : <EOL> output . Log ( '<STR_LIT>' % configpath , <NUM_LIT:0> ) <EOL> xml . sax . parse ( configpath , sitemap ) <EOL> except IOError : <EOL> output . Error ( '<STR_LIT>' % configpath ) <EOL> except xml . sax . _exceptions . SAXParseException , e : <EOL> output . Error ( '<STR_LIT>' % <EOL> ( e . _linenum , e . _colnum , e . getMessage ( ) ) ) <EOL> except xml . sax . _exceptions . SAXReaderNotAvailable : <EOL> output . Error ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if num_errors == output . num_errors : <EOL> return sitemap <EOL> return None <EOL> def ProcessCommandFlags ( args ) : <EOL> """<STR_LIT>""" <EOL> flags = { } <EOL> rkeyval = '<STR_LIT>' <EOL> roption = '<STR_LIT>' <EOL> r = '<STR_LIT:(>' + rkeyval + '<STR_LIT>' + roption + '<STR_LIT:)>' <EOL> rc = re . compile ( r ) <EOL> for a in args : <EOL> try : <EOL> rcg = rc . search ( a ) . groupdict ( ) <EOL> if rcg . has_key ( '<STR_LIT:key>' ) : <EOL> flags [ rcg [ '<STR_LIT:key>' ] ] = rcg [ '<STR_LIT:value>' ] <EOL> if rcg . has_key ( '<STR_LIT>' ) : <EOL> flags [ rcg [ '<STR_LIT>' ] ] = rcg [ '<STR_LIT>' ] <EOL> except AttributeError : <EOL> return None <EOL> return flags <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> flags = ProcessCommandFlags ( sys . argv [ <NUM_LIT:1> : ] ) <EOL> if not flags or not flags . has_key ( '<STR_LIT>' ) or flags . has_key ( '<STR_LIT>' ) : <EOL> output . Log ( __usage__ , <NUM_LIT:0> ) <EOL> else : <EOL> suppress_notify = flags . has_key ( '<STR_LIT>' ) <EOL> sitemap = CreateSitemapFromFile ( flags [ '<STR_LIT>' ] , suppress_notify ) <EOL> if not sitemap : <EOL> output . Log ( '<STR_LIT>' , <NUM_LIT:0> ) <EOL> else : <EOL> sitemap . Generate ( ) <EOL> output . Log ( '<STR_LIT>' % output . num_errors , <NUM_LIT:1> ) <EOL> output . Log ( '<STR_LIT>' % output . num_warns , <NUM_LIT:1> ) </s>
