<s> """<STR_LIT>""" <EOL> __version__ = "<STR_LIT>" <EOL> import sys <EOL> import os <EOL> from types import * <EOL> import StringIO <EOL> import getopt <EOL> import pickle <EOL> import urllib <EOL> import urlparse <EOL> import sgmllib <EOL> import cgi <EOL> import mimetypes <EOL> import robotparser <EOL> if __version__ [ <NUM_LIT:0> ] == '<STR_LIT:$>' : <EOL> _v = __version__ . split ( ) <EOL> if len ( _v ) == <NUM_LIT:3> : <EOL> __version__ = _v [ <NUM_LIT:1> ] <EOL> DEFROOT = "<STR_LIT>" <EOL> CHECKEXT = <NUM_LIT:1> <EOL> VERBOSE = <NUM_LIT:1> <EOL> MAXPAGE = <NUM_LIT> <EOL> ROUNDSIZE = <NUM_LIT:50> <EOL> DUMPFILE = "<STR_LIT>" <EOL> AGENTNAME = "<STR_LIT>" <EOL> NONAMES = <NUM_LIT:0> <EOL> def main ( ) : <EOL> checkext = CHECKEXT <EOL> verbose = VERBOSE <EOL> maxpage = MAXPAGE <EOL> roundsize = ROUNDSIZE <EOL> dumpfile = DUMPFILE <EOL> restart = <NUM_LIT:0> <EOL> norun = <NUM_LIT:0> <EOL> try : <EOL> opts , args = getopt . getopt ( sys . argv [ <NUM_LIT:1> : ] , '<STR_LIT>' ) <EOL> except getopt . error , msg : <EOL> sys . stdout = sys . stderr <EOL> print msg <EOL> print __doc__ % globals ( ) <EOL> sys . exit ( <NUM_LIT:2> ) <EOL> extra_roots = [ ] <EOL> nonames = NONAMES <EOL> for o , a in opts : <EOL> if o == '<STR_LIT>' : <EOL> restart = <NUM_LIT:1> <EOL> if o == '<STR_LIT>' : <EOL> dumpfile = a <EOL> if o == '<STR_LIT>' : <EOL> maxpage = int ( a ) <EOL> if o == '<STR_LIT>' : <EOL> norun = <NUM_LIT:1> <EOL> if o == '<STR_LIT>' : <EOL> verbose = <NUM_LIT:0> <EOL> if o == '<STR_LIT>' : <EOL> roundsize = int ( a ) <EOL> if o == '<STR_LIT>' : <EOL> extra_roots . append ( a ) <EOL> if o == '<STR_LIT>' : <EOL> nonames = not nonames <EOL> if o == '<STR_LIT>' : <EOL> verbose = verbose + <NUM_LIT:1> <EOL> if o == '<STR_LIT>' : <EOL> checkext = not checkext <EOL> if verbose > <NUM_LIT:0> : <EOL> print AGENTNAME , "<STR_LIT:version>" , __version__ <EOL> if restart : <EOL> c = load_pickle ( dumpfile = dumpfile , verbose = verbose ) <EOL> else : <EOL> c = Checker ( ) <EOL> c . setflags ( checkext = checkext , verbose = verbose , <EOL> maxpage = maxpage , roundsize = roundsize , <EOL> nonames = nonames <EOL> ) <EOL> if not restart and not args : <EOL> args . append ( DEFROOT ) <EOL> for arg in args : <EOL> c . addroot ( arg ) <EOL> if not checkext : <EOL> for root in extra_roots : <EOL> if root [ - <NUM_LIT:1> ] != "<STR_LIT:/>" : <EOL> root = root + "<STR_LIT:/>" <EOL> c . addroot ( root , add_to_do = <NUM_LIT:0> ) <EOL> try : <EOL> if not norun : <EOL> try : <EOL> c . run ( ) <EOL> except KeyboardInterrupt : <EOL> if verbose > <NUM_LIT:0> : <EOL> print "<STR_LIT>" <EOL> try : <EOL> c . report ( ) <EOL> except KeyboardInterrupt : <EOL> if verbose > <NUM_LIT:0> : <EOL> print "<STR_LIT>" <EOL> finally : <EOL> if c . save_pickle ( dumpfile ) : <EOL> if dumpfile == DUMPFILE : <EOL> print "<STR_LIT>" % sys . argv [ <NUM_LIT:0> ] <EOL> else : <EOL> print "<STR_LIT>" % ( sys . argv [ <NUM_LIT:0> ] , <EOL> dumpfile ) <EOL> def load_pickle ( dumpfile = DUMPFILE , verbose = VERBOSE ) : <EOL> if verbose > <NUM_LIT:0> : <EOL> print "<STR_LIT>" % dumpfile <EOL> f = open ( dumpfile , "<STR_LIT:rb>" ) <EOL> c = pickle . load ( f ) <EOL> f . close ( ) <EOL> if verbose > <NUM_LIT:0> : <EOL> print "<STR_LIT>" <EOL> print "<STR_LIT>" , "<STR_LIT>" . join ( c . roots ) <EOL> return c <EOL> class Checker : <EOL> checkext = CHECKEXT <EOL> verbose = VERBOSE <EOL> maxpage = MAXPAGE <EOL> roundsize = ROUNDSIZE <EOL> nonames = NONAMES <EOL> validflags = tuple ( dir ( ) ) <EOL> def __init__ ( self ) : <EOL> self . reset ( ) <EOL> def setflags ( self , ** kw ) : <EOL> for key in kw . keys ( ) : <EOL> if key not in self . validflags : <EOL> raise NameError , "<STR_LIT>" % str ( key ) <EOL> for key , value in kw . items ( ) : <EOL> setattr ( self , key , value ) <EOL> def reset ( self ) : <EOL> self . roots = [ ] <EOL> self . todo = { } <EOL> self . done = { } <EOL> self . bad = { } <EOL> self . name_table = { } <EOL> self . round = <NUM_LIT:0> <EOL> self . robots = { } <EOL> self . errors = { } <EOL> self . urlopener = MyURLopener ( ) <EOL> self . changed = <NUM_LIT:0> <EOL> def note ( self , level , format , * args ) : <EOL> if self . verbose > level : <EOL> if args : <EOL> format = format % args <EOL> self . message ( format ) <EOL> def message ( self , format , * args ) : <EOL> if args : <EOL> format = format % args <EOL> print format <EOL> def __getstate__ ( self ) : <EOL> return ( self . roots , self . todo , self . done , self . bad , self . round ) <EOL> def __setstate__ ( self , state ) : <EOL> self . reset ( ) <EOL> ( self . roots , self . todo , self . done , self . bad , self . round ) = state <EOL> for root in self . roots : <EOL> self . addrobot ( root ) <EOL> for url in self . bad . keys ( ) : <EOL> self . markerror ( url ) <EOL> def addroot ( self , root , add_to_do = <NUM_LIT:1> ) : <EOL> if root not in self . roots : <EOL> troot = root <EOL> scheme , netloc , path , params , query , fragment = urlparse . urlparse ( root ) <EOL> i = path . rfind ( "<STR_LIT:/>" ) + <NUM_LIT:1> <EOL> if <NUM_LIT:0> < i < len ( path ) : <EOL> path = path [ : i ] <EOL> troot = urlparse . urlunparse ( ( scheme , netloc , path , <EOL> params , query , fragment ) ) <EOL> self . roots . append ( troot ) <EOL> self . addrobot ( root ) <EOL> if add_to_do : <EOL> self . newlink ( ( root , "<STR_LIT>" ) , ( "<STR_LIT>" , root ) ) <EOL> def addrobot ( self , root ) : <EOL> root = urlparse . urljoin ( root , "<STR_LIT:/>" ) <EOL> if self . robots . has_key ( root ) : return <EOL> url = urlparse . urljoin ( root , "<STR_LIT>" ) <EOL> self . robots [ root ] = rp = robotparser . RobotFileParser ( ) <EOL> self . note ( <NUM_LIT:2> , "<STR_LIT>" , url ) <EOL> rp . debug = self . verbose > <NUM_LIT:3> <EOL> rp . set_url ( url ) <EOL> try : <EOL> rp . read ( ) <EOL> except ( OSError , IOError ) , msg : <EOL> self . note ( <NUM_LIT:1> , "<STR_LIT>" , url , msg ) <EOL> def run ( self ) : <EOL> while self . todo : <EOL> self . round = self . round + <NUM_LIT:1> <EOL> self . note ( <NUM_LIT:0> , "<STR_LIT>" , self . round , self . status ( ) ) <EOL> urls = self . todo . keys ( ) <EOL> urls . sort ( ) <EOL> del urls [ self . roundsize : ] <EOL> for url in urls : <EOL> self . dopage ( url ) <EOL> def status ( self ) : <EOL> return "<STR_LIT>" % ( <EOL> len ( self . todo ) + len ( self . done ) , <EOL> len ( self . todo ) , len ( self . done ) , <EOL> len ( self . bad ) ) <EOL> def report ( self ) : <EOL> self . message ( "<STR_LIT>" ) <EOL> if not self . todo : s = "<STR_LIT>" <EOL> else : s = "<STR_LIT>" <EOL> self . message ( "<STR_LIT>" , s , self . status ( ) ) <EOL> self . report_errors ( ) <EOL> def report_errors ( self ) : <EOL> if not self . bad : <EOL> self . message ( "<STR_LIT>" ) <EOL> return <EOL> self . message ( "<STR_LIT>" ) <EOL> sources = self . errors . keys ( ) <EOL> sources . sort ( ) <EOL> for source in sources : <EOL> triples = self . errors [ source ] <EOL> self . message ( "<STR_LIT>" ) <EOL> if len ( triples ) > <NUM_LIT:1> : <EOL> self . message ( "<STR_LIT>" , len ( triples ) , source ) <EOL> else : <EOL> self . message ( "<STR_LIT>" , source ) <EOL> for url , rawlink , msg in triples : <EOL> if rawlink != self . format_url ( url ) : s = "<STR_LIT>" % rawlink <EOL> else : s = "<STR_LIT>" <EOL> self . message ( "<STR_LIT>" , <EOL> self . format_url ( url ) , s , msg ) <EOL> def dopage ( self , url_pair ) : <EOL> if self . verbose > <NUM_LIT:1> : <EOL> if self . verbose > <NUM_LIT:2> : <EOL> self . show ( "<STR_LIT>" , self . format_url ( url_pair ) , <EOL> "<STR_LIT>" , self . todo [ url_pair ] ) <EOL> else : <EOL> self . message ( "<STR_LIT>" , self . format_url ( url_pair ) ) <EOL> url , local_fragment = url_pair <EOL> if local_fragment and self . nonames : <EOL> self . markdone ( url_pair ) <EOL> return <EOL> try : <EOL> page = self . getpage ( url_pair ) <EOL> except sgmllib . SGMLParseError , msg : <EOL> msg = self . sanitize ( msg ) <EOL> self . note ( <NUM_LIT:0> , "<STR_LIT>" , <EOL> self . format_url ( url_pair ) , msg ) <EOL> page = None <EOL> if page : <EOL> self . name_table [ url ] = page <EOL> if local_fragment and local_fragment not in page . getnames ( ) : <EOL> self . setbad ( url_pair , ( "<STR_LIT>" % local_fragment ) ) <EOL> for info in page . getlinkinfos ( ) : <EOL> link , rawlink , fragment = info <EOL> origin = url , rawlink <EOL> self . newlink ( ( link , fragment ) , origin ) <EOL> else : <EOL> self . name_table [ url_pair [ <NUM_LIT:0> ] ] = None <EOL> self . markdone ( url_pair ) <EOL> def newlink ( self , url , origin ) : <EOL> if self . done . has_key ( url ) : <EOL> self . newdonelink ( url , origin ) <EOL> else : <EOL> self . newtodolink ( url , origin ) <EOL> def newdonelink ( self , url , origin ) : <EOL> if origin not in self . done [ url ] : <EOL> self . done [ url ] . append ( origin ) <EOL> self . note ( <NUM_LIT:3> , "<STR_LIT>" , self . format_url ( url ) ) <EOL> if self . bad . has_key ( url ) : <EOL> source , rawlink = origin <EOL> triple = url , rawlink , self . bad [ url ] <EOL> self . seterror ( source , triple ) <EOL> def newtodolink ( self , url , origin ) : <EOL> if self . todo . has_key ( url ) : <EOL> if origin not in self . todo [ url ] : <EOL> self . todo [ url ] . append ( origin ) <EOL> self . note ( <NUM_LIT:3> , "<STR_LIT>" , self . format_url ( url ) ) <EOL> else : <EOL> self . todo [ url ] = [ origin ] <EOL> self . note ( <NUM_LIT:3> , "<STR_LIT>" , self . format_url ( url ) ) <EOL> def format_url ( self , url ) : <EOL> link , fragment = url <EOL> if fragment : return link + "<STR_LIT:#>" + fragment <EOL> else : return link <EOL> def markdone ( self , url ) : <EOL> self . done [ url ] = self . todo [ url ] <EOL> del self . todo [ url ] <EOL> self . changed = <NUM_LIT:1> <EOL> def inroots ( self , url ) : <EOL> for root in self . roots : <EOL> if url [ : len ( root ) ] == root : <EOL> return self . isallowed ( root , url ) <EOL> return <NUM_LIT:0> <EOL> def isallowed ( self , root , url ) : <EOL> root = urlparse . urljoin ( root , "<STR_LIT:/>" ) <EOL> return self . robots [ root ] . can_fetch ( AGENTNAME , url ) <EOL> def getpage ( self , url_pair ) : <EOL> url , fragment = url_pair <EOL> if self . name_table . has_key ( url ) : <EOL> return self . name_table [ url ] <EOL> scheme , path = urllib . splittype ( url ) <EOL> if scheme in ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) : <EOL> self . note ( <NUM_LIT:1> , "<STR_LIT>" % scheme ) <EOL> return None <EOL> isint = self . inroots ( url ) <EOL> if not isint : <EOL> if not self . checkext : <EOL> self . note ( <NUM_LIT:1> , "<STR_LIT>" ) <EOL> return None <EOL> f = self . openpage ( url_pair ) <EOL> if f : <EOL> self . safeclose ( f ) <EOL> return None <EOL> text , nurl = self . readhtml ( url_pair ) <EOL> if nurl != url : <EOL> self . note ( <NUM_LIT:1> , "<STR_LIT>" , nurl ) <EOL> url = nurl <EOL> if text : <EOL> return Page ( text , url , maxpage = self . maxpage , checker = self ) <EOL> def readhtml ( self , url_pair ) : <EOL> url , fragment = url_pair <EOL> text = None <EOL> f , url = self . openhtml ( url_pair ) <EOL> if f : <EOL> text = f . read ( ) <EOL> f . close ( ) <EOL> return text , url <EOL> def openhtml ( self , url_pair ) : <EOL> url , fragment = url_pair <EOL> f = self . openpage ( url_pair ) <EOL> if f : <EOL> url = f . geturl ( ) <EOL> info = f . info ( ) <EOL> if not self . checkforhtml ( info , url ) : <EOL> self . safeclose ( f ) <EOL> f = None <EOL> return f , url <EOL> def openpage ( self , url_pair ) : <EOL> url , fragment = url_pair <EOL> try : <EOL> return self . urlopener . open ( url ) <EOL> except ( OSError , IOError ) , msg : <EOL> msg = self . sanitize ( msg ) <EOL> self . note ( <NUM_LIT:0> , "<STR_LIT>" , msg ) <EOL> if self . verbose > <NUM_LIT:0> : <EOL> self . show ( "<STR_LIT>" , url , "<STR_LIT>" , self . todo [ url_pair ] ) <EOL> self . setbad ( url_pair , msg ) <EOL> return None <EOL> def checkforhtml ( self , info , url ) : <EOL> if info . has_key ( '<STR_LIT>' ) : <EOL> ctype = cgi . parse_header ( info [ '<STR_LIT>' ] ) [ <NUM_LIT:0> ] . lower ( ) <EOL> if '<STR_LIT:;>' in ctype : <EOL> ctype = ctype . split ( '<STR_LIT:;>' , <NUM_LIT:1> ) [ <NUM_LIT:0> ] . strip ( ) <EOL> else : <EOL> if url [ - <NUM_LIT:1> : ] == "<STR_LIT:/>" : <EOL> return <NUM_LIT:1> <EOL> ctype , encoding = mimetypes . guess_type ( url ) <EOL> if ctype == '<STR_LIT>' : <EOL> return <NUM_LIT:1> <EOL> else : <EOL> self . note ( <NUM_LIT:1> , "<STR_LIT>" , ctype ) <EOL> return <NUM_LIT:0> <EOL> def setgood ( self , url ) : <EOL> if self . bad . has_key ( url ) : <EOL> del self . bad [ url ] <EOL> self . changed = <NUM_LIT:1> <EOL> self . note ( <NUM_LIT:0> , "<STR_LIT>" ) <EOL> def setbad ( self , url , msg ) : <EOL> if self . bad . has_key ( url ) and self . bad [ url ] == msg : <EOL> self . note ( <NUM_LIT:0> , "<STR_LIT>" ) <EOL> return <EOL> self . bad [ url ] = msg <EOL> self . changed = <NUM_LIT:1> <EOL> self . markerror ( url ) <EOL> def markerror ( self , url ) : <EOL> try : <EOL> origins = self . todo [ url ] <EOL> except KeyError : <EOL> origins = self . done [ url ] <EOL> for source , rawlink in origins : <EOL> triple = url , rawlink , self . bad [ url ] <EOL> self . seterror ( source , triple ) <EOL> def seterror ( self , url , triple ) : <EOL> try : <EOL> if triple not in self . errors [ url ] : <EOL> self . errors [ url ] . append ( triple ) <EOL> except KeyError : <EOL> self . errors [ url ] = [ triple ] <EOL> def show ( self , p1 , link , p2 , origins ) : <EOL> self . message ( "<STR_LIT>" , p1 , link ) <EOL> i = <NUM_LIT:0> <EOL> for source , rawlink in origins : <EOL> i = i + <NUM_LIT:1> <EOL> if i == <NUM_LIT:2> : <EOL> p2 = '<STR_LIT:U+0020>' * len ( p2 ) <EOL> if rawlink != link : s = "<STR_LIT>" % rawlink <EOL> else : s = "<STR_LIT>" <EOL> self . message ( "<STR_LIT>" , p2 , source , s ) <EOL> def sanitize ( self , msg ) : <EOL> if isinstance ( IOError , ClassType ) and isinstance ( msg , IOError ) : <EOL> msg . args = self . sanitize ( msg . args ) <EOL> elif isinstance ( msg , TupleType ) : <EOL> if len ( msg ) >= <NUM_LIT:4> and msg [ <NUM_LIT:0> ] == '<STR_LIT>' and isinstance ( msg [ <NUM_LIT:3> ] , InstanceType ) : <EOL> msg = msg [ : <NUM_LIT:3> ] + msg [ <NUM_LIT:4> : ] <EOL> return msg <EOL> def safeclose ( self , f ) : <EOL> try : <EOL> url = f . geturl ( ) <EOL> except AttributeError : <EOL> pass <EOL> else : <EOL> if url [ : <NUM_LIT:4> ] == '<STR_LIT>' or url [ : <NUM_LIT:7> ] == '<STR_LIT>' : <EOL> text = f . read ( ) <EOL> f . close ( ) <EOL> def save_pickle ( self , dumpfile = DUMPFILE ) : <EOL> if not self . changed : <EOL> self . note ( <NUM_LIT:0> , "<STR_LIT>" ) <EOL> elif not dumpfile : <EOL> self . note ( <NUM_LIT:0> , "<STR_LIT>" ) <EOL> else : <EOL> self . note ( <NUM_LIT:0> , "<STR_LIT>" , dumpfile ) <EOL> newfile = dumpfile + "<STR_LIT>" <EOL> f = open ( newfile , "<STR_LIT:wb>" ) <EOL> pickle . dump ( self , f ) <EOL> f . close ( ) <EOL> try : <EOL> os . unlink ( dumpfile ) <EOL> except os . error : <EOL> pass <EOL> os . rename ( newfile , dumpfile ) <EOL> self . note ( <NUM_LIT:0> , "<STR_LIT>" ) <EOL> return <NUM_LIT:1> <EOL> class Page : <EOL> def __init__ ( self , text , url , verbose = VERBOSE , maxpage = MAXPAGE , checker = None ) : <EOL> self . text = text <EOL> self . url = url <EOL> self . verbose = verbose <EOL> self . maxpage = maxpage <EOL> self . checker = checker <EOL> size = len ( self . text ) <EOL> if size > self . maxpage : <EOL> self . note ( <NUM_LIT:0> , "<STR_LIT>" , self . url , ( size * <NUM_LIT> ) ) <EOL> self . parser = None <EOL> return <EOL> self . checker . note ( <NUM_LIT:2> , "<STR_LIT>" , self . url , size ) <EOL> self . parser = MyHTMLParser ( url , verbose = self . verbose , <EOL> checker = self . checker ) <EOL> self . parser . feed ( self . text ) <EOL> self . parser . close ( ) <EOL> def note ( self , level , msg , * args ) : <EOL> if self . checker : <EOL> apply ( self . checker . note , ( level , msg ) + args ) <EOL> else : <EOL> if self . verbose >= level : <EOL> if args : <EOL> msg = msg % args <EOL> print msg <EOL> def getnames ( self ) : <EOL> if self . parser : <EOL> return self . parser . names <EOL> else : <EOL> return [ ] <EOL> def getlinkinfos ( self ) : <EOL> if not self . parser : return [ ] <EOL> rawlinks = self . parser . getlinks ( ) <EOL> base = urlparse . urljoin ( self . url , self . parser . getbase ( ) or "<STR_LIT>" ) <EOL> infos = [ ] <EOL> for rawlink in rawlinks : <EOL> t = urlparse . urlparse ( rawlink ) <EOL> fragment = t [ - <NUM_LIT:1> ] <EOL> t = t [ : - <NUM_LIT:1> ] + ( '<STR_LIT>' , ) <EOL> rawlink = urlparse . urlunparse ( t ) <EOL> link = urlparse . urljoin ( base , rawlink ) <EOL> infos . append ( ( link , rawlink , fragment ) ) <EOL> return infos <EOL> class MyStringIO ( StringIO . StringIO ) : <EOL> def __init__ ( self , url , info ) : <EOL> self . __url = url <EOL> self . __info = info <EOL> StringIO . StringIO . __init__ ( self ) <EOL> def info ( self ) : <EOL> return self . __info <EOL> def geturl ( self ) : <EOL> return self . __url <EOL> class MyURLopener ( urllib . FancyURLopener ) : <EOL> http_error_default = urllib . URLopener . http_error_default <EOL> def __init__ ( * args ) : <EOL> self = args [ <NUM_LIT:0> ] <EOL> apply ( urllib . FancyURLopener . __init__ , args ) <EOL> self . addheaders = [ <EOL> ( '<STR_LIT>' , '<STR_LIT>' % __version__ ) , <EOL> ] <EOL> def http_error_401 ( self , url , fp , errcode , errmsg , headers ) : <EOL> return None <EOL> def open_file ( self , url ) : <EOL> path = urllib . url2pathname ( urllib . unquote ( url ) ) <EOL> if os . path . isdir ( path ) : <EOL> if path [ - <NUM_LIT:1> ] != os . sep : <EOL> url = url + '<STR_LIT:/>' <EOL> indexpath = os . path . join ( path , "<STR_LIT>" ) <EOL> if os . path . exists ( indexpath ) : <EOL> return self . open_file ( url + "<STR_LIT>" ) <EOL> try : <EOL> names = os . listdir ( path ) <EOL> except os . error , msg : <EOL> exc_type , exc_value , exc_tb = sys . exc_info ( ) <EOL> raise IOError , msg , exc_tb <EOL> names . sort ( ) <EOL> s = MyStringIO ( "<STR_LIT>" + url , { '<STR_LIT>' : '<STR_LIT>' } ) <EOL> s . write ( '<STR_LIT>' % <EOL> urllib . quote ( os . path . join ( path , "<STR_LIT>" ) ) ) <EOL> for name in names : <EOL> q = urllib . quote ( name ) <EOL> s . write ( '<STR_LIT>' % ( q , q ) ) <EOL> s . seek ( <NUM_LIT:0> ) <EOL> return s <EOL> return urllib . FancyURLopener . open_file ( self , url ) <EOL> class MyHTMLParser ( sgmllib . SGMLParser ) : <EOL> def __init__ ( self , url , verbose = VERBOSE , checker = None ) : <EOL> self . myverbose = verbose <EOL> self . checker = checker <EOL> self . base = None <EOL> self . links = { } <EOL> self . names = [ ] <EOL> self . url = url <EOL> sgmllib . SGMLParser . __init__ ( self ) <EOL> def check_name_id ( self , attributes ) : <EOL> """<STR_LIT>""" <EOL> for name , value in attributes : <EOL> if name == "<STR_LIT:name>" or name == "<STR_LIT:id>" : <EOL> if value in self . names : <EOL> self . checker . message ( "<STR_LIT>" , <EOL> value , self . url ) <EOL> else : self . names . append ( value ) <EOL> break <EOL> def unknown_starttag ( self , tag , attributes ) : <EOL> """<STR_LIT>""" <EOL> self . check_name_id ( attributes ) <EOL> def start_a ( self , attributes ) : <EOL> self . link_attr ( attributes , '<STR_LIT>' ) <EOL> self . check_name_id ( attributes ) <EOL> def end_a ( self ) : pass <EOL> def do_area ( self , attributes ) : <EOL> self . link_attr ( attributes , '<STR_LIT>' ) <EOL> self . check_name_id ( attributes ) <EOL> def do_body ( self , attributes ) : <EOL> self . link_attr ( attributes , '<STR_LIT>' , '<STR_LIT>' ) <EOL> self . check_name_id ( attributes ) <EOL> def do_img ( self , attributes ) : <EOL> self . link_attr ( attributes , '<STR_LIT:src>' , '<STR_LIT>' ) <EOL> self . check_name_id ( attributes ) <EOL> def do_frame ( self , attributes ) : <EOL> self . link_attr ( attributes , '<STR_LIT:src>' , '<STR_LIT>' ) <EOL> self . check_name_id ( attributes ) <EOL> def do_iframe ( self , attributes ) : <EOL> self . link_attr ( attributes , '<STR_LIT:src>' , '<STR_LIT>' ) <EOL> self . check_name_id ( attributes ) <EOL> def do_link ( self , attributes ) : <EOL> for name , value in attributes : <EOL> if name == "<STR_LIT>" : <EOL> parts = value . lower ( ) . split ( ) <EOL> if ( parts == [ "<STR_LIT>" ] <EOL> or parts == [ "<STR_LIT>" , "<STR_LIT>" ] ) : <EOL> self . link_attr ( attributes , "<STR_LIT>" ) <EOL> break <EOL> self . check_name_id ( attributes ) <EOL> def do_object ( self , attributes ) : <EOL> self . link_attr ( attributes , '<STR_LIT:data>' , '<STR_LIT>' ) <EOL> self . check_name_id ( attributes ) <EOL> def do_script ( self , attributes ) : <EOL> self . link_attr ( attributes , '<STR_LIT:src>' ) <EOL> self . check_name_id ( attributes ) <EOL> def do_table ( self , attributes ) : <EOL> self . link_attr ( attributes , '<STR_LIT>' ) <EOL> self . check_name_id ( attributes ) <EOL> def do_td ( self , attributes ) : <EOL> self . link_attr ( attributes , '<STR_LIT>' ) <EOL> self . check_name_id ( attributes ) <EOL> def do_th ( self , attributes ) : <EOL> self . link_attr ( attributes , '<STR_LIT>' ) <EOL> self . check_name_id ( attributes ) <EOL> def do_tr ( self , attributes ) : <EOL> self . link_attr ( attributes , '<STR_LIT>' ) <EOL> self . check_name_id ( attributes ) <EOL> def link_attr ( self , attributes , * args ) : <EOL> for name , value in attributes : <EOL> if name in args : <EOL> if value : value = value . strip ( ) <EOL> if value : self . links [ value ] = None <EOL> def do_base ( self , attributes ) : <EOL> for name , value in attributes : <EOL> if name == '<STR_LIT>' : <EOL> if value : value = value . strip ( ) <EOL> if value : <EOL> if self . checker : <EOL> self . checker . note ( <NUM_LIT:1> , "<STR_LIT>" , value ) <EOL> self . base = value <EOL> self . check_name_id ( attributes ) <EOL> def getlinks ( self ) : <EOL> return self . links . keys ( ) <EOL> def getbase ( self ) : <EOL> return self . base <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> main ( ) </s>
