<s> import hashlib , uuid , base64 <EOL> import datetime <EOL> from hanzo . warctools import WarcRecord <EOL> @ staticmethod <EOL> def make_warc_uuid ( text = None ) : <EOL> if text is None : <EOL> text = str ( datetime . datetime . utcnow ( ) ) <EOL> return "<STR_LIT>" % uuid . UUID ( hashlib . sha1 ( text ) . hexdigest ( ) [ <NUM_LIT:0> : <NUM_LIT:32> ] ) <EOL> WarcRecord . make_warc_uuid = make_warc_uuid <EOL> def block_digest ( self , content_buffer ) : <EOL> hash = base64 . b32encode ( hashlib . sha1 ( content_buffer ) . digest ( ) ) <EOL> return "<STR_LIT>" % hash <EOL> WarcRecord . block_digest = block_digest <EOL> """<STR_LIT>""" <EOL> class WarcinfoFields : <EOL> _common_fields = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ] <EOL> CONFORMS_TO = '<STR_LIT>' <EOL> def __init__ ( self , fields = None , defaults = True , ** kwargs ) : <EOL> if fields is None : <EOL> self . _fields = [ ] <EOL> else : <EOL> self . _fields = fields <EOL> for field in self . _common_fields : <EOL> if field in kwargs : <EOL> self . _fields . append ( ( field , kwargs [ field ] ) ) <EOL> if defaults : <EOL> if not self . has_field ( '<STR_LIT>' ) : <EOL> self . _fields . append ( ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> if not self . has_field ( '<STR_LIT>' ) : <EOL> self . _fields . append ( ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> if not self . has_field ( '<STR_LIT>' ) : <EOL> self . _fields . append ( ( '<STR_LIT>' , self . CONFORMS_TO ) ) <EOL> if not self . has_field ( '<STR_LIT>' ) : <EOL> self . _fields . append ( ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> """<STR_LIT>""" <EOL> def append ( self , tuple ) : <EOL> self . _fields . append ( tuple ) <EOL> """<STR_LIT>""" <EOL> def has_field ( self , key ) : <EOL> for ( k , v ) in self . _fields : <EOL> if k . lower ( ) == key . lower ( ) : <EOL> return True <EOL> return False <EOL> """<STR_LIT>""" <EOL> def get_field ( self , key ) : <EOL> for ( k , v ) in self . _fields : <EOL> if k . lower ( ) == key . lower ( ) : <EOL> return v <EOL> return None <EOL> """<STR_LIT>""" <EOL> def set_field ( self , key , value = None ) : <EOL> if value is None : <EOL> key , value = key <EOL> for n , i in enumerate ( self . _fields ) : <EOL> k , v = i <EOL> if k . lower ( ) == key . lower ( ) : <EOL> self . _fields [ n ] = ( key , value ) <EOL> return <EOL> self . append ( ( key , value ) ) <EOL> """<STR_LIT>""" <EOL> def __str__ ( self ) : <EOL> return '<STR_LIT:\r\n>' . join ( k + '<STR_LIT>' + v for ( k , v ) in self . _fields ) + '<STR_LIT>' <EOL> def __getitem__ ( self , key ) : <EOL> if key == <NUM_LIT:0> : <EOL> return '<STR_LIT>' <EOL> elif key == <NUM_LIT:1> : <EOL> str ( self ) <EOL> def __len__ ( self ) : <EOL> return len ( self . _fields ) <EOL> def __iter__ ( self ) : <EOL> for i in [ '<STR_LIT>' , str ( self ) ] : <EOL> yield i <EOL> """<STR_LIT>""" <EOL> class WarcinfoRecord ( WarcRecord ) : <EOL> def __init__ ( self , id = None , date = None , filename = None , content = None , <EOL> headers = None , defaults = True ) : <EOL> if headers is None : <EOL> headers = [ ] <EOL> headers . append ( ( WarcRecord . TYPE , WarcRecord . WARCINFO ) ) <EOL> if id : <EOL> headers . append ( ( WarcRecord . ID , id ) ) <EOL> elif defaults : <EOL> headers . append ( ( WarcRecord . ID , self . make_warc_uuid ( ) ) ) <EOL> if date : <EOL> headers . append ( ( WarcRecord . DATE , date ) ) <EOL> elif defaults : <EOL> date = datetime . datetime . utcnow ( ) . strftime ( '<STR_LIT>' ) <EOL> headers . append ( ( WarcRecord . DATE , date ) ) <EOL> if filename : <EOL> headers . append ( ( WarcRecord . FILENAME , filename ) ) <EOL> if not content and defaults : <EOL> content = WarcinfoFields ( defaults = defaults ) <EOL> super ( WarcinfoRecord , self ) . __init__ ( headers = headers , content = content ) <EOL> class WarcRequestRecord ( WarcRecord ) : <EOL> def __init__ ( self , id = None , date = None , url = None , block = None , <EOL> concurrent_to = None , headers = None , defaults = True ) : <EOL> assert block is not None <EOL> if headers is None : <EOL> headers = [ ] <EOL> headers . append ( ( WarcRecord . TYPE , WarcRecord . REQUEST ) ) <EOL> if id : <EOL> headers . append ( ( WarcRecord . ID , id ) ) <EOL> elif defaults : <EOL> headers . append ( ( WarcRecord . ID , self . make_warc_uuid ( ) ) ) <EOL> if date : <EOL> headers . append ( ( WarcRecord . DATE , date ) ) <EOL> elif defaults : <EOL> date = datetime . datetime . utcnow ( ) . strftime ( '<STR_LIT>' ) <EOL> headers . append ( ( WarcRecord . DATE , date ) ) <EOL> if url : <EOL> headers . append ( ( WarcRecord . URL , url ) ) <EOL> if concurrent_to : <EOL> headers . append ( ( WarcRecord . CONCURRENT_TO , concurrent_to ) ) <EOL> content = ( '<STR_LIT>' , block ) <EOL> super ( WarcRequestRecord , self ) . __init__ ( headers = headers , content = content ) <EOL> class WarcResponseRecord ( WarcRecord ) : <EOL> def __init__ ( self , id = None , date = None , url = None , block = None , <EOL> concurrent_to = None , headers = None , defaults = True ) : <EOL> assert block is not None <EOL> if headers is None : <EOL> headers = [ ] <EOL> headers . append ( ( WarcRecord . TYPE , WarcRecord . RESPONSE ) ) <EOL> if id : <EOL> headers . append ( ( WarcRecord . ID , id ) ) <EOL> elif defaults : <EOL> headers . append ( ( WarcRecord . ID , self . make_warc_uuid ( ) ) ) <EOL> if date : <EOL> headers . append ( ( WarcRecord . DATE , date ) ) <EOL> elif defaults : <EOL> date = datetime . datetime . utcnow ( ) . strftime ( '<STR_LIT>' ) <EOL> headers . append ( ( WarcRecord . DATE , date ) ) <EOL> if url : <EOL> headers . append ( ( WarcRecord . URL , url ) ) <EOL> if concurrent_to : <EOL> headers . append ( ( WarcRecord . CONCURRENT_TO , concurrent_to ) ) <EOL> content = ( '<STR_LIT>' , block ) <EOL> super ( WarcResponseRecord , self ) . __init__ ( headers = headers , content = content ) </s>
