<s> """<STR_LIT>""" <EOL> import re <EOL> import hashlib <EOL> from hanzo . warctools . record import ArchiveRecord , ArchiveParser <EOL> from hanzo . warctools . archive_detect import register_record_type <EOL> bad_lines = <NUM_LIT:5> <EOL> @ ArchiveRecord . HEADERS ( <EOL> DATE = '<STR_LIT>' , <EOL> TYPE = '<STR_LIT>' , <EOL> ID = '<STR_LIT>' , <EOL> CONCURRENT_TO = '<STR_LIT>' , <EOL> REFERS_TO = '<STR_LIT>' , <EOL> CONTENT_LENGTH = '<STR_LIT>' , <EOL> CONTENT_TYPE = '<STR_LIT:Content-Type>' , <EOL> URL = '<STR_LIT>' , <EOL> BLOCK_DIGEST = '<STR_LIT>' , <EOL> IP_ADDRESS = '<STR_LIT>' , <EOL> FILENAME = '<STR_LIT>' , <EOL> WARCINFO_ID = '<STR_LIT>' , <EOL> ) <EOL> class WarcRecord ( ArchiveRecord ) : <EOL> VERSION = "<STR_LIT>" <EOL> VERSION18 = "<STR_LIT>" <EOL> VERSION17 = "<STR_LIT>" <EOL> RESPONSE = "<STR_LIT>" <EOL> RESOURCE = "<STR_LIT>" <EOL> REQUEST = "<STR_LIT>" <EOL> METADATA = "<STR_LIT>" <EOL> CONVERSION = "<STR_LIT>" <EOL> WARCINFO = "<STR_LIT>" <EOL> def __init__ ( self , version = VERSION , headers = None , content = None , <EOL> errors = None ) : <EOL> ArchiveRecord . __init__ ( self , headers , content , errors ) <EOL> self . version = version <EOL> @ property <EOL> def id ( self ) : <EOL> return self . get_header ( self . ID ) <EOL> def _write_to ( self , out , nl ) : <EOL> """<STR_LIT>""" <EOL> out . write ( self . version ) <EOL> out . write ( nl ) <EOL> for k , v in self . headers : <EOL> if k not in ( self . CONTENT_TYPE , <EOL> self . CONTENT_LENGTH , <EOL> self . BLOCK_DIGEST ) : <EOL> out . write ( k ) <EOL> out . write ( "<STR_LIT>" ) <EOL> out . write ( v ) <EOL> out . write ( nl ) <EOL> content_type , content_buffer = self . content <EOL> content_buffer = buffer ( content_buffer ) <EOL> if content_type : <EOL> out . write ( self . CONTENT_TYPE ) <EOL> out . write ( "<STR_LIT>" ) <EOL> out . write ( content_type ) <EOL> out . write ( nl ) <EOL> if content_buffer is None : <EOL> content_buffer = "<STR_LIT>" <EOL> content_length = len ( content_buffer ) <EOL> out . write ( self . CONTENT_LENGTH ) <EOL> out . write ( "<STR_LIT>" ) <EOL> out . write ( str ( content_length ) ) <EOL> out . write ( nl ) <EOL> block_digest = self . block_digest ( content_buffer ) <EOL> out . write ( self . BLOCK_DIGEST ) <EOL> out . write ( "<STR_LIT>" ) <EOL> out . write ( block_digest ) <EOL> out . write ( nl ) <EOL> out . write ( nl ) <EOL> if content_buffer : <EOL> out . write ( content_buffer [ : content_length ] ) <EOL> out . write ( nl ) <EOL> out . write ( nl ) <EOL> out . flush ( ) <EOL> def repair ( self ) : <EOL> pass <EOL> def validate ( self ) : <EOL> return self . errors <EOL> @ classmethod <EOL> def make_parser ( self ) : <EOL> return WarcParser ( ) <EOL> def block_digest ( self , content_buffer ) : <EOL> block_hash = hashlib . sha256 ( ) <EOL> block_hash . update ( content_buffer ) <EOL> digest = "<STR_LIT>" % block_hash . hexdigest ( ) <EOL> return digest <EOL> def rx ( pat ) : <EOL> """<STR_LIT>""" <EOL> return re . compile ( pat , flags = re . IGNORECASE ) <EOL> version_rx = rx ( r'<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> header_rx = rx ( r'<STR_LIT>' '<STR_LIT>' ) <EOL> value_rx = rx ( r'<STR_LIT>' '<STR_LIT>' ) <EOL> nl_rx = rx ( '<STR_LIT>' ) <EOL> length_rx = rx ( '<STR_LIT>' % WarcRecord . CONTENT_LENGTH ) <EOL> type_rx = rx ( '<STR_LIT>' % WarcRecord . CONTENT_TYPE ) <EOL> required_headers = set ( ( <EOL> WarcRecord . TYPE . lower ( ) , <EOL> WarcRecord . ID . lower ( ) , <EOL> WarcRecord . CONTENT_LENGTH . lower ( ) , <EOL> WarcRecord . DATE . lower ( ) , <EOL> ) ) <EOL> class WarcParser ( ArchiveParser ) : <EOL> KNOWN_VERSIONS = set ( ( '<STR_LIT:1.0>' , '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> def __init__ ( self ) : <EOL> self . trailing_newlines = <NUM_LIT:0> <EOL> def parse ( self , stream , offset , line = None ) : <EOL> """<STR_LIT>""" <EOL> errors = [ ] <EOL> version = None <EOL> if line is None : <EOL> line = stream . readline ( ) <EOL> newlines = self . trailing_newlines <EOL> if newlines > <NUM_LIT:0> : <EOL> while line : <EOL> match = nl_rx . match ( line ) <EOL> if match and newlines > <NUM_LIT:0> : <EOL> if offset is not None : <EOL> offset += len ( line ) <EOL> newlines -= <NUM_LIT:1> <EOL> if match . group ( '<STR_LIT>' ) != '<STR_LIT>' : <EOL> errors . append ( ( '<STR_LIT>' , <EOL> match . group ( '<STR_LIT>' ) ) ) <EOL> line = stream . readline ( ) <EOL> if newlines == <NUM_LIT:0> : <EOL> break <EOL> else : <EOL> break <EOL> if newlines > <NUM_LIT:0> : <EOL> errors += ( '<STR_LIT>' , newlines ) <EOL> while line : <EOL> match = version_rx . match ( line ) <EOL> if match : <EOL> version = match . group ( '<STR_LIT:version>' ) <EOL> if offset is not None : <EOL> offset += len ( match . group ( '<STR_LIT>' ) ) <EOL> break <EOL> else : <EOL> if offset is not None : <EOL> offset += len ( line ) <EOL> if not nl_rx . match ( line ) : <EOL> errors . append ( ( '<STR_LIT>' , line ) ) <EOL> if len ( errors ) > bad_lines : <EOL> errors . append ( ( '<STR_LIT>' , ) ) <EOL> return ( None , errors , offset ) <EOL> line = stream . readline ( ) <EOL> if not line : <EOL> if version : <EOL> errors . append ( ( '<STR_LIT>' , version ) ) <EOL> self . trailing_newlines = <NUM_LIT:0> <EOL> return ( None , errors , offset ) <EOL> if line : <EOL> content_length = <NUM_LIT:0> <EOL> content_type = None <EOL> record = WarcRecord ( errors = errors , version = version ) <EOL> if match . group ( '<STR_LIT>' ) != '<STR_LIT>' : <EOL> record . error ( '<STR_LIT>' , match . group ( '<STR_LIT>' ) ) <EOL> if match . group ( '<STR_LIT>' ) not in self . KNOWN_VERSIONS : <EOL> record . error ( '<STR_LIT>' <EOL> % ( "<STR_LIT:U+002C>" . join ( self . KNOWN_VERSIONS ) ) , <EOL> match . group ( '<STR_LIT>' ) ) <EOL> prefix = match . group ( '<STR_LIT>' ) <EOL> if prefix : <EOL> record . error ( '<STR_LIT>' , prefix ) <EOL> line = stream . readline ( ) <EOL> while line and not nl_rx . match ( line ) : <EOL> match = header_rx . match ( line ) <EOL> if match : <EOL> if match . group ( '<STR_LIT>' ) != '<STR_LIT>' : <EOL> record . error ( '<STR_LIT>' , <EOL> match . group ( '<STR_LIT>' ) ) <EOL> name = match . group ( '<STR_LIT:name>' ) . strip ( ) <EOL> value = [ match . group ( '<STR_LIT:value>' ) . strip ( ) ] <EOL> line = stream . readline ( ) <EOL> match = value_rx . match ( line ) <EOL> while match : <EOL> if match . group ( '<STR_LIT>' ) != '<STR_LIT>' : <EOL> record . error ( '<STR_LIT>' , <EOL> line , match . group ( '<STR_LIT>' ) ) <EOL> value . append ( match . group ( '<STR_LIT:value>' ) . strip ( ) ) <EOL> line = stream . readline ( ) <EOL> match = value_rx . match ( line ) <EOL> value = "<STR_LIT:U+0020>" . join ( value ) <EOL> if type_rx . match ( name ) : <EOL> if value : <EOL> content_type = value <EOL> else : <EOL> record . error ( '<STR_LIT>' , name , value ) <EOL> elif length_rx . match ( name ) : <EOL> try : <EOL> content_length = int ( value ) <EOL> except ValueError : <EOL> record . error ( '<STR_LIT>' , name , value ) <EOL> else : <EOL> record . headers . append ( ( name , value ) ) <EOL> if content_length is not None : <EOL> if content_length > <NUM_LIT:0> : <EOL> content = [ ] <EOL> length = <NUM_LIT:0> <EOL> while length < content_length : <EOL> line = stream . readline ( ) <EOL> if not line : <EOL> break <EOL> content . append ( line ) <EOL> length += len ( line ) <EOL> content = "<STR_LIT>" . join ( content ) <EOL> content , line = content [ <NUM_LIT:0> : content_length ] , content [ content_length : ] <EOL> if len ( content ) != content_length : <EOL> record . error ( '<STR_LIT>' , <EOL> len ( content ) , content_length ) <EOL> record . content = ( content_type , content ) <EOL> if nl_rx . match ( line ) : <EOL> self . trailing_newlines = <NUM_LIT:1> <EOL> else : <EOL> self . trailing_newlines = <NUM_LIT:2> <EOL> else : <EOL> record . error ( '<STR_LIT>' , WarcRecord . CONTENT_LENGTH ) <EOL> self . trailing_newlines = <NUM_LIT:2> <EOL> return ( record , ( ) , offset ) <EOL> def trim ( self , stream ) : <EOL> """<STR_LIT>""" <EOL> newlines = self . trailing_newlines <EOL> self . trailing_newlines = <NUM_LIT:0> <EOL> errors = [ ] <EOL> if newlines : <EOL> line = stream . readline ( ) <EOL> while line : <EOL> match = nl_rx . match ( line ) <EOL> if match : <EOL> if match . group ( '<STR_LIT>' ) != '<STR_LIT>' : <EOL> errors . append ( '<STR_LIT>' , <EOL> match . group ( '<STR_LIT>' ) ) <EOL> newlines -= <NUM_LIT:1> <EOL> if newlines == <NUM_LIT:0> : <EOL> break <EOL> else : <EOL> newlines = <NUM_LIT:0> <EOL> errors . append ( ( '<STR_LIT>' , line ) ) <EOL> line = stream . readline ( ) <EOL> if newlines > <NUM_LIT:0> : <EOL> errors . append ( <EOL> ( '<STR_LIT>' , <EOL> newlines ) ) <EOL> return errors <EOL> blank_rx = rx ( r'<STR_LIT>' ) <EOL> register_record_type ( version_rx , WarcRecord ) <EOL> register_record_type ( blank_rx , WarcRecord ) <EOL> def make_response ( id , date , url , content , request_id ) : <EOL> headers = [ <EOL> ( WarcRecord . TYPE , WarcRecord . RESPONSE ) , <EOL> ( WarcRecord . ID , id ) , <EOL> ( WarcRecord . DATE , date ) , <EOL> ( WarcRecord . URL , url ) , <EOL> ] <EOL> if request_id : <EOL> headers . append ( ( WarcRecord . CONCURRENT_TO , request_id ) ) <EOL> record = WarcRecord ( headers = headers , content = content ) <EOL> return record <EOL> def make_request ( request_id , date , url , content , response_id ) : <EOL> headers = [ <EOL> ( WarcRecord . TYPE , WarcRecord . REQUEST ) , <EOL> ( WarcRecord . ID , request_id ) , <EOL> ( WarcRecord . DATE , date ) , <EOL> ( WarcRecord . URL , url ) , <EOL> ] <EOL> if response_id : <EOL> headers . append ( ( WarcRecord . CONCURRENT_TO , response_id ) ) <EOL> record = WarcRecord ( headers = headers , content = content ) <EOL> return record <EOL> def make_metadata ( meta_id , date , content , concurrent_to = None , url = None ) : <EOL> headers = [ <EOL> ( WarcRecord . TYPE , WarcRecord . METADATA ) , <EOL> ( WarcRecord . ID , meta_id ) , <EOL> ( WarcRecord . DATE , date ) , <EOL> ] <EOL> if concurrent_to : <EOL> headers . append ( ( WarcRecord . CONCURRENT_TO , concurrent_to ) ) <EOL> if url : <EOL> headers . append ( ( WarcRecord . URL , url ) ) <EOL> record = WarcRecord ( headers = headers , content = content ) <EOL> return record <EOL> def make_conversion ( conv_id , date , content , refers_to = None , url = None ) : <EOL> headers = [ <EOL> ( WarcRecord . TYPE , WarcRecord . CONVERSION ) , <EOL> ( WarcRecord . ID , conv_id ) , <EOL> ( WarcRecord . DATE , date ) , <EOL> ] <EOL> if refers_to : <EOL> headers . append ( ( WarcRecord . REFERS_TO , refers_to ) ) <EOL> if url : <EOL> headers . append ( ( WarcRecord . URL , url ) ) <EOL> record = WarcRecord ( headers = headers , content = content ) <EOL> return record <EOL> def warc_datetime_str ( d ) : <EOL> s = d . isoformat ( ) <EOL> if '<STR_LIT:.>' in s : <EOL> s = s [ : s . find ( '<STR_LIT:.>' ) ] <EOL> return s + '<STR_LIT>' </s>
