def browse(self):
        """Utility to browse through the records in the warc file.
        
        This returns an iterator over (record, offset, size) for each record in 
        the file. If the file is gzip compressed, the offset and size will 
        corresponds to the compressed file. 
        
        The payload of each record is limited to 1MB to keep memory consumption 
        under control.
        """
        offset = 0
        for record in self.reader:
            # Just read the first 1MB of the payload.
            # This will make sure memory consuption is under control and it 
            # is possible to look at the first MB of the payload, which is 
            # typically sufficient to read http headers in the payload.
            record.payload = StringIO(record.payload.read(1024*1024))
            self.reader.finish_reading_current_record()
            next_offset = self.tell()
            yield record, offset, next_offset-offset
            offset = next_offset
def browse ( self ) : offset = 0 for record in self . reader : record . payload = StringIO ( record . payload . read ( 1024 * 1024 ) ) self . reader . finish_reading_current_record ( ) next_offset = self . tell ( ) yield record , offset , next_offset - offset offset = next_offset
Utility to browse through the records in the warc file.
        
        This returns an iterator over (record, offset, size) for each record in 
        the file. If the file is gzip compressed, the offset and size will 
        corresponds to the compressed file. 
        
        The payload of each record is limited to 1MB to keep memory consumption 
        under control.