<s> import sys <EOL> import re <EOL> import codecs <EOL> from collections import namedtuple <EOL> from os . path import basename <EOL> Textbound = namedtuple ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> TEXTBOUND_RE = re . compile ( r'<STR_LIT>' ) <EOL> class FormatError ( Exception ) : <EOL> pass <EOL> def txt_for_ann ( fn ) : <EOL> tfn = re . sub ( r'<STR_LIT>' , '<STR_LIT>' , fn ) <EOL> if tfn == fn : <EOL> raise FormatError <EOL> return tfn <EOL> def parse_textbound ( s ) : <EOL> m = TEXTBOUND_RE . match ( s ) <EOL> if not m : <EOL> raise FormatError <EOL> id_ , type_ , start , end , text = m . groups ( ) <EOL> start , end = int ( start ) , int ( end ) <EOL> return Textbound ( id_ , type_ , start , end , text ) <EOL> def process ( fn ) : <EOL> textbounds = [ ] <EOL> with codecs . open ( fn , '<STR_LIT>' , encoding = '<STR_LIT:utf8>' , errors = '<STR_LIT:strict>' ) as f : <EOL> for l in f : <EOL> l = l . rstrip ( '<STR_LIT:\n>' ) <EOL> if not l or l . isspace ( ) : <EOL> continue <EOL> if l [ <NUM_LIT:0> ] != '<STR_LIT:T>' : <EOL> continue <EOL> else : <EOL> textbounds . append ( parse_textbound ( l ) ) <EOL> with codecs . open ( txt_for_ann ( fn ) , '<STR_LIT>' , encoding = '<STR_LIT:utf8>' , <EOL> errors = '<STR_LIT:strict>' ) as f : <EOL> text = f . read ( ) <EOL> for id_ , type_ , start , end , ttext in textbounds : <EOL> try : <EOL> assert text [ start : end ] == ttext <EOL> except : <EOL> print '<STR_LIT>' % ( basename ( fn ) , id_ , start , end ) <EOL> print '<STR_LIT>' % ttext . encode ( '<STR_LIT:utf-8>' ) . replace ( '<STR_LIT:\n>' , '<STR_LIT>' ) <EOL> print '<STR_LIT>' % text [ start : end ] . encode ( '<STR_LIT:utf-8>' ) . replace ( '<STR_LIT:\n>' , '<STR_LIT>' ) <EOL> def main ( argv = None ) : <EOL> if argv is None : <EOL> argv = sys . argv <EOL> if len ( argv ) < <NUM_LIT:2> : <EOL> print >> sys . stderr , '<STR_LIT>' , argv [ <NUM_LIT:0> ] , '<STR_LIT>' <EOL> return <NUM_LIT:1> <EOL> for fn in argv [ <NUM_LIT:1> : ] : <EOL> process ( fn ) <EOL> return <NUM_LIT:0> <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> sys . exit ( main ( sys . argv ) ) </s>
