<s> from __future__ import with_statement <EOL> import sys <EOL> import os <EOL> import re <EOL> import codecs <EOL> INPUT_ENCODING = "<STR_LIT>" <EOL> OUTPUT_ENCODING = "<STR_LIT>" <EOL> ENTITY_TYPE = "<STR_LIT>" <EOL> ATTR_TYPE = "<STR_LIT>" <EOL> FILE_PREFIX = "<STR_LIT>" <EOL> output_directory = None <EOL> def output ( docid , text , anns ) : <EOL> global output_directory <EOL> if output_directory is None : <EOL> txtout = sys . stdout <EOL> soout = sys . stdout <EOL> else : <EOL> outfn = os . path . join ( output_directory , FILE_PREFIX + docid ) <EOL> txtout = codecs . open ( outfn + '<STR_LIT>' , '<STR_LIT>' , encoding = OUTPUT_ENCODING ) <EOL> soout = codecs . open ( outfn + '<STR_LIT>' , '<STR_LIT>' , encoding = OUTPUT_ENCODING ) <EOL> txtout . write ( text ) <EOL> idseq = <NUM_LIT:1> <EOL> for start , end , type_ , text in anns : <EOL> print >> soout , "<STR_LIT>" % ( idseq , ENTITY_TYPE , start , end , <EOL> text ) <EOL> print >> soout , "<STR_LIT>" % ( idseq , ATTR_TYPE , idseq , type_ ) <EOL> idseq += <NUM_LIT:1> <EOL> if output_directory is not None : <EOL> txtout . close ( ) <EOL> soout . close ( ) <EOL> def parse ( s ) : <EOL> text , anns = "<STR_LIT>" , [ ] <EOL> s = re . sub ( r'<STR_LIT>' , r'<STR_LIT>' , s ) <EOL> s = re . sub ( r'<STR_LIT>' , r'<STR_LIT>' , s ) <EOL> rest = s . strip ( ) <EOL> while True : <EOL> m = re . match ( r'<STR_LIT>' , rest ) <EOL> if not m : <EOL> break <EOL> pre , type_ , tagged , rest = m . groups ( ) <EOL> text += pre <EOL> anns . append ( ( len ( text ) , len ( text ) + len ( tagged ) , type_ , tagged ) ) <EOL> text += tagged <EOL> text += rest <EOL> return text , anns <EOL> def process ( fn ) : <EOL> docnum = <NUM_LIT:1> <EOL> sentences = [ ] <EOL> with codecs . open ( fn , encoding = INPUT_ENCODING ) as f : <EOL> for l in f : <EOL> l = l . strip ( '<STR_LIT>' ) <EOL> try : <EOL> PMID , title , body = l . split ( '<STR_LIT:\t>' , <NUM_LIT:2> ) <EOL> except ValueError : <EOL> assert False , "<STR_LIT>" % l <EOL> body = body . replace ( '<STR_LIT:\t>' , '<STR_LIT:U+0020>' ) <EOL> t_text , t_anns = parse ( title ) <EOL> b_text , b_anns = parse ( body ) <EOL> t_text += '<STR_LIT:\n>' <EOL> b_text += '<STR_LIT:\n>' <EOL> text = t_text + b_text <EOL> anns = t_anns + [ ( a [ <NUM_LIT:0> ] + len ( t_text ) , a [ <NUM_LIT:1> ] + len ( t_text ) , a [ <NUM_LIT:2> ] , a [ <NUM_LIT:3> ] ) <EOL> for a in b_anns ] <EOL> output ( PMID , text , anns ) <EOL> def main ( argv ) : <EOL> global output_directory <EOL> output_directory = None <EOL> filenames = argv [ <NUM_LIT:1> : ] <EOL> if len ( argv ) > <NUM_LIT:2> and argv [ <NUM_LIT:1> ] == "<STR_LIT>" : <EOL> output_directory = argv [ <NUM_LIT:2> ] <EOL> print >> sys . stderr , "<STR_LIT>" % output_directory <EOL> filenames = argv [ <NUM_LIT:3> : ] <EOL> fail_count = <NUM_LIT:0> <EOL> for fn in filenames : <EOL> try : <EOL> process ( fn ) <EOL> except Exception , e : <EOL> print >> sys . stderr , "<STR_LIT>" % ( fn , e ) <EOL> fail_count += <NUM_LIT:1> <EOL> if fail_count > <NUM_LIT:0> : <EOL> print >> sys . stderr , """<STR_LIT>""" % ( fail_count , len ( filenames ) ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> sys . exit ( main ( sys . argv ) ) </s>
