<s> '''<STR_LIT>''' <EOL> from __future__ import with_statement <EOL> from collections import defaultdict <EOL> from itertools import chain <EOL> from sys import argv , path as sys_path , stderr , stdout <EOL> from os . path import dirname , join as path_join <EOL> from xml . etree import ElementTree <EOL> from ptbesc import unescape as ptb_unescape <EOL> try : <EOL> from collections import namedtuple <EOL> except ImportError : <EOL> sys_path . append ( path_join ( dirname ( __file__ ) , '<STR_LIT:..>' , '<STR_LIT:..>' , '<STR_LIT>' ) ) <EOL> from altnamedtuple import namedtuple <EOL> try : <EOL> from annotation import ( BinaryRelationAnnotation , EquivAnnotation , <EOL> TextBoundAnnotation ) <EOL> except ImportError : <EOL> sys_path . append ( path_join ( dirname ( __file__ ) , '<STR_LIT:..>' ) ) <EOL> from annotation import ( BinaryRelationAnnotation , EquivAnnotation , <EOL> TextBoundAnnotation ) <EOL> Token = namedtuple ( '<STR_LIT>' , ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:start>' , '<STR_LIT:end>' , '<STR_LIT>' , '<STR_LIT>' , ) ) <EOL> def _escape_pos_tags ( pos ) : <EOL> pos_res = pos <EOL> for _from , to in ( <EOL> ( "<STR_LIT:'>" , '<STR_LIT>' , ) , <EOL> ( '<STR_LIT:">' , '<STR_LIT>' , ) , <EOL> ( '<STR_LIT:$>' , '<STR_LIT>' , ) , <EOL> ( '<STR_LIT:U+002C>' , '<STR_LIT>' , ) , <EOL> ( '<STR_LIT:.>' , '<STR_LIT>' , ) , <EOL> ( '<STR_LIT::>' , '<STR_LIT>' , ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' , ) , <EOL> ) : <EOL> pos_res = pos_res . replace ( _from , to ) <EOL> return pos_res <EOL> def _token_by_ids ( soup ) : <EOL> token_by_ids = defaultdict ( dict ) <EOL> for sent_e in _find_sentences_element ( soup ) . getiterator ( '<STR_LIT>' ) : <EOL> sent_id = int ( sent_e . get ( '<STR_LIT:id>' ) ) <EOL> for tok_e in sent_e . getiterator ( '<STR_LIT>' ) : <EOL> tok_id = int ( tok_e . get ( '<STR_LIT:id>' ) ) <EOL> tok_word = unicode ( tok_e . find ( '<STR_LIT>' ) . text ) <EOL> tok_lemma = unicode ( tok_e . find ( '<STR_LIT>' ) . text ) <EOL> tok_start = int ( tok_e . find ( '<STR_LIT>' ) . text ) <EOL> tok_end = int ( tok_e . find ( '<STR_LIT>' ) . text ) <EOL> tok_pos = unicode ( tok_e . find ( '<STR_LIT>' ) . text ) <EOL> tok_ner = unicode ( tok_e . find ( '<STR_LIT>' ) . text ) <EOL> token_by_ids [ sent_id ] [ tok_id ] = Token ( <EOL> word = tok_word , <EOL> lemma = tok_lemma , <EOL> start = tok_start , <EOL> end = tok_end , <EOL> pos = _escape_pos_tags ( tok_pos ) , <EOL> ner = tok_ner <EOL> ) <EOL> return token_by_ids <EOL> def _tok_it ( token_by_ids ) : <EOL> for s_id in sorted ( k for k in token_by_ids ) : <EOL> for t_id in sorted ( k for k in token_by_ids [ s_id ] ) : <EOL> yield s_id , t_id , token_by_ids [ s_id ] [ t_id ] <EOL> def _soup ( xml ) : <EOL> return ElementTree . fromstring ( xml . encode ( '<STR_LIT:utf-8>' ) ) <EOL> def token_offsets ( xml ) : <EOL> soup = _soup ( xml ) <EOL> token_by_ids = _token_by_ids ( soup ) <EOL> return [ ( tok . start , tok . end ) for _ , _ , tok in _tok_it ( token_by_ids ) ] <EOL> def sentence_offsets ( xml ) : <EOL> soup = _soup ( xml ) <EOL> token_by_ids = _token_by_ids ( soup ) <EOL> sent_min_max = defaultdict ( lambda : ( <NUM_LIT:2> ** <NUM_LIT:32> , - <NUM_LIT:1> , ) ) <EOL> for s_id , _ , tok in _tok_it ( token_by_ids ) : <EOL> s_entry = sent_min_max [ s_id ] <EOL> sent_min_max [ s_id ] = ( min ( tok . start , s_entry [ <NUM_LIT:0> ] ) , max ( tok . end , s_entry [ <NUM_LIT:1> ] ) , ) <EOL> return sorted ( ( s_start , s_end ) for s_start , s_end in sent_min_max . itervalues ( ) ) <EOL> def text ( xml ) : <EOL> soup = _soup ( xml ) <EOL> token_by_ids = _token_by_ids ( soup ) <EOL> max_offset = - <NUM_LIT:1> <EOL> for _ , _ , tok in _tok_it ( token_by_ids ) : <EOL> max_offset = max ( max_offset , tok . end ) <EOL> text = list ( '<STR_LIT:U+0020>' * max_offset ) <EOL> for _ , _ , tok in _tok_it ( token_by_ids ) : <EOL> unesc_word = ptb_unescape ( tok . word ) <EOL> text [ tok . start : len ( unesc_word ) ] = unesc_word <EOL> return u'<STR_LIT>' . join ( text ) <EOL> def _pos ( xml , start_id = <NUM_LIT:1> ) : <EOL> soup = _soup ( xml ) <EOL> token_by_ids = _token_by_ids ( soup ) <EOL> curr_id = start_id <EOL> for s_id , t_id , tok in _tok_it ( token_by_ids ) : <EOL> yield s_id , t_id , TextBoundAnnotation ( ( ( tok . start , tok . end , ) , ) , <EOL> '<STR_LIT>' % curr_id , tok . pos , '<STR_LIT>' ) <EOL> curr_id += <NUM_LIT:1> <EOL> def pos ( xml , start_id = <NUM_LIT:1> ) : <EOL> return ( a for _ , _ , a in _pos ( xml , start_id = start_id ) ) <EOL> def ner ( xml , start_id = <NUM_LIT:1> ) : <EOL> soup = _soup ( xml ) <EOL> token_by_ids = _token_by_ids ( soup ) <EOL> nes = [ ] <EOL> last_ne_tok = None <EOL> prev_tok = None <EOL> for _ , _ , tok in _tok_it ( token_by_ids ) : <EOL> if tok . ner != '<STR_LIT:O>' : <EOL> if last_ne_tok is None : <EOL> last_ne_tok = tok <EOL> elif tok . ner != last_ne_tok . ner : <EOL> nes . append ( ( last_ne_tok . start , prev_tok . end , last_ne_tok . ner , ) ) <EOL> last_ne_tok = tok <EOL> else : <EOL> pass <EOL> elif last_ne_tok is not None : <EOL> nes . append ( ( last_ne_tok . start , prev_tok . end , last_ne_tok . ner , ) ) <EOL> last_ne_tok = None <EOL> prev_tok = tok <EOL> else : <EOL> if last_ne_tok is not None : <EOL> nes . append ( ( last_ne_tok . start , prev_tok . end , last_ne_tok . ner , ) ) <EOL> curr_id = start_id <EOL> for start , end , _type in nes : <EOL> yield TextBoundAnnotation ( ( ( start , end ) , ) , '<STR_LIT>' % curr_id , _type , '<STR_LIT>' ) <EOL> curr_id += <NUM_LIT:1> <EOL> def coref ( xml , start_id = <NUM_LIT:1> ) : <EOL> soup = _soup ( xml ) <EOL> token_by_ids = _token_by_ids ( soup ) <EOL> docs_e = soup . findall ( '<STR_LIT>' ) <EOL> assert len ( docs_e ) == <NUM_LIT:1> <EOL> docs_e = docs_e [ <NUM_LIT:0> ] <EOL> corefs_e = docs_e . findall ( '<STR_LIT>' ) <EOL> if not corefs_e : <EOL> raise StopIteration <EOL> assert len ( corefs_e ) == <NUM_LIT:1> <EOL> corefs_e = corefs_e [ <NUM_LIT:0> ] <EOL> curr_id = start_id <EOL> for coref_e in corefs_e : <EOL> if corefs_e . tag != '<STR_LIT>' : <EOL> continue <EOL> chain = [ ] <EOL> for mention_e in coref_e . getiterator ( '<STR_LIT>' ) : <EOL> sentence_id = int ( mention_e . find ( '<STR_LIT>' ) . text ) <EOL> start_tok_id = int ( mention_e . find ( '<STR_LIT:start>' ) . text ) <EOL> end_tok_id = int ( mention_e . find ( '<STR_LIT:end>' ) . text ) - <NUM_LIT:1> <EOL> mention_id = '<STR_LIT>' % ( curr_id , ) <EOL> chain . append ( mention_id ) <EOL> curr_id += <NUM_LIT:1> <EOL> yield TextBoundAnnotation ( <EOL> ( ( token_by_ids [ sentence_id ] [ start_tok_id ] . start , <EOL> token_by_ids [ sentence_id ] [ end_tok_id ] . end ) , ) , <EOL> mention_id , '<STR_LIT>' , '<STR_LIT>' ) <EOL> yield EquivAnnotation ( '<STR_LIT>' , chain , '<STR_LIT>' ) <EOL> def _find_sentences_element ( soup ) : <EOL> docs_e = soup . findall ( '<STR_LIT>' ) <EOL> assert len ( docs_e ) == <NUM_LIT:1> <EOL> docs_e = docs_e [ <NUM_LIT:0> ] <EOL> sents_e = docs_e . findall ( '<STR_LIT>' ) <EOL> assert len ( sents_e ) == <NUM_LIT:1> <EOL> sents_e = sents_e [ <NUM_LIT:0> ] <EOL> return sents_e <EOL> def _dep ( xml , source_element = '<STR_LIT>' ) : <EOL> soup = _soup ( xml ) <EOL> token_by_ids = _token_by_ids ( soup ) <EOL> ann_by_ids = defaultdict ( dict ) <EOL> for s_id , t_id , ann in _pos ( xml ) : <EOL> ann_by_ids [ s_id ] [ t_id ] = ann <EOL> yield ann <EOL> curr_rel_id = <NUM_LIT:1> <EOL> for sent_e in _find_sentences_element ( soup ) . getiterator ( '<STR_LIT>' ) : <EOL> sent_id = int ( sent_e . get ( '<STR_LIT:id>' ) ) <EOL> deps_e = sent_e . findall ( source_element ) <EOL> if len ( deps_e ) == <NUM_LIT:0> : <EOL> deps_e = list ( e for e in sent_e . getiterator ( '<STR_LIT>' ) <EOL> if e . attrib [ '<STR_LIT:type>' ] == source_element ) <EOL> assert len ( deps_e ) == <NUM_LIT:1> <EOL> deps_e = deps_e [ <NUM_LIT:0> ] <EOL> for dep_e in deps_e : <EOL> if dep_e . tag != '<STR_LIT>' : <EOL> continue <EOL> dep_type = dep_e . get ( '<STR_LIT:type>' ) <EOL> assert dep_type is not None <EOL> if dep_type == '<STR_LIT:root>' : <EOL> continue <EOL> gov_tok_id = int ( dep_e . find ( '<STR_LIT>' ) . get ( '<STR_LIT>' ) ) <EOL> dep_tok_id = int ( dep_e . find ( '<STR_LIT>' ) . get ( '<STR_LIT>' ) ) <EOL> yield BinaryRelationAnnotation ( <EOL> '<STR_LIT>' % curr_rel_id , dep_type , <EOL> '<STR_LIT>' , ann_by_ids [ sent_id ] [ gov_tok_id ] . id , <EOL> '<STR_LIT>' , ann_by_ids [ sent_id ] [ dep_tok_id ] . id , <EOL> '<STR_LIT>' <EOL> ) <EOL> curr_rel_id += <NUM_LIT:1> <EOL> def basic_dep ( xml ) : <EOL> return _dep ( xml ) <EOL> def collapsed_dep ( xml ) : <EOL> return _dep ( xml , source_element = '<STR_LIT>' ) <EOL> def collapsed_ccproc_dep ( xml ) : <EOL> return _dep ( xml , source_element = '<STR_LIT>' ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> STANFORD_XML = '''<STR_LIT>''' <EOL> def _test_xml ( xml_string ) : <EOL> stdout . write ( '<STR_LIT>' ) <EOL> stdout . write ( text ( xml_string ) . encode ( '<STR_LIT:utf-8>' ) ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT>' ) <EOL> for ann in pos ( xml_string ) : <EOL> stdout . write ( unicode ( ann ) ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT>' ) <EOL> for ann in ner ( xml_string ) : <EOL> stdout . write ( unicode ( ann ) ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT>' ) <EOL> for ann in coref ( xml_string ) : <EOL> stdout . write ( unicode ( ann ) ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT>' ) <EOL> for ann in basic_dep ( xml_string ) : <EOL> stdout . write ( unicode ( ann ) ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT>' ) <EOL> for ann in basic_dep ( xml_string ) : <EOL> stdout . write ( unicode ( ann ) ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT>' ) <EOL> for ann in collapsed_dep ( xml_string ) : <EOL> stdout . write ( unicode ( ann ) ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT>' ) <EOL> for ann in collapsed_ccproc_dep ( xml_string ) : <EOL> stdout . write ( unicode ( ann ) ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT>' ) <EOL> stdout . write ( unicode ( token_offsets ( xml_string ) ) ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> stdout . write ( '<STR_LIT>' ) <EOL> stdout . write ( unicode ( sentence_offsets ( xml_string ) ) ) <EOL> stdout . write ( '<STR_LIT:\n>' ) <EOL> if len ( argv ) < <NUM_LIT:2> : <EOL> xml_strings = ( ( '<STR_LIT>' , STANFORD_XML ) , ) <EOL> else : <EOL> def _xml_gen ( ) : <EOL> for xml_path in argv [ <NUM_LIT:1> : ] : <EOL> with open ( xml_path , '<STR_LIT:r>' ) as xml_file : <EOL> yield ( xml_path , xml_file . read ( ) . decode ( '<STR_LIT:utf-8>' ) ) <EOL> xml_strings = _xml_gen ( ) <EOL> for xml_source , xml_string in xml_strings : <EOL> try : <EOL> print >> stderr , xml_source <EOL> _test_xml ( xml_string ) <EOL> except : <EOL> print >> stderr , '<STR_LIT>' , xml_source <EOL> raise </s>
