<s> """<STR_LIT>""" <EOL> from __future__ import absolute_import <EOL> from __future__ import unicode_literals <EOL> import codecs <EOL> import time <EOL> import datetime <EOL> import re <EOL> import xml . etree . ElementTree as ET <EOL> from xml . etree . ElementTree import tostring <EOL> from xml . dom import minidom <EOL> import poioapi . io . graf <EOL> import poioapi . annotationgraph <EOL> import poioapi . data <EOL> import poioapi . mapper <EOL> tier_map = { <EOL> poioapi . data . TIER_UTTERANCE : [ '<STR_LIT>' , '<STR_LIT>' ] , <EOL> poioapi . data . TIER_WORD : [ '<STR_LIT>' , '<STR_LIT:t>' ] , <EOL> poioapi . data . TIER_MORPHEME : [ '<STR_LIT>' , '<STR_LIT:m>' ] , <EOL> poioapi . data . TIER_POS : [ '<STR_LIT>' , '<STR_LIT:p>' ] , <EOL> poioapi . data . TIER_GLOSS : [ '<STR_LIT>' , '<STR_LIT:g>' ] , <EOL> poioapi . data . TIER_TRANSLATION : [ '<STR_LIT>' , '<STR_LIT:f>' ] , <EOL> poioapi . data . TIER_COMMENT : [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> } <EOL> class Parser ( poioapi . io . graf . BaseParser ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , filepath ) : <EOL> """<STR_LIT>""" <EOL> self . filepath = filepath <EOL> self . parse ( ) <EOL> def parse ( self ) : <EOL> """<STR_LIT>""" <EOL> self . nodetree = ET . parse ( self . filepath ) <EOL> self . tree = self . nodetree . getroot ( ) <EOL> self . namespace = { '<STR_LIT>' : re . findall ( r"<STR_LIT>" , self . tree . tag ) [ <NUM_LIT:0> ] } <EOL> self . _current_id = <NUM_LIT:0> <EOL> self . _elements_map = { "<STR_LIT>" : [ ] , "<STR_LIT>" : [ ] , "<STR_LIT>" : [ ] , <EOL> "<STR_LIT:description>" : [ ] , "<STR_LIT>" : [ ] , "<STR_LIT>" : [ ] , <EOL> "<STR_LIT>" : [ ] } <EOL> self . parse_element_tree ( self . tree ) <EOL> def parse_element_tree ( self , tree ) : <EOL> """<STR_LIT>""" <EOL> for element in tree : <EOL> if element . tag == "<STR_LIT:{>" + self . namespace [ '<STR_LIT>' ] + "<STR_LIT:}>" + "<STR_LIT>" : <EOL> self . _elements_map [ "<STR_LIT>" ] . append ( { "<STR_LIT:id>" : element . attrib [ "<STR_LIT:id>" ] , <EOL> "<STR_LIT>" : element . attrib } ) <EOL> self . _current_phrase_id = element . attrib [ "<STR_LIT:id>" ] <EOL> elif element . tag == "<STR_LIT:{>" + self . namespace [ '<STR_LIT>' ] + "<STR_LIT:}>" + "<STR_LIT>" : <EOL> for i , el in enumerate ( self . _elements_map [ "<STR_LIT>" ] ) : <EOL> if el [ "<STR_LIT:id>" ] == self . _current_phrase_id : <EOL> el [ "<STR_LIT:value>" ] = element . text <EOL> self . _elements_map [ "<STR_LIT>" ] [ <NUM_LIT:0> ] = el <EOL> elif element . tag == "<STR_LIT:{>" + self . namespace [ '<STR_LIT>' ] + "<STR_LIT:}>" + "<STR_LIT>" : <EOL> self . _current_word_id = self . _next_id ( ) <EOL> self . _elements_map [ "<STR_LIT>" ] . append ( { "<STR_LIT:id>" : self . _current_word_id , <EOL> "<STR_LIT>" : element . attrib , <EOL> "<STR_LIT>" : self . _current_phrase_id } ) <EOL> elif element . tag == "<STR_LIT:{>" + self . namespace [ '<STR_LIT>' ] + "<STR_LIT:}>" + "<STR_LIT>" : <EOL> self . _elements_map [ "<STR_LIT>" ] . append ( { "<STR_LIT:id>" : self . _next_id ( ) , <EOL> "<STR_LIT:value>" : element . text , <EOL> "<STR_LIT>" : self . _current_word_id } ) <EOL> elif element . tag == "<STR_LIT:{>" + self . namespace [ '<STR_LIT>' ] + "<STR_LIT:}>" + "<STR_LIT>" : <EOL> self . _current_morpheme_id = self . _next_id ( ) <EOL> self . _elements_map [ "<STR_LIT>" ] . append ( { "<STR_LIT:id>" : self . _current_morpheme_id , <EOL> "<STR_LIT>" : element . attrib , <EOL> "<STR_LIT>" : self . _current_word_id } ) <EOL> elif element . tag == "<STR_LIT:{>" + self . namespace [ '<STR_LIT>' ] + "<STR_LIT:}>" + "<STR_LIT>" : <EOL> self . _elements_map [ "<STR_LIT>" ] . append ( { "<STR_LIT:id>" : self . _next_id ( ) , <EOL> "<STR_LIT:value>" : element . text , <EOL> "<STR_LIT>" : self . _current_morpheme_id } ) <EOL> elif element . tag == "<STR_LIT:{>" + self . namespace [ '<STR_LIT>' ] + "<STR_LIT:}>" + "<STR_LIT:description>" : <EOL> self . _elements_map [ "<STR_LIT:description>" ] . append ( { "<STR_LIT:id>" : self . _next_id ( ) , <EOL> "<STR_LIT:value>" : element . text , <EOL> "<STR_LIT>" : self . _current_phrase_id } ) <EOL> elif element . tag == "<STR_LIT:{>" + self . namespace [ '<STR_LIT>' ] + "<STR_LIT:}>" + "<STR_LIT>" : <EOL> self . _elements_map [ "<STR_LIT>" ] . append ( { "<STR_LIT:id>" : self . _next_id ( ) , <EOL> "<STR_LIT:value>" : element . text , <EOL> "<STR_LIT>" : self . _current_phrase_id } ) <EOL> if len ( element . getchildren ( ) ) > <NUM_LIT:0> : <EOL> self . parse_element_tree ( element ) <EOL> def get_root_tiers ( self ) : <EOL> return [ poioapi . io . graf . Tier ( "<STR_LIT>" ) ] <EOL> def get_child_tiers_for_tier ( self , tier ) : <EOL> if tier . name == "<STR_LIT>" : <EOL> return [ poioapi . io . graf . Tier ( "<STR_LIT>" ) , <EOL> poioapi . io . graf . Tier ( "<STR_LIT>" ) , <EOL> poioapi . io . graf . Tier ( "<STR_LIT:description>" ) ] <EOL> elif tier . name == "<STR_LIT>" : <EOL> return [ poioapi . io . graf . Tier ( "<STR_LIT>" ) , <EOL> poioapi . io . graf . Tier ( "<STR_LIT>" ) ] <EOL> elif tier . name == "<STR_LIT>" : <EOL> return [ poioapi . io . graf . Tier ( "<STR_LIT>" ) ] <EOL> def get_annotations_for_tier ( self , tier , annotation_parent = None ) : <EOL> if tier . name == "<STR_LIT>" : <EOL> return [ poioapi . io . graf . Annotation ( e [ "<STR_LIT:id>" ] , e [ "<STR_LIT:value>" ] , <EOL> self . _get_features ( e [ "<STR_LIT>" ] ) ) <EOL> for e in self . _elements_map [ tier . name ] ] <EOL> elif tier . name == "<STR_LIT>" or tier . name == "<STR_LIT>" : <EOL> annotations = [ ] <EOL> for e in self . _elements_map [ tier . name ] : <EOL> if e [ "<STR_LIT>" ] == annotation_parent . id : <EOL> features = self . _get_features ( e [ "<STR_LIT>" ] ) <EOL> value = e [ "<STR_LIT>" ] [ "<STR_LIT:text>" ] <EOL> annotations . append ( poioapi . io . graf . Annotation ( e [ "<STR_LIT:id>" ] , <EOL> value , features ) ) <EOL> return annotations <EOL> else : <EOL> return [ poioapi . io . graf . Annotation ( e [ "<STR_LIT:id>" ] , e [ "<STR_LIT:value>" ] ) <EOL> for e in self . _elements_map [ tier . name ] <EOL> if e [ "<STR_LIT>" ] == annotation_parent . id ] <EOL> def get_primary_data ( self ) : <EOL> """<STR_LIT>""" <EOL> primary_data = poioapi . io . graf . PrimaryData ( ) <EOL> primary_data . type = poioapi . io . graf . NONE <EOL> primary_data . filename = "<STR_LIT>" <EOL> return primary_data <EOL> def _get_features ( self , attributes ) : <EOL> """<STR_LIT>""" <EOL> features = { } <EOL> for key , value in attributes . items ( ) : <EOL> if key != "<STR_LIT:id>" : <EOL> if key != "<STR_LIT:text>" : <EOL> features [ key ] = value <EOL> return features <EOL> def _next_id ( self ) : <EOL> current_id = str ( int ( self . _current_id ) + <NUM_LIT:1> ) <EOL> self . _current_id = current_id <EOL> return current_id <EOL> def region_for_annotation ( self , annotation ) : <EOL> pass <EOL> def tier_has_regions ( self , tier ) : <EOL> pass <EOL> class Writer ( poioapi . io . graf . BaseWriter ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . _current_text_id = <NUM_LIT:0> <EOL> self . _current_phrase_id = <NUM_LIT:0> <EOL> self . _additional_maps_file = '<STR_LIT>' <EOL> self . _root = None <EOL> self . _text = None <EOL> self . _phrase_element = None <EOL> self . _word_element = None <EOL> self . _pos_element = None <EOL> self . _morpheme_element = None <EOL> self . _annotation_mapper = None <EOL> self . _elan_begin_nodes = None <EOL> self . _elan_end_nodes = None <EOL> self . _elan_participant_nodes = None <EOL> self . _body = None <EOL> def _init_root_node ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _root = ET . Element ( "<STR_LIT>" , { "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" } ) <EOL> def _create_text_node ( self , language = '<STR_LIT>' , original_title = '<STR_LIT>' , <EOL> translation_title = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> attribs = { '<STR_LIT:id>' : self . _next_text_id ( ) , '<STR_LIT>' : language } <EOL> self . _text = ET . SubElement ( self . _root , "<STR_LIT:text>" , attribs ) <EOL> ET . SubElement ( self . _text , "<STR_LIT:title>" ) . text = original_title <EOL> ET . SubElement ( self . _text , "<STR_LIT>" ) . text = translation_title <EOL> def _create_metadata_elements ( self , metadata , set_name ) : <EOL> """<STR_LIT>""" <EOL> meta_elem = ET . SubElement ( self . _text , '<STR_LIT>' , <EOL> { '<STR_LIT>' : set_name } ) <EOL> for meta in metadata : <EOL> ET . SubElement ( meta_elem , '<STR_LIT>' , <EOL> { '<STR_LIT:name>' : meta } ) . text = metadata [ meta ] <EOL> def write ( self , outputfile , converter , pretty_print = False , <EOL> extra_tag_map = '<STR_LIT>' , language = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> self . _additional_maps_file = extra_tag_map <EOL> self . _annotation_mapper = poioapi . mapper . AnnotationMapper ( <EOL> converter . source_type , poioapi . data . TYPECRAFT ) <EOL> self . _annotation_mapper . load_mappings ( extra_tag_map ) <EOL> self . _init_root_node ( ) <EOL> phrase_nodes = [ ] <EOL> if converter . source_type == poioapi . data . ODIN : <EOL> text_nodes = [ ] <EOL> text_nodes . extend ( converter . nodes_for_tier ( '<STR_LIT:source>' ) ) <EOL> language = converter . meta_information [ '<STR_LIT>' ] <EOL> for text in text_nodes : <EOL> phrase_nodes = [ ] <EOL> annot = text . annotations . get_first ( ) <EOL> text_md = converter . meta_information [ annot . id ] <EOL> self . _create_text_node ( language ) <EOL> self . _create_metadata_elements ( text_md , '<STR_LIT>' ) <EOL> for marker in converter . tier_mapper . tier_labels ( <EOL> poioapi . data . TIER_UTTERANCE ) : <EOL> phrase_nodes . extend ( converter . nodes_for_tier ( marker , text ) ) <EOL> ET . SubElement ( self . _text , '<STR_LIT:body>' ) . text = '<STR_LIT:U+0020>' . join ( <EOL> [ converter . annotation_value_for_node ( phrase ) for phrase in <EOL> phrase_nodes ] ) <EOL> self . _write_phrases ( phrase_nodes , converter ) <EOL> elif converter . source_type == poioapi . data . TOOLBOX : <EOL> ref_nodes = converter . nodes_for_tier ( '<STR_LIT>' ) <EOL> self . _create_text_node ( language = language ) <EOL> self . _body = ET . SubElement ( self . _text , '<STR_LIT:body>' ) <EOL> self . _body . text = '<STR_LIT>' <EOL> for ref in ref_nodes : <EOL> phrase_nodes = [ ] <EOL> for marker in converter . tier_mapper . tier_labels ( <EOL> poioapi . data . TIER_UTTERANCE ) : <EOL> phrase_nodes . extend ( converter . nodes_for_tier ( marker , ref ) ) <EOL> self . _elan_begin_nodes = converter . nodes_for_tier ( '<STR_LIT>' , ref ) <EOL> self . _elan_end_nodes = converter . nodes_for_tier ( '<STR_LIT>' , ref ) <EOL> self . _elan_participant_nodes = converter . nodes_for_tier ( '<STR_LIT>' , ref ) <EOL> self . _body . text += '<STR_LIT:U+0020>' . join ( <EOL> [ converter . annotation_value_for_node ( phrase ) for phrase in <EOL> phrase_nodes ] ) <EOL> self . _write_phrases ( phrase_nodes , converter ) <EOL> else : <EOL> self . _create_text_node ( language = language ) <EOL> for marker in converter . tier_mapper . tier_labels ( <EOL> poioapi . data . TIER_UTTERANCE ) : <EOL> phrase_nodes . extend ( converter . nodes_for_tier ( marker , None ) ) <EOL> ET . SubElement ( self . _text , '<STR_LIT:body>' ) . text = '<STR_LIT:U+0020>' . join ( <EOL> [ converter . annotation_value_for_node ( phrase ) for phrase in <EOL> phrase_nodes ] ) <EOL> self . _write_phrases ( phrase_nodes , converter ) <EOL> self . write_xml ( self . _root , outputfile ) <EOL> def _write_phrases ( self , phrase_nodes , converter ) : <EOL> for phrase in phrase_nodes : <EOL> annotation = converter . annotation_value_for_node ( phrase ) <EOL> if annotation == '<STR_LIT>' or re . match ( r'<STR_LIT>' , annotation ) : <EOL> continue <EOL> self . _phrase_element = ET . SubElement ( self . _text , '<STR_LIT>' , <EOL> { '<STR_LIT:id>' : self . _next_phrase_id ( ) , <EOL> '<STR_LIT>' : '<STR_LIT>' } ) <EOL> if converter . source_type == poioapi . data . TOOLBOX : <EOL> self . _write_elan_attributes ( converter ) <EOL> ET . SubElement ( self . _phrase_element , '<STR_LIT>' ) . text = annotation <EOL> self . _write_translations ( converter , phrase ) <EOL> ET . SubElement ( self . _phrase_element , '<STR_LIT:description>' ) <EOL> ET . SubElement ( self . _phrase_element , '<STR_LIT>' , <EOL> { '<STR_LIT:id>' : '<STR_LIT:1>' , '<STR_LIT>' : '<STR_LIT>' } ) <EOL> self . _write_words ( converter , phrase ) <EOL> def _write_words ( self , converter , phrase ) : <EOL> """<STR_LIT>""" <EOL> word_tier_markers = converter . tier_mapper . tier_labels ( <EOL> poioapi . data . TIER_WORD ) <EOL> word_nodes = [ ] <EOL> for marker in word_tier_markers : <EOL> word_nodes . extend ( converter . nodes_for_tier ( marker , phrase ) ) <EOL> if len ( word_nodes ) == <NUM_LIT:0> : <EOL> self . _word_element = ET . SubElement ( self . _phrase_element , '<STR_LIT>' , <EOL> { '<STR_LIT:text>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT:false>' } ) <EOL> ET . SubElement ( self . _word_element , '<STR_LIT>' ) <EOL> ET . SubElement ( self . _word_element , '<STR_LIT>' , { '<STR_LIT:text>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' } ) <EOL> for word in word_nodes : <EOL> annotation = converter . annotation_value_for_node ( word ) <EOL> self . _word_element = ET . SubElement ( self . _phrase_element , '<STR_LIT>' , <EOL> { '<STR_LIT:text>' : annotation , <EOL> '<STR_LIT>' : '<STR_LIT:false>' } ) <EOL> self . _pos_element = ET . SubElement ( self . _word_element , '<STR_LIT>' ) <EOL> self . _write_pos ( converter , word ) <EOL> if self . _pos_element . text == '<STR_LIT>' : <EOL> check_pos_in_morphemes = True <EOL> else : <EOL> check_pos_in_morphemes = False <EOL> self . _write_morphemes ( converter , word , check_pos_in_morphemes ) <EOL> def _write_elan_attributes ( self , converter ) : <EOL> """<STR_LIT>""" <EOL> if len ( self . _elan_begin_nodes ) == <NUM_LIT:0> and len ( self . _elan_end_nodes ) == <NUM_LIT:0> and len ( self . _elan_participant_nodes ) == <NUM_LIT:0> : <EOL> return <EOL> if len ( self . _elan_begin_nodes ) > <NUM_LIT:0> : <EOL> begin_node = self . _elan_begin_nodes [ <NUM_LIT:0> ] <EOL> begin_annotation = converter . annotation_value_for_node ( begin_node ) <EOL> else : <EOL> begin_annotation = '<STR_LIT:0>' <EOL> if len ( self . _elan_end_nodes ) > <NUM_LIT:0> : <EOL> end_node = self . _elan_end_nodes [ <NUM_LIT:0> ] <EOL> end_annotation = converter . annotation_value_for_node ( end_node ) <EOL> else : <EOL> end_annotation = '<STR_LIT:0>' <EOL> if len ( self . _elan_participant_nodes ) > <NUM_LIT:0> : <EOL> participant_node = self . _elan_participant_nodes [ <NUM_LIT:0> ] <EOL> participant_annotation = converter . annotation_value_for_node ( participant_node ) <EOL> else : <EOL> participant_annotation = '<STR_LIT>' <EOL> begin_time = self . _string_to_milliseconds ( begin_annotation ) <EOL> end_time = self . _string_to_milliseconds ( end_annotation ) <EOL> duration = end_time - begin_time <EOL> self . _phrase_element . set ( '<STR_LIT>' , str ( begin_time ) ) <EOL> self . _phrase_element . set ( '<STR_LIT>' , str ( duration ) ) <EOL> self . _phrase_element . set ( '<STR_LIT>' , participant_annotation ) <EOL> def _write_pos ( self , converter , parent_node ) : <EOL> """<STR_LIT>""" <EOL> pos_annotations = [ ] <EOL> for marker in converter . tier_mapper . tier_labels ( poioapi . data . TIER_POS ) : <EOL> pos_annotations . extend ( converter . annotations_for_tier ( marker , <EOL> parent_node ) ) <EOL> if len ( pos_annotations ) == <NUM_LIT:1> : <EOL> annotation = converter . annotation_value_for_annotation ( <EOL> pos_annotations [ <NUM_LIT:0> ] ) <EOL> if not any ( ( c in annotation ) for c in "<STR_LIT>" ) : <EOL> mapping = self . _annotation_mapper . validate_tag ( <EOL> poioapi . data . TIER_POS , annotation ) <EOL> if mapping is None : <EOL> self . _annotation_mapper . add_to_missing ( <EOL> poioapi . data . TIER_POS , annotation ) <EOL> else : <EOL> self . _pos_element . text = mapping <EOL> else : <EOL> self . _pos_element . text = '<STR_LIT>' <EOL> def _write_morphemes ( self , converter , word , check_for_pos = False ) : <EOL> """<STR_LIT>""" <EOL> morpheme_nodes = [ ] <EOL> for marker in converter . tier_mapper . tier_labels ( <EOL> poioapi . data . TIER_MORPHEME ) : <EOL> morpheme_nodes . extend ( converter . nodes_for_tier ( marker , word ) ) <EOL> for morpheme in morpheme_nodes : <EOL> annotation = converter . annotation_value_for_node ( morpheme ) <EOL> if check_for_pos is True : <EOL> self . _write_pos ( converter , morpheme ) <EOL> self . _morpheme_element = ET . SubElement ( self . _word_element , <EOL> '<STR_LIT>' , <EOL> { '<STR_LIT:text>' : annotation , <EOL> '<STR_LIT>' : annotation } ) <EOL> self . _write_gloss ( converter , morpheme ) <EOL> if self . _word_element . find ( '<STR_LIT>' ) is None : <EOL> ET . SubElement ( self . _word_element , '<STR_LIT>' ) <EOL> def _write_gloss ( self , converter , morpheme ) : <EOL> """<STR_LIT>""" <EOL> gloss_annotations = [ ] <EOL> for marker in converter . tier_mapper . tier_labels ( poioapi . data . TIER_GLOSS ) : <EOL> gloss_annotations . extend ( converter . annotations_for_tier ( marker , <EOL> morpheme ) ) <EOL> for gloss in gloss_annotations : <EOL> annotation = converter . annotation_value_for_annotation ( gloss ) <EOL> splitter = '<STR_LIT:.>' <EOL> if "<STR_LIT:?>" in annotation and annotation != "<STR_LIT>" : <EOL> return None <EOL> if "<STR_LIT::>" in annotation : <EOL> splitter = "<STR_LIT::>" <EOL> for token in annotation . split ( splitter ) : <EOL> gloss_list = self . _annotation_mapper . validate_tag ( <EOL> poioapi . data . TIER_GLOSS , token ) <EOL> if gloss_list is not None : <EOL> if isinstance ( gloss_list , tuple ) and len ( gloss_list ) == <NUM_LIT:2> : <EOL> if gloss_list [ <NUM_LIT:0> ] in converter . tier_mapper . tier_labels ( <EOL> poioapi . data . TIER_POS ) : <EOL> self . _pos_element . text = gloss_list [ <NUM_LIT:1> ] <EOL> else : <EOL> self . _split_destination_tags ( gloss_list ) <EOL> else : <EOL> if not annotation . isupper ( ) : <EOL> if "<STR_LIT::>" in annotation : <EOL> annotation = annotation . split ( "<STR_LIT::>" ) [ <NUM_LIT:0> ] <EOL> elif "<STR_LIT:->" in annotation : <EOL> annotation = annotation . split ( "<STR_LIT:->" ) [ <NUM_LIT:0> ] <EOL> elif "<STR_LIT:/>" in annotation : <EOL> annotation = annotation . split ( "<STR_LIT:/>" ) [ <NUM_LIT:0> ] <EOL> self . _morpheme_element . set ( "<STR_LIT>" , annotation ) <EOL> else : <EOL> self . _annotation_mapper . add_to_missing ( <EOL> poioapi . data . TIER_GLOSS , token ) <EOL> def _split_destination_tags ( self , tags ) : <EOL> tokens = re . split ( poioapi . mapper . tag_separators , tags ) <EOL> for token in tokens : <EOL> if token != '<STR_LIT>' : <EOL> ET . SubElement ( self . _morpheme_element , '<STR_LIT>' ) . text = token <EOL> def _write_translations ( self , converter , phrase ) : <EOL> """<STR_LIT>""" <EOL> translation_annotations = [ ] <EOL> for marker in converter . tier_mapper . tier_labels ( poioapi . data . TIER_TRANSLATION ) : <EOL> translation_annotations . extend ( converter . annotations_for_tier ( marker , phrase ) ) <EOL> if len ( translation_annotations ) == <NUM_LIT:1> : <EOL> ET . SubElement ( self . _phrase_element , '<STR_LIT>' ) . text = converter . annotation_value_for_annotation ( translation_annotations [ <NUM_LIT:0> ] ) <EOL> else : <EOL> ET . SubElement ( self . _phrase_element , '<STR_LIT>' ) <EOL> def write_xml ( self , root , outputfile , pretty_print = True ) : <EOL> """<STR_LIT>""" <EOL> if pretty_print : <EOL> doc = minidom . parseString ( tostring ( root ) ) <EOL> text_re = re . compile ( r'<STR_LIT>' , re . DOTALL ) <EOL> pretty_xml = text_re . sub ( r'<STR_LIT>' , doc . toprettyxml ( indent = '<STR_LIT:U+0020>' ) ) <EOL> file = codecs . open ( outputfile , '<STR_LIT:wb>' , encoding = '<STR_LIT:utf-8>' ) <EOL> file . write ( pretty_xml ) <EOL> file . close ( ) <EOL> else : <EOL> tree = ET . ElementTree ( root ) <EOL> tree . write ( outputfile ) <EOL> def _string_to_milliseconds ( self , value ) : <EOL> """<STR_LIT>""" <EOL> if value == '<STR_LIT:0>' : <EOL> return <NUM_LIT:0> <EOL> time_start = value . split ( "<STR_LIT:.>" ) <EOL> microseconds = int ( time_start [ <NUM_LIT:1> ] ) <EOL> try : <EOL> x = time . strptime ( time_start [ <NUM_LIT:0> ] , '<STR_LIT>' ) <EOL> offset = int ( datetime . timedelta ( hours = x . tm_hour , minutes = x . tm_min , <EOL> seconds = x . tm_sec ) . seconds * <NUM_LIT:1000> ) + microseconds <EOL> except ValueError : <EOL> offset = ( int ( time_start [ <NUM_LIT:0> ] ) * <NUM_LIT:1000> ) + microseconds <EOL> return offset <EOL> def _next_text_id ( self ) : <EOL> """<STR_LIT>""" <EOL> current_id = str ( int ( self . _current_text_id ) + <NUM_LIT:1> ) <EOL> self . _current_text_id = current_id <EOL> return str ( current_id ) <EOL> def _next_phrase_id ( self ) : <EOL> """<STR_LIT>""" <EOL> current_id = str ( int ( self . _current_phrase_id ) + <NUM_LIT:1> ) <EOL> self . _current_phrase_id = current_id <EOL> return str ( current_id ) <EOL> def missing_tags ( self , outputfile , annotation_graph , additional_map_path ) : <EOL> """<STR_LIT>""" <EOL> pos_nodes = [ ] <EOL> for marker in annotation_graph . tier_mapper . tier_labels ( poioapi . data . TIER_POS ) : <EOL> pos_nodes . extend ( annotation_graph . nodes_for_tier ( marker ) ) <EOL> gloss_nodes = [ ] <EOL> for marker in annotation_graph . tier_mapper . tier_labels ( poioapi . data . TIER_GLOSS ) : <EOL> gloss_nodes . extend ( annotation_graph . nodes_for_tier ( marker ) ) <EOL> if self . _annotation_mapper is None : <EOL> self . _annotation_mapper = poioapi . mapper . AnnotationMapper ( annotation_graph . source_type , poioapi . data . TYPECRAFT ) <EOL> self . _annotation_mapper . load_mappings ( additional_map_path ) <EOL> for node in pos_nodes : <EOL> annotation = annotation_graph . annotation_value_for_node ( node ) <EOL> if not any ( ( c in annotation ) for c in "<STR_LIT>" ) : <EOL> mapping = self . _annotation_mapper . validate_tag ( poioapi . data . TIER_POS , annotation ) <EOL> if mapping is None : <EOL> self . _annotation_mapper . add_to_missing ( poioapi . data . TIER_POS , annotation ) <EOL> for node in gloss_nodes : <EOL> annotation = annotation_graph . annotation_value_for_node ( node ) <EOL> splitter = '<STR_LIT:.>' <EOL> if "<STR_LIT:?>" in annotation and annotation != "<STR_LIT>" : <EOL> continue <EOL> if "<STR_LIT::>" in annotation : <EOL> splitter = "<STR_LIT::>" <EOL> for token in annotation . split ( splitter ) : <EOL> gloss_list = self . _annotation_mapper . validate_tag ( poioapi . data . TIER_GLOSS , token . strip ( '<STR_LIT:->' ) ) <EOL> if gloss_list is None and ( token . isupper ( ) or token . strip ( '<STR_LIT:->' ) . isdigit ( ) ) : <EOL> self . _annotation_mapper . add_to_missing ( poioapi . data . TIER_GLOSS , token . strip ( '<STR_LIT:->' ) ) <EOL> self . _annotation_mapper . export_missing_tags ( outputfile ) </s>
