<s> from __future__ import print_function <EOL> from subprocess import call <EOL> import time <EOL> import re <EOL> import os <EOL> import codecs <EOL> from marmot . util . alignments import train_alignments <EOL> from marmot . util . force_align import Aligner <EOL> from marmot . representations . representation_generator import RepresentationGenerator <EOL> from marmot . experiment . import_utils import mk_tmp_dir <EOL> class SegmentationRepresentationGenerator ( RepresentationGenerator ) : <EOL> def __init__ ( self , align_model = None , src_file = None , tg_file = None , lex_prefix = None , tmp_dir = None , moses_dir = None , moses_config = None , workers = <NUM_LIT:1> ) : <EOL> self . tmp_dir = mk_tmp_dir ( tmp_dir ) <EOL> self . time_stamp = str ( time . time ( ) ) <EOL> self . moses_dir = moses_dir <EOL> self . moses_config = moses_config <EOL> self . workers = workers <EOL> self . lex_prob = lex_prefix <EOL> if align_model is None : <EOL> if src_file is not None and tg_file is not None : <EOL> self . align_model = train_alignments ( src_file , tg_file , tmp_dir , align_model = align_model ) <EOL> else : <EOL> print ( "<STR_LIT>" ) <EOL> return <EOL> else : <EOL> self . align_model = align_model <EOL> def write_command_file ( self , data_obj , alignments_file ) : <EOL> command_name = os . path . join ( self . tmp_dir , '<STR_LIT>' + self . time_stamp + '<STR_LIT>' ) <EOL> command = open ( command_name , '<STR_LIT:w>' ) <EOL> command . write ( "<STR_LIT>" % self . tmp_dir ) <EOL> command . write ( '<STR_LIT>' % ( self . moses_dir , self . moses_dir , data_obj [ '<STR_LIT>' ] , data_obj [ '<STR_LIT>' ] , alignments_file , self . tmp_dir , self . time_stamp ) ) <EOL> command . write ( '<STR_LIT>' % ( self . moses_dir , self . time_stamp , self . lex_prob , self . tmp_dir , self . time_stamp ) ) <EOL> command . write ( '<STR_LIT>' % ( self . moses_dir , self . time_stamp , self . lex_prob , self . tmp_dir , self . time_stamp ) ) <EOL> command . write ( '<STR_LIT>' % ( self . tmp_dir , self . time_stamp , self . tmp_dir , self . time_stamp ) ) <EOL> command . write ( '<STR_LIT>' % ( self . tmp_dir , self . time_stamp , self . tmp_dir , self . time_stamp ) ) <EOL> command . write ( '<STR_LIT>' % ( self . moses_dir , self . tmp_dir , self . time_stamp , self . tmp_dir , self . time_stamp , self . tmp_dir , self . time_stamp , self . tmp_dir , self . time_stamp ) ) <EOL> command . write ( '<STR_LIT>' % self . tmp_dir ) <EOL> command . write ( '<STR_LIT>' % ( self . tmp_dir , self . time_stamp , self . tmp_dir , self . moses_dir , self . tmp_dir , self . time_stamp ) ) <EOL> command . write ( '<STR_LIT>' % ( self . tmp_dir , self . time_stamp ) ) <EOL> command . write ( '<STR_LIT>' ) <EOL> command . close ( ) <EOL> phrase_table = os . path . join ( self . tmp_dir , '<STR_LIT>' . format ( self . time_stamp ) ) <EOL> return command_name , phrase_table <EOL> def write_moses_config ( self , phrase_table , target_file ) : <EOL> new_config_name = os . path . join ( self . tmp_dir , '<STR_LIT>' + self . time_stamp + '<STR_LIT>' ) <EOL> new_config = open ( new_config_name , '<STR_LIT:w>' ) <EOL> constrained = False <EOL> for line in open ( self . moses_config ) : <EOL> if line . startswith ( "<STR_LIT>" ) : <EOL> good_line = [ s for s in line . strip ( ) . split ( ) if not s . startswith ( '<STR_LIT:path>' ) ] <EOL> new_config . write ( "<STR_LIT>" % ( '<STR_LIT:U+0020>' . join ( good_line ) , phrase_table ) ) <EOL> elif line . startswith ( "<STR_LIT>" ) : <EOL> new_config . write ( "<STR_LIT>" % target_file ) <EOL> elif line . startswith ( "<STR_LIT>" ) and not constrained : <EOL> new_config . write ( "<STR_LIT>" % target_file ) <EOL> new_config . write ( "<STR_LIT>" ) <EOL> else : <EOL> new_config . write ( line ) <EOL> new_config . close ( ) <EOL> return new_config_name <EOL> def get_alignments ( self , src , tg , align_model , align_file ) : <EOL> alignments = [ [ [ ] for j in range ( len ( tg [ i ] ) ) ] for i in range ( len ( tg ) ) ] <EOL> align_stream = open ( align_file , '<STR_LIT:w>' ) <EOL> aligner = Aligner ( align_model + '<STR_LIT>' , align_model + '<STR_LIT>' , align_model + '<STR_LIT>' , align_model + '<STR_LIT>' ) <EOL> for idx , ( src_list , tg_list ) in enumerate ( zip ( src , tg ) ) : <EOL> align_string = aligner . align ( '<STR_LIT:U+0020>' . join ( src_list ) + '<STR_LIT>' + '<STR_LIT:U+0020>' . join ( tg_list ) ) <EOL> align_stream . write ( '<STR_LIT>' % align_string ) <EOL> pairs = align_string . split ( ) <EOL> for p_str in pairs : <EOL> p = p_str . split ( '<STR_LIT:->' ) <EOL> alignments [ idx ] [ int ( p [ <NUM_LIT:1> ] ) ] . append ( int ( p [ <NUM_LIT:0> ] ) ) <EOL> aligner . close ( ) <EOL> align_stream . close ( ) <EOL> return alignments <EOL> def get_segments ( self , data_obj , segmentation_file ) : <EOL> seg_regexp = re . compile ( "<STR_LIT>" ) <EOL> source_segments = [ ] <EOL> target_segments = [ ] <EOL> with codecs . open ( segmentation_file , encoding = '<STR_LIT:utf-8>' ) as segmentation : <EOL> for idx , line in enumerate ( segmentation ) : <EOL> if line == "<STR_LIT:\n>" : <EOL> source_segments . append ( [ ] ) <EOL> target_segments . append ( [ ( i , i + <NUM_LIT:1> ) for i in range ( len ( data_obj [ '<STR_LIT:target>' ] [ idx ] ) ) ] ) <EOL> continue <EOL> source_seg_strings = seg_regexp . findall ( line ) <EOL> source_seg_list = [ ] <EOL> for a_seg in source_seg_strings : <EOL> a_pair = a_seg . strip ( '<STR_LIT:|>' ) . split ( '<STR_LIT:->' ) <EOL> source_seg_list . append ( ( int ( a_pair [ <NUM_LIT:0> ] ) , int ( a_pair [ <NUM_LIT:1> ] ) + <NUM_LIT:1> ) ) <EOL> target_seg_strings = [ ll for ll in [ l . strip ( ) for l in seg_regexp . split ( line ) ] if ll != '<STR_LIT>' ] <EOL> target_seg_list = [ ] <EOL> cur_pos = <NUM_LIT:0> <EOL> for a_seg in target_seg_strings : <EOL> seg_words = a_seg . split ( ) <EOL> for ( s_w , t_w ) in zip ( seg_words , data_obj [ '<STR_LIT:target>' ] [ idx ] [ cur_pos : len ( seg_words ) ] ) : <EOL> assert ( s_w . lower ( ) == t_w . lower ( ) ) , "<STR_LIT>" . format ( idx , s_w . lower ( ) , t_w . lower ( ) ) <EOL> target_seg_list . append ( ( cur_pos , cur_pos + len ( seg_words ) ) ) <EOL> cur_pos += len ( seg_words ) <EOL> assert ( len ( source_seg_list ) == len ( target_seg_list ) ) , "<STR_LIT>" . format ( len ( source_seg_list ) , len ( target_seg_list ) ) <EOL> source_segments . append ( source_seg_list ) <EOL> target_segments . append ( target_seg_list ) <EOL> return target_segments , source_segments <EOL> def generate ( self , data_obj ) : <EOL> data_time_stamp = str ( time . time ( ) ) <EOL> if '<STR_LIT:target>' not in data_obj or '<STR_LIT:source>' not in data_obj : <EOL> print ( "<STR_LIT>" ) <EOL> assert ( len ( data_obj [ '<STR_LIT:target>' ] ) == len ( data_obj [ '<STR_LIT:source>' ] ) ) <EOL> alignments_file = os . path . join ( self . tmp_dir , '<STR_LIT>' + self . time_stamp ) <EOL> all_alignments = self . get_alignments ( data_obj [ '<STR_LIT:source>' ] , data_obj [ '<STR_LIT:target>' ] , self . align_model , alignments_file ) <EOL> data_obj [ '<STR_LIT>' ] = all_alignments <EOL> command , phrase_table = self . write_command_file ( data_obj , alignments_file ) <EOL> call ( [ '<STR_LIT>' , command ] ) <EOL> moses_config = self . write_moses_config ( phrase_table , data_obj [ '<STR_LIT>' ] ) <EOL> moses_seg_file_name = os . path . join ( self . tmp_dir , '<STR_LIT>' + self . time_stamp + '<STR_LIT:.>' + data_time_stamp ) <EOL> moses_seg_file = open ( moses_seg_file_name , '<STR_LIT:w>' ) <EOL> src = open ( data_obj [ '<STR_LIT>' ] ) <EOL> call ( [ os . path . join ( self . moses_dir , '<STR_LIT>' ) , '<STR_LIT>' , moses_config , '<STR_LIT>' , '<STR_LIT:0>' , '<STR_LIT>' ] , stdin = src , stdout = moses_seg_file ) <EOL> moses_seg_file . close ( ) <EOL> src . close ( ) <EOL> data_obj [ '<STR_LIT>' ] , data_obj [ '<STR_LIT>' ] = self . get_segments ( data_obj , moses_seg_file_name ) <EOL> return data_obj </s>
