<s> __author__ = "<STR_LIT>" <EOL> __date__ = "<STR_LIT>" <EOL> import sys <EOL> MAX_SENT_LEN = <NUM_LIT> <EOL> def cleanCorpus ( src_file , tgt_file , src_cleaned , tgt_cleaned ) : <EOL> line_cnt = <NUM_LIT:0> <EOL> filt_line_cnt = <NUM_LIT:0> <EOL> sF = open ( src_file , '<STR_LIT:r>' ) <EOL> tF = open ( tgt_file , '<STR_LIT:r>' ) <EOL> oF1 = open ( src_cleaned , '<STR_LIT:w>' ) <EOL> oF2 = open ( tgt_cleaned , '<STR_LIT:w>' ) <EOL> while True : <EOL> src_line = sF . readline ( ) <EOL> tgt_line = tF . readline ( ) <EOL> if src_line == '<STR_LIT>' and tgt_line == '<STR_LIT>' : break <EOL> line_cnt += <NUM_LIT:1> <EOL> src_line = src_line . strip ( ) <EOL> tgt_line = tgt_line . strip ( ) <EOL> src_len = len ( src_line . split ( ) ) <EOL> tgt_len = len ( tgt_line . split ( ) ) <EOL> if src_len == <NUM_LIT:0> or src_len > MAX_SENT_LEN : continue <EOL> elif tgt_len == <NUM_LIT:0> or tgt_len > MAX_SENT_LEN : continue <EOL> else : <EOL> oF1 . write ( "<STR_LIT>" % src_line ) <EOL> oF2 . write ( "<STR_LIT>" % tgt_line ) <EOL> filt_line_cnt += <NUM_LIT:1> <EOL> sF . close ( ) <EOL> tF . close ( ) <EOL> oF1 . close ( ) <EOL> oF2 . close ( ) <EOL> print "<STR_LIT>" % line_cnt <EOL> print "<STR_LIT>" % filt_line_cnt <EOL> print "<STR_LIT>" % ( line_cnt - filt_line_cnt ) <EOL> return None <EOL> def main ( ) : <EOL> global MAX_SENT_LEN <EOL> d_dir = sys . argv [ <NUM_LIT:1> ] <EOL> out_dir = sys . argv [ <NUM_LIT:2> ] <EOL> file_prefix = sys . argv [ <NUM_LIT:3> ] <EOL> src = sys . argv [ <NUM_LIT:4> ] <EOL> tgt = sys . argv [ <NUM_LIT:5> ] <EOL> if len ( sys . argv ) == <NUM_LIT:7> : <EOL> try : <EOL> MAX_SENT_LEN = int ( sys . argv [ <NUM_LIT:6> ] ) <EOL> except TypeError : <EOL> print "<STR_LIT>" <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> if not d_dir . endswith ( "<STR_LIT:/>" ) : d_dir += "<STR_LIT:/>" <EOL> if not out_dir . endswith ( "<STR_LIT:/>" ) : out_dir += "<STR_LIT:/>" <EOL> src_file = d_dir + file_prefix + "<STR_LIT:.>" + src <EOL> tgt_file = d_dir + file_prefix + "<STR_LIT:.>" + tgt <EOL> src_cleaned = out_dir + file_prefix + "<STR_LIT>" + src <EOL> tgt_cleaned = out_dir + file_prefix + "<STR_LIT>" + tgt <EOL> cleanCorpus ( src_file , tgt_file , src_cleaned , tgt_cleaned ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> main ( ) </s>
