<s> import sys , os , re , string <EOL> def debug_print ( level , debug_level , str ) : <EOL> if level == debug_level : <EOL> print str <EOL> sys . stdout . flush ( ) <EOL> SENTENCE = <NUM_LIT:0> <EOL> SENT_START = <NUM_LIT:1> <EOL> SENT_END = <NUM_LIT:2> <EOL> WORD_LIST = <NUM_LIT:3> <EOL> WORD_START_LIST = <NUM_LIT:4> <EOL> WORD_END_LIST = <NUM_LIT:5> <EOL> WORD_POS_LIST = <NUM_LIT:6> <EOL> MARKUP_LIST = <NUM_LIT:7> <EOL> wrd = "<STR_LIT>" <EOL> capwrd = "<STR_LIT>" <EOL> num = "<STR_LIT>" <EOL> abbr1 = "<STR_LIT>" <EOL> abbr2 = "<STR_LIT>" <EOL> spc = "<STR_LIT>" <EOL> terma = "<STR_LIT>" <EOL> terma = "<STR_LIT>" <EOL> quot = "<STR_LIT>" <EOL> endline = "<STR_LIT>" <EOL> punca = "<STR_LIT>" <EOL> puncb = "<STR_LIT>" <EOL> punc = "<STR_LIT:[>" + punca + puncb + "<STR_LIT>" <EOL> fstart = quot + "<STR_LIT>" + capwrd + "<STR_LIT:|>" + num + "<STR_LIT:|>" + abbr1 + "<STR_LIT:|>" + abbr2 + "<STR_LIT:)>" + quot + "<STR_LIT>" + punc + "<STR_LIT>" + spc + "<STR_LIT:)>" <EOL> fmid = quot + "<STR_LIT>" + wrd + "<STR_LIT:|>" + num + "<STR_LIT:|>" + abbr1 + "<STR_LIT:|>" + abbr2 + "<STR_LIT:|>" + punc + "<STR_LIT:)>" + quot + "<STR_LIT>" + punc + "<STR_LIT>" + spc + "<STR_LIT:)>" <EOL> fend = "<STR_LIT>" + wrd + "<STR_LIT:|>" + num + "<STR_LIT:|>" + abbr1 + "<STR_LIT:|>" + abbr2 + "<STR_LIT:)>" + quot + "<STR_LIT>" + abbr1 + "<STR_LIT:|>" + abbr2 + "<STR_LIT:|>" + terma + "<STR_LIT:)>" + quot + endline <EOL> FSENT = "<STR_LIT>" + fstart + "<STR_LIT>" + fmid + "<STR_LIT>" + fend + "<STR_LIT:)>" <EOL> FWORD = "<STR_LIT>" + wrd + "<STR_LIT:|>" + num + "<STR_LIT:|>" + abbr1 + "<STR_LIT:|>" + abbr2 + "<STR_LIT>" + terma + "<STR_LIT>" <EOL> TOKEN_WIDTH = <NUM_LIT:10> <EOL> def sent_details ( sent ) : <EOL> lst = [ ] <EOL> for idx1 in range ( <NUM_LIT:0> , len ( sent [ WORD_LIST ] ) ) : <EOL> row = [ ] <EOL> for idx2 in range ( WORD_LIST , len ( sent ) ) : <EOL> row . append ( sent [ idx2 ] [ idx1 ] ) <EOL> lst . append ( tuple ( row ) ) <EOL> return lst <EOL> def extract_sentences ( text ) : <EOL> regx = re . compile ( FSENT ) <EOL> iter = regx . finditer ( text ) <EOL> sentences = [ ] <EOL> for sentence in iter : <EOL> sentences . append ( [ sentence . group ( ) , sentence . start ( ) , sentence . end ( ) ] ) <EOL> return sentences <EOL> def extract_words ( sent ) : <EOL> regx = re . compile ( FWORD ) <EOL> iter = regx . finditer ( sent [ SENTENCE ] ) <EOL> wlist = [ ] <EOL> wslist = [ ] <EOL> welist = [ ] <EOL> for word in iter : <EOL> echar = word . group ( ) [ - <NUM_LIT:1> ] <EOL> if echar == '<STR_LIT:?>' or echar == '<STR_LIT:!>' or echar == '<STR_LIT:.>' : <EOL> wlist . append ( word . group ( ) [ : - <NUM_LIT:1> ] ) <EOL> wslist . append ( sent [ SENT_START ] + word . start ( ) ) <EOL> welist . append ( sent [ SENT_START ] + word . end ( ) - <NUM_LIT:1> ) <EOL> wlist . append ( echar ) <EOL> wslist . append ( sent [ SENT_START ] + word . end ( ) ) <EOL> welist . append ( sent [ SENT_START ] + word . end ( ) ) <EOL> else : <EOL> wlist . append ( word . group ( ) ) <EOL> wslist . append ( sent [ SENT_START ] + word . start ( ) ) <EOL> welist . append ( sent [ SENT_START ] + word . end ( ) ) <EOL> return [ wlist , wslist , welist ] <EOL> def file_readable ( file_path ) : <EOL> if os . access ( file_path , os . R_OK ) : <EOL> return True <EOL> else : <EOL> return False <EOL> def file_writable ( file_path ) : <EOL> try : <EOL> outfile = open ( file_path , "<STR_LIT:w>" ) <EOL> outfile . close ( ) <EOL> os . remove ( file_path ) <EOL> return True <EOL> except IOError : <EOL> return False <EOL> def dir_exists ( dir_path ) : <EOL> if os . access ( dir_path , os . F_OK ) : <EOL> return True <EOL> else : <EOL> return False <EOL> def translator ( frm = '<STR_LIT>' , to = '<STR_LIT>' , delete = '<STR_LIT>' , keep = None ) : <EOL> allchars = string . maketrans ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> if len ( to ) == <NUM_LIT:1> : <EOL> to = to * len ( frm ) <EOL> trans = string . maketrans ( frm , to ) <EOL> if keep is not None : <EOL> delete = allchars . translate ( allchars , keep . translate ( allchars , delete ) ) <EOL> def callable ( s ) : <EOL> return s . translate ( trans , delete ) <EOL> return callable <EOL> def phrase_cnt ( instring , phrase ) : <EOL> regex = phrase <EOL> rxc = re . compile ( regex ) <EOL> cnt = len ( rxc . findall ( instring ) ) <EOL> return cnt </s>
