<s> import re , operator <EOL> def str_find_all ( str , ch ) : <EOL> result = [ ] <EOL> index = <NUM_LIT:0> <EOL> while index >= <NUM_LIT:0> : <EOL> index = str . find ( ch , index ) <EOL> if index >= <NUM_LIT:0> : <EOL> result . append ( index ) <EOL> index += <NUM_LIT:1> <EOL> return result <EOL> word_pattern = re . compile ( "<STR_LIT>" ) <EOL> def markwords ( str , iswordfun ) : <EOL> markers = { True : "<STR_LIT:x>" , False : "<STR_LIT:o>" } <EOL> return "<STR_LIT>" . join ( [ markers [ iswordfun ( ch ) ] for ch in str ] ) <EOL> def split_words ( str , iswordfun ) : <EOL> return [ x for x in word_pattern . split ( markwords ( str , iswordfun ) ) if x != "<STR_LIT>" ] <EOL> def mark_start_segment ( str , is_segment ) : <EOL> def mark_start ( s ) : <EOL> if s [ <NUM_LIT:0> : <NUM_LIT:1> ] == "<STR_LIT:x>" : <EOL> return "<STR_LIT:s>" + s [ <NUM_LIT:1> : ] <EOL> else : <EOL> return s <EOL> return "<STR_LIT>" . join ( map ( mark_start , split_words ( str , is_segment ) ) ) <EOL> def mark_end_segment ( str , is_segment ) : <EOL> def mark_start ( s ) : <EOL> if s [ <NUM_LIT:0> : <NUM_LIT:1> ] == "<STR_LIT:x>" : <EOL> return s [ : - <NUM_LIT:1> ] + "<STR_LIT:s>" <EOL> else : <EOL> return s <EOL> return "<STR_LIT>" . join ( map ( mark_start , split_words ( str , is_segment ) ) ) <EOL> def mark_start_segment_index ( str , is_segment ) : <EOL> return str_find_all ( mark_start_segment ( str , is_segment ) , "<STR_LIT:s>" ) <EOL> def mark_end_segment_index ( str , is_segment ) : <EOL> return [ x + <NUM_LIT:1> for x in str_find_all ( mark_end_segment ( str , is_segment ) , "<STR_LIT:s>" ) ] <EOL> def is_word_token ( str ) : <EOL> return not is_non_word_token ( str ) <EOL> def is_non_word_token ( str ) : <EOL> if len ( str ) != <NUM_LIT:1> or str in "<STR_LIT>" : <EOL> return True <EOL> else : <EOL> return False <EOL> def next_start_segment ( str , is_segment ) : <EOL> str = "<STR_LIT>" . join ( str ) <EOL> result = [ ] <EOL> for start in mark_start_segment_index ( str , is_segment ) : <EOL> result [ len ( result ) : start ] = [ start for x in range ( start - len ( result ) ) ] <EOL> result [ len ( result ) : len ( str ) ] = [ len ( str ) for x in range ( len ( str ) - len ( result ) + <NUM_LIT:1> ) ] <EOL> return result <EOL> def next_end_segment ( str , is_segment ) : <EOL> str = "<STR_LIT>" . join ( str ) <EOL> result = [ ] <EOL> for start in mark_end_segment_index ( str , is_segment ) : <EOL> result [ len ( result ) : start ] = [ start for x in range ( start - len ( result ) ) ] <EOL> result [ len ( result ) : len ( str ) ] = [ len ( str ) for x in range ( len ( str ) - len ( result ) + <NUM_LIT:1> ) ] <EOL> return result <EOL> def prev_start_segment ( str , is_segment ) : <EOL> str = "<STR_LIT>" . join ( str ) <EOL> result = [ ] <EOL> prev = <NUM_LIT:0> <EOL> for start in mark_start_segment_index ( str , is_segment ) : <EOL> result [ len ( result ) : start + <NUM_LIT:1> ] = [ prev for x in range ( start - len ( result ) + <NUM_LIT:1> ) ] <EOL> prev = start <EOL> result [ len ( result ) : len ( str ) ] = [ prev for x in range ( len ( str ) - len ( result ) + <NUM_LIT:1> ) ] <EOL> return result <EOL> def prev_end_segment ( str , is_segment ) : <EOL> str = "<STR_LIT>" . join ( str ) <EOL> result = [ ] <EOL> prev = <NUM_LIT:0> <EOL> for start in mark_end_segment_index ( str , is_segment ) : <EOL> result [ len ( result ) : start + <NUM_LIT:1> ] = [ prev for x in range ( start - len ( result ) + <NUM_LIT:1> ) ] <EOL> prev = start <EOL> result [ len ( result ) : len ( str ) ] = [ len ( str ) for x in range ( len ( str ) - len ( result ) + <NUM_LIT:1> ) ] <EOL> return result </s>
