<s> import sys <EOL> import re <EOL> class Parser : <EOL> SENTENCE_START_SYMBOL = '<STR_LIT>' <EOL> SENTENCE_END_SYMBOL = '<STR_LIT:$>' <EOL> def __init__ ( self , name , db , sentence_split_char = '<STR_LIT:\n>' , word_split_char = '<STR_LIT>' ) : <EOL> self . name = name <EOL> self . db = db <EOL> self . sentence_split_char = sentence_split_char <EOL> self . word_split_char = word_split_char <EOL> self . whitespace_regex = re . compile ( '<STR_LIT>' ) <EOL> def parse ( self , txt ) : <EOL> depth = self . db . get_depth ( ) <EOL> sentences = txt . split ( self . sentence_split_char ) <EOL> i = <NUM_LIT:0> <EOL> for sentence in sentences : <EOL> sentence = self . whitespace_regex . sub ( "<STR_LIT:U+0020>" , sentence ) . strip ( ) <EOL> list_of_words = None <EOL> if self . word_split_char : <EOL> list_of_words = sentence . split ( self . word_split_char ) <EOL> else : <EOL> list_of_words = list ( sentence . lower ( ) ) <EOL> words = [ Parser . SENTENCE_START_SYMBOL ] * ( depth - <NUM_LIT:1> ) + list_of_words + [ Parser . SENTENCE_END_SYMBOL ] * ( depth - <NUM_LIT:1> ) <EOL> for n in range ( <NUM_LIT:0> , len ( words ) - depth + <NUM_LIT:1> ) : <EOL> self . db . add_word ( words [ n : n + depth ] ) <EOL> self . db . commit ( ) <EOL> i += <NUM_LIT:1> <EOL> if i % <NUM_LIT:1000> == <NUM_LIT:0> : <EOL> print i <EOL> sys . stdout . flush ( ) </s>
