def segment_text(text=os.path.join(DATA_PATH, 'goodreads-omniscient-books.txt'),
                 start=None, stop=r'^Rate\ this', ignore=r'^[\d]'):
    """ Split text into segments (sections, paragraphs) using regular expressions to trigger breaks.start
    """
    start = start if hasattr(start, 'match') else re.compile(start) if start else None
    stop = stop if hasattr(stop, 'match') else re.compile(stop) if stop else None
    ignore = ignore if hasattr(ignore, 'match') else re.compile(ignore) if ignore else None

    segments = []
    segment = []
    with open(text) as fin:
        for line in fin:
            if start is not None and start.match(line):
                segments += [segment] if len(segment) else []
                segment = [line]
            elif stop is not None and stop.match(line):
                segments += [segment]
                segment = []
            elif ignore is not None and ignore.match(line):
                continue
            else:
                segment += [segment]
def segment_text ( text = os . path . join ( DATA_PATH , 'goodreads-omniscient-books.txt' ) , start = None , stop = r'^Rate\ this' , ignore = r'^[\d]' ) : start = start if hasattr ( start , 'match' ) else re . compile ( start ) if start else None stop = stop if hasattr ( stop , 'match' ) else re . compile ( stop ) if stop else None ignore = ignore if hasattr ( ignore , 'match' ) else re . compile ( ignore ) if ignore else None segments = [ ] segment = [ ] with open ( text ) as fin : for line in fin : if start is not None and start . match ( line ) : segments += [ segment ] if len ( segment ) else [ ] segment = [ line ] elif stop is not None and stop . match ( line ) : segments += [ segment ] segment = [ ] elif ignore is not None and ignore . match ( line ) : continue else : segment += [ segment ]
Split text into segments (sections, paragraphs) using regular expressions to trigger breaks.start