def parse_token_stream(stream, soft_delimiter, hard_delimiter):
 return [[sum((len(token) for token in sentence_it)) for sentence_it in split_at(block_it, soft_delimiter)] for block_it in split_at(stream, hard_delimiter)]

def parse_token_stream ( stream , soft_delimiter , hard_delimiter ) : return [ [ sum ( ( len ( token ) for token in sentence_it ) ) for sentence_it in split_at ( block_it , soft_delimiter ) ] for block_it in split_at ( stream , hard_delimiter ) ]
parses a stream of tokens and splits it into sentences and blocks for use with the l{align_texts} function .