<s> import click <EOL> import nltk <EOL> from textkit . utils import output , read_tokens <EOL> @ click . command ( '<STR_LIT>' ) <EOL> @ click . argument ( '<STR_LIT>' , type = click . File ( '<STR_LIT:r>' ) , default = click . open_file ( '<STR_LIT:->' ) ) <EOL> @ click . option ( '<STR_LIT>' , default = '<STR_LIT:U+0020>' , <EOL> help = '<STR_LIT>' , <EOL> show_default = True ) <EOL> @ click . option ( '<STR_LIT>' , '<STR_LIT>' , default = <NUM_LIT:2> , <EOL> help = '<STR_LIT>' , <EOL> show_default = True ) <EOL> def words2ngrams ( sep , length , tokens ) : <EOL> '''<STR_LIT>''' <EOL> content = read_tokens ( tokens ) <EOL> ngrams = list ( nltk . ngrams ( content , length ) ) <EOL> [ output ( sep . join ( ngram ) ) for ngram in ngrams ] </s>
