def get_ngrams(sent_iterator, n):
 for sent in sent_iterator:
  w_boundary = ((n - 1) * [(None, '*')])
  w_boundary.extend(sent)
  w_boundary.append((None, 'STOP'))
  ngrams = (tuple(w_boundary[i:(i + n)]) for i in xrange(((len(w_boundary) - n) + 1)))
  for n_gram in ngrams:
   (yield n_gram)

def get_ngrams ( sent_iterator , n ) : for sent in sent_iterator : w_boundary = ( ( n - 1 ) * [ ( None , '*' ) ] ) w_boundary . extend ( sent ) w_boundary . append ( ( None , 'STOP' ) ) ngrams = ( tuple ( w_boundary [ i : ( i + n ) ] ) for i in xrange ( ( ( len ( w_boundary ) - n ) + 1 ) ) ) for n_gram in ngrams : ( yield n_gram )
get a generator that returns n-grams over the entire corpus .