def build_words_dataset(words=[], vocabulary_size=50000, printable=True, unk_key='UNK'):
 import collections
 count = [[unk_key, (-1)]]
 count.extend(collections.Counter(words).most_common((vocabulary_size - 1)))
 dictionary = dict()
 for (word, _) in count:
  dictionary[word] = len(dictionary)
 data = list()
 unk_count = 0
 for word in words:
  if (word in dictionary):
   index = dictionary[word]
  else:
   index = 0
   unk_count += 1
  data.append(index)
 count[0][1] = unk_count
 reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
 if printable:
  print ('Real vocabulary size    %d' % len(collections.Counter(words).keys()))
  print 'Limited vocabulary size {}'.format(vocabulary_size)
 assert (len(collections.Counter(words).keys()) >= vocabulary_size), 'the limited vocabulary_size must be less than or equal to the read vocabulary_size'
 return (data, count, dictionary, reverse_dictionary)

def build_words_dataset ( words = [ ] , vocabulary_size = 50000 , printable = True , unk_key = 'UNK' ) : import collections count = [ [ unk_key , ( - 1 ) ] ] count . extend ( collections . Counter ( words ) . most_common ( ( vocabulary_size - 1 ) ) ) dictionary = dict ( ) for ( word , _ ) in count : dictionary [ word ] = len ( dictionary ) data = list ( ) unk_count = 0 for word in words : if ( word in dictionary ) : index = dictionary [ word ] else : index = 0 unk_count += 1 data . append ( index ) count [ 0 ] [ 1 ] = unk_count reverse_dictionary = dict ( zip ( dictionary . values ( ) , dictionary . keys ( ) ) ) if printable : print ( 'Real vocabulary size    %d' % len ( collections . Counter ( words ) . keys ( ) ) ) print 'Limited vocabulary size {}' . format ( vocabulary_size ) assert ( len ( collections . Counter ( words ) . keys ( ) ) >= vocabulary_size ) , 'the limited vocabulary_size must be less than or equal to the read vocabulary_size' return ( data , count , dictionary , reverse_dictionary )
build the words dictionary and replace rare words with unk token .