public void buildVocab() {


        val constructor = new VocabConstructor.Builder<T>().addSource(iterator, minWordFrequency)
                        .setTargetVocabCache(vocab).fetchLabels(trainSequenceVectors).setStopWords(stopWords)
                        .enableScavenger(enableScavenger).setEntriesLimit(vocabLimit)
                        .allowParallelTokenization(configuration.isAllowParallelTokenization())
                        .setUnk(useUnknown && unknownElement != null ? unknownElement : null).build();

        if (existingModel != null && lookupTable instanceof InMemoryLookupTable
                        && existingModel.lookupTable() instanceof InMemoryLookupTable) {
            log.info("Merging existing vocabulary into the current one...");
            /*
                if we have existing model defined, we're forced to fetch labels only.
                the rest of vocabulary & weights should be transferred from existing model
             */

            constructor.buildMergedVocabulary(existingModel, true);

            /*
                Now we have vocab transferred, and we should transfer syn0 values into lookup table
             */
            ((InMemoryLookupTable<VocabWord>) lookupTable)
                            .consume((InMemoryLookupTable<VocabWord>) existingModel.lookupTable());
        } else {
            log.info("Starting vocabulary building...");
            // if we don't have existing model defined, we just build vocabulary


            constructor.buildJointVocabulary(false, true);

            /*
            if (useUnknown && unknownElement != null && !vocab.containsWord(unknownElement.getLabel())) {
                log.info("Adding UNK element...");
                unknownElement.setSpecial(true);
                unknownElement.markAsLabel(false);
                unknownElement.setIndex(vocab.numWords());
                vocab.addToken(unknownElement);
            }
            */


            // check for malformed inputs. if numWords/numSentences ratio is huge, then user is passing something weird
            if (vocab.numWords() / constructor.getNumberOfSequences() > 1000) {
                log.warn("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
                log.warn("!                                                                                       !");
                log.warn("! Your input looks malformed: number of sentences is too low, model accuracy may suffer !");
                log.warn("!                                                                                       !");
                log.warn("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!");
            }
        }


    }
public void buildVocab ( ) { val constructor = new VocabConstructor . Builder < T > ( ) . addSource ( iterator , minWordFrequency ) . setTargetVocabCache ( vocab ) . fetchLabels ( trainSequenceVectors ) . setStopWords ( stopWords ) . enableScavenger ( enableScavenger ) . setEntriesLimit ( vocabLimit ) . allowParallelTokenization ( configuration . isAllowParallelTokenization ( ) ) . setUnk ( useUnknown && unknownElement != null ? unknownElement : null ) . build ( ) ; if ( existingModel != null && lookupTable instanceof InMemoryLookupTable && existingModel . lookupTable ( ) instanceof InMemoryLookupTable ) { log . info ( "Merging existing vocabulary into the current one..." ) ; constructor . buildMergedVocabulary ( existingModel , true ) ; ( ( InMemoryLookupTable < VocabWord > ) lookupTable ) . consume ( ( InMemoryLookupTable < VocabWord > ) existingModel . lookupTable ( ) ) ; } else { log . info ( "Starting vocabulary building..." ) ; constructor . buildJointVocabulary ( false , true ) ; if ( vocab . numWords ( ) / constructor . getNumberOfSequences ( ) > 1000 ) { log . warn ( "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" ) ; log . warn ( "!                                                                                       !" ) ; log . warn ( "! Your input looks malformed: number of sentences is too low, model accuracy may suffer !" ) ; log . warn ( "!                                                                                       !" ) ; log . warn ( "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" ) ; } } }
Builds vocabulary from provided SequenceIterator instance