deeplearning4j-scaleout/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/word2vec/wordstore/VocabConstructor.java
/**
     * This method scans all sources passed through builder, and returns all words as vocab.
     * If TargetVocabCache was set during instance creation, it'll be filled too.
     *
     *
     * @return
     */
This method scans all sources passed through builder, and returns all words as vocab.
If TargetVocabCache was set during instance creation, it'll be filled too.

@return
This method scans all sources passed through builder and returns all words as vocab . If TargetVocabCache was set during instance creation it ' ll be filled too . @return
public VocabCache<T> buildJointVocabulary(boolean resetCounters, boolean buildHuffmanTree) {
        if (resetCounters && buildHuffmanTree) throw new IllegalStateException("You can't reset counters and build Huffman tree at the same time!");
        if (cache == null) throw new IllegalStateException("Cache is null, building fresh one");
        if (cache == null) cache = new AbstractCache.Builder<T>().build();
        log.debug("Target vocab size before building: [" + cache.numWords() + "]");
        final AtomicLong sequenceCounter = new AtomicLong(0);
        final AtomicLong elementsCounter = new AtomicLong(0);
        AbstractCache<T> topHolder = new AbstractCache.Builder<T>()
                .minElementFrequency(0)
                .build();
        int cnt = 0;
        for(VocabSource<T> source: sources) {
            SequenceIterator<T> iterator = source.getIterator();
            iterator.reset();
            log.debug("Trying source iterator: ["+ cnt+"]");
            log.debug("Target vocab size before building: [" + cache.numWords() + "]");
            cnt++;
            AbstractCache<T> tempHolder = new AbstractCache.Builder<T>().build();
            int sequences = 0;
            long counter = 0;
            while (iterator.hasMoreSequences()) {
                Sequence<T> document = iterator.nextSequence();
                sequenceCounter.incrementAndGet();
                if (fetchLabels) {
                    T labelWord = document.getSequenceLabel();
                    labelWord.setSpecial(true);
                    labelWord.setElementFrequency(1);
                    tempHolder.addToken(labelWord);
                }
                List<String> tokens = document.asLabels();
                for (String token: tokens) {
                    if (stopWords !=null && stopWords.contains(token)) continue;
                    if (token == null || token.isEmpty()) continue;
                    if (!tempHolder.containsWord(token)) {
                        tempHolder.addToken(document.getElementByLabel(token));
                        elementsCounter.incrementAndGet();
                        counter++;
                        /*
                        if (useAdaGrad) {
                            VocabularyWord word = tempHolder.getVocabularyWordByString(token);
                            word.setHistoricalGradient(new double[layerSize]);
                        }
                        */
                    } else {
                        counter++;
                        tempHolder.incrementWordCount(token);
                    }
                }
                sequences++;
                if (sequenceCounter.get() % 100000 == 0) log.info("Sequences checked: [" + sequenceCounter.get() +"], Current vocabulary size: [" + elementsCounter.get() +"]");
            }
            log.debug("Vocab size before truncation: [" + tempHolder.numWords() + "],  NumWords: [" + tempHolder.totalWordOccurrences()+ "], sequences parsed: [" + sequences+ "], counter: ["+counter+"]");
            if (source.getMinWordFrequency() > 0) {
                LinkedBlockingQueue<String> labelsToRemove = new LinkedBlockingQueue<>();
                for (T element : tempHolder.vocabWords()) {
                    if (element.getElementFrequency() < source.getMinWordFrequency() && !element.isSpecial())
                        labelsToRemove.add(element.getLabel());
                }
                for (String label: labelsToRemove) {
                    log.debug("Removing label: '" + label + "'");
                    tempHolder.removeElement(label);
                }
            }
            log.debug("Vocab size after truncation: [" + tempHolder.numWords() + "],  NumWords: [" + tempHolder.totalWordOccurrences()+ "], sequences parsed: [" + sequences+ "], counter: ["+counter+"]");
            topHolder.importVocabulary(tempHolder);
            log.debug("Top holder size: ["+ topHolder.numWords()+"]");
            log.debug("Target vocab size before building: [" + cache.numWords() + "]");
        }
        cache.importVocabulary(topHolder);
        if (resetCounters) {
            for (T element: cache.vocabWords()) {
                element.setElementFrequency(0);
            }
            cache.updateWordsOccurencies();
        }
        if (buildHuffmanTree) {
            Huffman huffman = new Huffman(cache.vocabWords());
            huffman.build();
            huffman.applyIndexes(cache);
        }
        log.info("Sequences checked: [" + sequenceCounter.get() +"], Current vocabulary size: [" + cache.numWords() +"]");
        return cache;
    }
public VocabCache < T > buildJointVocabulary ( boolean resetCounters , boolean buildHuffmanTree ) { if ( resetCounters && buildHuffmanTree ) throw new IllegalStateException ( "You can't reset counters and build Huffman tree at the same time!" ) ; if ( cache == null ) throw new IllegalStateException ( "Cache is null, building fresh one" ) ; if ( cache == null ) cache = new AbstractCache . Builder < T > ( ) . build ( ) ; log . debug ( "Target vocab size before building: [" + cache . numWords ( ) + "]" ) ; final AtomicLong sequenceCounter = new AtomicLong ( 0 ) ; final AtomicLong elementsCounter = new AtomicLong ( 0 ) ; AbstractCache < T > topHolder = new AbstractCache . Builder < T > ( ) . minElementFrequency ( 0 ) . build ( ) ; int cnt = 0 ; for ( VocabSource < T > source : sources ) { SequenceIterator < T > iterator = source . getIterator ( ) ; iterator . reset ( ) ; log . debug ( "Trying source iterator: [" + cnt + "]" ) ; log . debug ( "Target vocab size before building: [" + cache . numWords ( ) + "]" ) ; cnt ++ ; AbstractCache < T > tempHolder = new AbstractCache . Builder < T > ( ) . build ( ) ; int sequences = 0 ; long counter = 0 ; while ( iterator . hasMoreSequences ( ) ) { Sequence < T > document = iterator . nextSequence ( ) ; sequenceCounter . incrementAndGet ( ) ; if ( fetchLabels ) { T labelWord = document . getSequenceLabel ( ) ; labelWord . setSpecial ( true ) ; labelWord . setElementFrequency ( 1 ) ; tempHolder . addToken ( labelWord ) ; } List < String > tokens = document . asLabels ( ) ; for ( String token : tokens ) { if ( stopWords != null && stopWords . contains ( token ) ) continue ; if ( token == null || token . isEmpty ( ) ) continue ; if ( ! tempHolder . containsWord ( token ) ) { tempHolder . addToken ( document . getElementByLabel ( token ) ) ; elementsCounter . incrementAndGet ( ) ; counter ++ ; /*
                        if (useAdaGrad) {
                            VocabularyWord word = tempHolder.getVocabularyWordByString(token);

                            word.setHistoricalGradient(new double[layerSize]);
                        }
                        */ } else { counter ++ ; tempHolder . incrementWordCount ( token ) ; } } sequences ++ ; if ( sequenceCounter . get ( ) % 100000 == 0 ) log . info ( "Sequences checked: [" + sequenceCounter . get ( ) + "], Current vocabulary size: [" + elementsCounter . get ( ) + "]" ) ; } log . debug ( "Vocab size before truncation: [" + tempHolder . numWords ( ) + "],  NumWords: [" + tempHolder . totalWordOccurrences ( ) + "], sequences parsed: [" + sequences + "], counter: [" + counter + "]" ) ; if ( source . getMinWordFrequency ( ) > 0 ) { LinkedBlockingQueue < String > labelsToRemove = new LinkedBlockingQueue < > ( ) ; for ( T element : tempHolder . vocabWords ( ) ) { if ( element . getElementFrequency ( ) < source . getMinWordFrequency ( ) && ! element . isSpecial ( ) ) labelsToRemove . add ( element . getLabel ( ) ) ; } for ( String label : labelsToRemove ) { log . debug ( "Removing label: '" + label + "'" ) ; tempHolder . removeElement ( label ) ; } } log . debug ( "Vocab size after truncation: [" + tempHolder . numWords ( ) + "],  NumWords: [" + tempHolder . totalWordOccurrences ( ) + "], sequences parsed: [" + sequences + "], counter: [" + counter + "]" ) ; topHolder . importVocabulary ( tempHolder ) ; log . debug ( "Top holder size: [" + topHolder . numWords ( ) + "]" ) ; log . debug ( "Target vocab size before building: [" + cache . numWords ( ) + "]" ) ; } cache . importVocabulary ( topHolder ) ; if ( resetCounters ) { for ( T element : cache . vocabWords ( ) ) { element . setElementFrequency ( 0 ) ; } cache . updateWordsOccurencies ( ) ; } if ( buildHuffmanTree ) { Huffman huffman = new Huffman ( cache . vocabWords ( ) ) ; huffman . build ( ) ; huffman . applyIndexes ( cache ) ; } log . info ( "Sequences checked: [" + sequenceCounter . get ( ) + "], Current vocabulary size: [" + cache . numWords ( ) + "]" ) ; return cache ; }
This method scans all sources passed through builder, and returns all words as vocab.
This method scans all sources passed through builder and returns all words as vocab .
//  log.info("Sequence length: ["+ document.getElements().size()+"]"); //   Tokenizer tokenizer = tokenizerFactory.create(document.getContent()); // TODO: this line should be uncommented only after AdaGrad is fixed, so the size of AdaGrad array is known // apply minWordFrequency set for this source // at this moment we're ready to transfer // at this moment, we have vocabulary full of words, and we have to reset counters before transfer everything back to VocabCache //topHolder.resetWordCounters(); //topHolder.updateHuffmanCodes();