deeplearning4j-scaleout/deeplearning4j-nlp/src/main/java/org/deeplearning4j/models/embeddings/loader/WordVectorSerializer.java
/**
     *
     * @param path - path to previously stored w2v json model
     * @return - Word2Vec instance
     */
@param path - path to previously stored w2v json model
@return - Word2Vec instance
@param path - path to previously stored w2v json model @return - Word2Vec instance
public static Word2Vec loadFullModel(@NonNull String path) {
        /*
            // TODO: implementation is in process
            We need to restore:
                     1. WeightLookupTable, including syn0 and syn1 matrices
                     2. VocabCache + mark it as SPECIAL, to avoid accidental word removals
         */
        LineSentenceIterator iterator = new LineSentenceIterator(new File(path));
        String confJson  = iterator.nextSentence();
        log.info("Word2Vec conf. JSON: " + confJson);
        VectorsConfiguration configuration = VectorsConfiguration.fromJson(confJson);
        String eTable = iterator.nextSentence();
        double[] expTable;
        String nTable = iterator.nextSentence();
        if (configuration.getNegative() > 0) {
        }
        /*
                Since we're restoring vocab from previously serialized model, we can expect minWordFrequency appliance in its vocabulary, so it should NOT be truncated.
                That's why i'm setting minWordFrequency to configuration value, but applying SPECIAL to each word, to avoid truncation
         */
        VocabularyHolder holder = new VocabularyHolder.Builder()
                .minWordFrequency(configuration.getMinWordFrequency())
                .hugeModelExpected(configuration.isHugeModelExpected())
                .scavengerActivationThreshold(configuration.getScavengerActivationThreshold())
                .scavengerRetentionDelay(configuration.getScavengerRetentionDelay())
                .build();
        while (iterator.hasNext()) {
            String wordJson = iterator.nextSentence();
            VocabularyWord word = VocabularyWord.fromJson(wordJson);
            word.setSpecial(true);
            holder.addWord(word);
        }
        InMemoryLookupCache vocabCache = new InMemoryLookupCache(false);
        holder.transferBackToVocabCache(vocabCache, false);
        InMemoryLookupTable lookupTable = (InMemoryLookupTable) new InMemoryLookupTable.Builder()
                .negative(configuration.getNegative())
                .useAdaGrad(configuration.isUseAdaGrad())
                .lr(configuration.getLearningRate())
                .cache(vocabCache)
                .vectorLength(configuration.getLayersSize())
                .build();
        lookupTable.resetWeights(true);
        for (VocabularyWord word: holder.getVocabulary()) {
            INDArray syn0 = lookupTable.getSyn0().getRow(vocabCache.indexOf(word.getWord()));
            syn0.assign(Nd4j.create(word.getSyn0()));
            INDArray syn1 =  lookupTable.getSyn1().getRow(vocabCache.indexOf(word.getWord()));
            syn1.assign(Nd4j.create(word.getSyn1()));
            if (configuration.getNegative() > 0) {
                INDArray syn1Neg =  lookupTable.getSyn1Neg().getRow(vocabCache.indexOf(word.getWord()));
                syn1Neg.assign(Nd4j.create(word.getSyn1Neg()));
            }
        }
        return new Word2Vec.Builder(configuration)
                .vocabCache(vocabCache)
                .lookupTable(lookupTable)
                .resetModel(false)
                .build();
    }
public static Word2Vec loadFullModel ( @ NonNull String path ) { /*
            // TODO: implementation is in process
            We need to restore:
                     1. WeightLookupTable, including syn0 and syn1 matrices
                     2. VocabCache + mark it as SPECIAL, to avoid accidental word removals
         */ LineSentenceIterator iterator = new LineSentenceIterator ( new File ( path ) ) ; String confJson = iterator . nextSentence ( ) ; log . info ( "Word2Vec conf. JSON: " + confJson ) ; VectorsConfiguration configuration = VectorsConfiguration . fromJson ( confJson ) ; String eTable = iterator . nextSentence ( ) ; double [ ] expTable ; String nTable = iterator . nextSentence ( ) ; if ( configuration . getNegative ( ) > 0 ) { } /*
                Since we're restoring vocab from previously serialized model, we can expect minWordFrequency appliance in its vocabulary, so it should NOT be truncated.
                That's why i'm setting minWordFrequency to configuration value, but applying SPECIAL to each word, to avoid truncation
         */ VocabularyHolder holder = new VocabularyHolder . Builder ( ) . minWordFrequency ( configuration . getMinWordFrequency ( ) ) . hugeModelExpected ( configuration . isHugeModelExpected ( ) ) . scavengerActivationThreshold ( configuration . getScavengerActivationThreshold ( ) ) . scavengerRetentionDelay ( configuration . getScavengerRetentionDelay ( ) ) . build ( ) ; while ( iterator . hasNext ( ) ) { String wordJson = iterator . nextSentence ( ) ; VocabularyWord word = VocabularyWord . fromJson ( wordJson ) ; word . setSpecial ( true ) ; holder . addWord ( word ) ; } InMemoryLookupCache vocabCache = new InMemoryLookupCache ( false ) ; holder . transferBackToVocabCache ( vocabCache , false ) ; InMemoryLookupTable lookupTable = ( InMemoryLookupTable ) new InMemoryLookupTable . Builder ( ) . negative ( configuration . getNegative ( ) ) . useAdaGrad ( configuration . isUseAdaGrad ( ) ) . lr ( configuration . getLearningRate ( ) ) . cache ( vocabCache ) . vectorLength ( configuration . getLayersSize ( ) ) . build ( ) ; lookupTable . resetWeights ( true ) ; for ( VocabularyWord word : holder . getVocabulary ( ) ) { INDArray syn0 = lookupTable . getSyn0 ( ) . getRow ( vocabCache . indexOf ( word . getWord ( ) ) ) ; syn0 . assign ( Nd4j . create ( word . getSyn0 ( ) ) ) ; INDArray syn1 = lookupTable . getSyn1 ( ) . getRow ( vocabCache . indexOf ( word . getWord ( ) ) ) ; syn1 . assign ( Nd4j . create ( word . getSyn1 ( ) ) ) ; if ( configuration . getNegative ( ) > 0 ) { INDArray syn1Neg = lookupTable . getSyn1Neg ( ) . getRow ( vocabCache . indexOf ( word . getWord ( ) ) ) ; syn1Neg . assign ( Nd4j . create ( word . getSyn1Neg ( ) ) ) ; } } return new Word2Vec . Builder ( configuration ) . vocabCache ( vocabCache ) . lookupTable ( lookupTable ) . resetModel ( false ) . build ( ) ; }
@param path - path to previously stored w2v json model
 @return - Word2Vec instance
@param path - path to previously stored w2v json model @return - Word2Vec instance
// first 3 lines should be processed separately // actually we dont need expTable, since it produces exact results on subsequent runs untill you dont modify expTable size :) // TODO: we probably should parse negTable, but it's not required until vocab changes are introduced. Since on the predefined vocab it will produce exact nTable, the same goes for expTable btw. //    log.info("got line: " + iterator.nextSentence()); // at this moment vocab is restored, and it's time to rebuild Huffman tree // since word counters are equal, huffman tree will be equal too //holder.updateHuffmanCodes(); // we definitely don't need UNK word in this scenarion // now, it's time to transfer syn0/syn1/syn1 neg values // we create all arrays // now, for each word from vocabHolder we'll just transfer actual values // syn0 transfer // syn1 transfer // syn1 values are being accessed via tree points, but since our goal is just deserialization - we can just push it row by row // syn1Neg transfer