public static WordVectors loadStaticModel(@NonNull File file) {
        if (!file.exists() || file.isDirectory())
            throw new RuntimeException(
                    new FileNotFoundException("File [" + file.getAbsolutePath() + "] was not found"));

        int originalFreq = Nd4j.getMemoryManager().getOccasionalGcFrequency();
        boolean originalPeriodic = Nd4j.getMemoryManager().isPeriodicGcActive();

        if (originalPeriodic)
            Nd4j.getMemoryManager().togglePeriodicGc(false);

        Nd4j.getMemoryManager().setOccasionalGcFrequency(50000);

        CompressedRamStorage<Integer> storage = new CompressedRamStorage.Builder<Integer>().useInplaceCompression(false)
                        .setCompressor(new NoOp()).emulateIsAbsent(false).build();

        VocabCache<VocabWord> vocabCache = new AbstractCache.Builder<VocabWord>().build();


        // now we need to define which file format we have here
        // if zip - that's dl4j format
        try {
            log.debug("Trying DL4j format...");
            File tmpFileSyn0 = DL4JFileUtils.createTempFile("word2vec", "syn");
            tmpFileSyn0.deleteOnExit();

            ZipFile zipFile = new ZipFile(file);
            ZipEntry syn0 = zipFile.getEntry("syn0.txt");
            InputStream stream = zipFile.getInputStream(syn0);

            FileUtils.copyInputStreamToFile(stream, tmpFileSyn0);
            storage.clear();

            try (Reader reader = new CSVReader(tmpFileSyn0)) {
                while (reader.hasNext()) {
                    Pair<VocabWord, float[]> pair = reader.next();
                    VocabWord word = pair.getFirst();
                    storage.store(word.getIndex(), pair.getSecond());

                    vocabCache.addToken(word);
                    vocabCache.addWordToIndex(word.getIndex(), word.getLabel());

                    Nd4j.getMemoryManager().invokeGcOccasionally();
                }
            } catch (Exception e) {
                throw new RuntimeException(e);
            } finally {
                if (originalPeriodic)
                    Nd4j.getMemoryManager().togglePeriodicGc(true);

                Nd4j.getMemoryManager().setOccasionalGcFrequency(originalFreq);

                try {
                    tmpFileSyn0.delete();
                } catch (Exception e) {
                    //
                }
            }
        } catch (Exception e) {
            //
            try {
                // try to load file as text csv
                vocabCache = new AbstractCache.Builder<VocabWord>().build();
                storage.clear();
                log.debug("Trying CSVReader...");
                try (Reader reader = new CSVReader(file)) {
                    while (reader.hasNext()) {
                        Pair<VocabWord, float[]> pair = reader.next();
                        VocabWord word = pair.getFirst();
                        storage.store(word.getIndex(), pair.getSecond());

                        vocabCache.addToken(word);
                        vocabCache.addWordToIndex(word.getIndex(), word.getLabel());

                        Nd4j.getMemoryManager().invokeGcOccasionally();
                    }
                } catch (Exception ef) {
                    // we throw away this exception, and trying to load data as binary model
                    throw new RuntimeException(ef);
                } finally {
                    if (originalPeriodic)
                        Nd4j.getMemoryManager().togglePeriodicGc(true);

                    Nd4j.getMemoryManager().setOccasionalGcFrequency(originalFreq);
                }
            } catch (Exception ex) {
                // otherwise it's probably google model. which might be compressed or not
                log.debug("Trying BinaryReader...");
                vocabCache = new AbstractCache.Builder<VocabWord>().build();
                storage.clear();
                try (Reader reader = new BinaryReader(file)) {
                    while (reader.hasNext()) {
                        Pair<VocabWord, float[]> pair = reader.next();
                        VocabWord word = pair.getFirst();

                        storage.store(word.getIndex(), pair.getSecond());

                        vocabCache.addToken(word);
                        vocabCache.addWordToIndex(word.getIndex(), word.getLabel());

                        Nd4j.getMemoryManager().invokeGcOccasionally();
                    }
                } catch (Exception ez) {
                    throw new RuntimeException("Unable to guess input file format");
                } finally {
                    if (originalPeriodic)
                        Nd4j.getMemoryManager().togglePeriodicGc(true);

                    Nd4j.getMemoryManager().setOccasionalGcFrequency(originalFreq);
                }
            } finally {
                if (originalPeriodic)
                    Nd4j.getMemoryManager().togglePeriodicGc(true);

                Nd4j.getMemoryManager().setOccasionalGcFrequency(originalFreq);
            }
        }


        StaticWord2Vec word2Vec = new StaticWord2Vec.Builder(storage, vocabCache).build();

        return word2Vec;
    }
public static WordVectors loadStaticModel ( @ NonNull File file ) { if ( ! file . exists ( ) || file . isDirectory ( ) ) throw new RuntimeException ( new FileNotFoundException ( "File [" + file . getAbsolutePath ( ) + "] was not found" ) ) ; int originalFreq = Nd4j . getMemoryManager ( ) . getOccasionalGcFrequency ( ) ; boolean originalPeriodic = Nd4j . getMemoryManager ( ) . isPeriodicGcActive ( ) ; if ( originalPeriodic ) Nd4j . getMemoryManager ( ) . togglePeriodicGc ( false ) ; Nd4j . getMemoryManager ( ) . setOccasionalGcFrequency ( 50000 ) ; CompressedRamStorage < Integer > storage = new CompressedRamStorage . Builder < Integer > ( ) . useInplaceCompression ( false ) . setCompressor ( new NoOp ( ) ) . emulateIsAbsent ( false ) . build ( ) ; VocabCache < VocabWord > vocabCache = new AbstractCache . Builder < VocabWord > ( ) . build ( ) ; try { log . debug ( "Trying DL4j format..." ) ; File tmpFileSyn0 = DL4JFileUtils . createTempFile ( "word2vec" , "syn" ) ; tmpFileSyn0 . deleteOnExit ( ) ; ZipFile zipFile = new ZipFile ( file ) ; ZipEntry syn0 = zipFile . getEntry ( "syn0.txt" ) ; InputStream stream = zipFile . getInputStream ( syn0 ) ; FileUtils . copyInputStreamToFile ( stream , tmpFileSyn0 ) ; storage . clear ( ) ; try ( Reader reader = new CSVReader ( tmpFileSyn0 ) ) { while ( reader . hasNext ( ) ) { Pair < VocabWord , float [ ] > pair = reader . next ( ) ; VocabWord word = pair . getFirst ( ) ; storage . store ( word . getIndex ( ) , pair . getSecond ( ) ) ; vocabCache . addToken ( word ) ; vocabCache . addWordToIndex ( word . getIndex ( ) , word . getLabel ( ) ) ; Nd4j . getMemoryManager ( ) . invokeGcOccasionally ( ) ; } } catch ( Exception e ) { throw new RuntimeException ( e ) ; } finally { if ( originalPeriodic ) Nd4j . getMemoryManager ( ) . togglePeriodicGc ( true ) ; Nd4j . getMemoryManager ( ) . setOccasionalGcFrequency ( originalFreq ) ; try { tmpFileSyn0 . delete ( ) ; } catch ( Exception e ) { } } } catch ( Exception e ) { try { vocabCache = new AbstractCache . Builder < VocabWord > ( ) . build ( ) ; storage . clear ( ) ; log . debug ( "Trying CSVReader..." ) ; try ( Reader reader = new CSVReader ( file ) ) { while ( reader . hasNext ( ) ) { Pair < VocabWord , float [ ] > pair = reader . next ( ) ; VocabWord word = pair . getFirst ( ) ; storage . store ( word . getIndex ( ) , pair . getSecond ( ) ) ; vocabCache . addToken ( word ) ; vocabCache . addWordToIndex ( word . getIndex ( ) , word . getLabel ( ) ) ; Nd4j . getMemoryManager ( ) . invokeGcOccasionally ( ) ; } } catch ( Exception ef ) { throw new RuntimeException ( ef ) ; } finally { if ( originalPeriodic ) Nd4j . getMemoryManager ( ) . togglePeriodicGc ( true ) ; Nd4j . getMemoryManager ( ) . setOccasionalGcFrequency ( originalFreq ) ; } } catch ( Exception ex ) { log . debug ( "Trying BinaryReader..." ) ; vocabCache = new AbstractCache . Builder < VocabWord > ( ) . build ( ) ; storage . clear ( ) ; try ( Reader reader = new BinaryReader ( file ) ) { while ( reader . hasNext ( ) ) { Pair < VocabWord , float [ ] > pair = reader . next ( ) ; VocabWord word = pair . getFirst ( ) ; storage . store ( word . getIndex ( ) , pair . getSecond ( ) ) ; vocabCache . addToken ( word ) ; vocabCache . addWordToIndex ( word . getIndex ( ) , word . getLabel ( ) ) ; Nd4j . getMemoryManager ( ) . invokeGcOccasionally ( ) ; } } catch ( Exception ez ) { throw new RuntimeException ( "Unable to guess input file format" ) ; } finally { if ( originalPeriodic ) Nd4j . getMemoryManager ( ) . togglePeriodicGc ( true ) ; Nd4j . getMemoryManager ( ) . setOccasionalGcFrequency ( originalFreq ) ; } } finally { if ( originalPeriodic ) Nd4j . getMemoryManager ( ) . togglePeriodicGc ( true ) ; Nd4j . getMemoryManager ( ) . setOccasionalGcFrequency ( originalFreq ) ; } } StaticWord2Vec word2Vec = new StaticWord2Vec . Builder ( storage , vocabCache ) . build ( ) ; return word2Vec ; }
TODO: this method needs better name :)