src/main/java/ch/uzh/ifi/seal/ase/cscc/RecommenderHelper.java
public void trainModel(String modelOutputDir) {

        addShutdownHook();

        String lastProcessedZip = readProgressFile(modelOutputDir);

        List<String> zips = IoHelper.findAllZips(contextsDir);

        IInvertedIndex diskIndex = new DiskBasedInvertedIndex(modelOutputDir);
        CompletionModel completionModel = new CompletionModel(diskIndex);

        int zipTotal = zips.size();

        int indexOfNextZipToProcess = 0;

        if (zips.size() > 0) {

            if (zips.contains(lastProcessedZip)) {

                int indexOfLastProcessedZip = zips.indexOf(lastProcessedZip);

                indexOfNextZipToProcess = indexOfLastProcessedZip + 1;

                if (indexOfNextZipToProcess >= zips.size()) {
                    LOGGER.info("No more zips to process.");
                    return;
                }

                LOGGER.info("Found previous indexing state. Continuing indexing with zip " +
                        (indexOfNextZipToProcess + 1) + "/" + zipTotal + "(" + zips.get(indexOfNextZipToProcess) + ")");

            } else {

                // Reaching this part of the code means we have to start training from scratch
                LOGGER.info("Did not find previous indexing state. Starting training from beginning " +
                        (indexOfNextZipToProcess + 1) + "/" + zipTotal + "(" + zips.get(0) + ")");
            }

            // Get the list of zips we still have to process
            List<String> zipsToDo = zips.subList(indexOfNextZipToProcess, zips.size());

            completionModel.startTraining();

            for (String zip : zipsToDo) {

                if (!CSCCConfiguration.keepRunning) break;

                int indexOfCurrentZip = zips.indexOf(zip);

                if (CSCCConfiguration.PRINT_PROGRESS) {
                    double perc = 100 * (indexOfCurrentZip + 1) / (double) zipTotal;
                    System.out.printf("## %s, processing %s...\n(%d/%d, %.1f%% done)\n", new Date(), zip, indexOfCurrentZip + 1, zipTotal,
                            perc);
                }

                try (IReadingArchive ra = new ReadingArchive(new File(zip))) {

                    while (ra.hasNext() && CSCCConfiguration.keepRunning) {
                        if (CSCCConfiguration.PRINT_PROGRESS_CONTEXTS) {
                            System.out.printf("."); // print '.' to indicate that a context is being processed
                        }
                        Context ctx = ra.getNext(Context.class);
                        completionModel.train(ctx);
                    }
                    ra.close();
                    if (CSCCConfiguration.PRINT_PROGRESS_CONTEXTS) {
                        System.out.println();
                    }
                }

                // only store progress if current zip file processing has not been interrupted
                if (CSCCConfiguration.keepRunning) {
                    writeProgressFile(modelOutputDir, zip);
                }
            }
            completionModel.finishTraining();
        }
    }
/**
     * Train the model on the full contexts data set. The model will be persisted at the given location as it is being
     * trained. This allows training to be interrupted and continued at a later point. If a previously trained model is
     * found in the modelOutputDir, training will pick up where it left off last time.
     * ATTENTION: You can only continue training on the same training data. This means the structure of the contextDir
     * must be the same (still contain the same zips) as last time.
     *
     * @param modelOutputDir Directory where to store the learned model. Will create a subdirectory called 'CSCCInvertedIndex'.
     */
Train the model on the full contexts data set. The model will be persisted at the given location as it is being
trained. This allows training to be interrupted and continued at a later point. If a previously trained model is
found in the modelOutputDir, training will pick up where it left off last time.
ATTENTION: You can only continue training on the same training data. This means the structure of the contextDir
must be the same (still contain the same zips) as last time.

@param modelOutputDir Directory where to store the learned model. Will create a subdirectory called 'CSCCInvertedIndex'.
Train the model on the full contexts data set . The model will be persisted at the given location as it is being trained . This allows training to be interrupted and continued at a later point . If a previously trained model is found in the modelOutputDir training will pick up where it left off last time . ATTENTION : You can only continue training on the same training data . This means the structure of the contextDir must be the same ( still contain the same zips ) as last time . @param modelOutputDir Directory where to store the learned model . Will create a subdirectory called ' CSCCInvertedIndex ' .
public void trainModel(String modelOutputDir) {
        addShutdownHook();
        String lastProcessedZip = readProgressFile(modelOutputDir);
        List<String> zips = IoHelper.findAllZips(contextsDir);
        IInvertedIndex diskIndex = new DiskBasedInvertedIndex(modelOutputDir);
        CompletionModel completionModel = new CompletionModel(diskIndex);
        int zipTotal = zips.size();
        int indexOfNextZipToProcess = 0;
        if (zips.size() > 0) {
            if (zips.contains(lastProcessedZip)) {
                int indexOfLastProcessedZip = zips.indexOf(lastProcessedZip);
                indexOfNextZipToProcess = indexOfLastProcessedZip + 1;
                if (indexOfNextZipToProcess >= zips.size()) {
                    LOGGER.info("No more zips to process.");
                    return;
                }
                LOGGER.info("Found previous indexing state. Continuing indexing with zip " +
                        (indexOfNextZipToProcess + 1) + "/" + zipTotal + "(" + zips.get(indexOfNextZipToProcess) + ")");
            } else {
                LOGGER.info("Did not find previous indexing state. Starting training from beginning " +
                        (indexOfNextZipToProcess + 1) + "/" + zipTotal + "(" + zips.get(0) + ")");
            }
            List<String> zipsToDo = zips.subList(indexOfNextZipToProcess, zips.size());
            completionModel.startTraining();
            for (String zip : zipsToDo) {
                if (!CSCCConfiguration.keepRunning) break;
                int indexOfCurrentZip = zips.indexOf(zip);
                if (CSCCConfiguration.PRINT_PROGRESS) {
                    double perc = 100 * (indexOfCurrentZip + 1) / (double) zipTotal;
                    System.out.printf("## %s, processing %s...\n(%d/%d, %.1f%% done)\n", new Date(), zip, indexOfCurrentZip + 1, zipTotal,
                            perc);
                }
                try (IReadingArchive ra = new ReadingArchive(new File(zip))) {
                    while (ra.hasNext() && CSCCConfiguration.keepRunning) {
                        if (CSCCConfiguration.PRINT_PROGRESS_CONTEXTS) {
                            System.out.printf("."); 
                        }
                        Context ctx = ra.getNext(Context.class);
                        completionModel.train(ctx);
                    }
                    ra.close();
                    if (CSCCConfiguration.PRINT_PROGRESS_CONTEXTS) {
                        System.out.println();
                    }
                }
                if (CSCCConfiguration.keepRunning) {
                    writeProgressFile(modelOutputDir, zip);
                }
            }
            completionModel.finishTraining();
        }
    }
public void trainModel ( String modelOutputDir ) { addShutdownHook ( ) ; String lastProcessedZip = readProgressFile ( modelOutputDir ) ; List < String > zips = IoHelper . findAllZips ( contextsDir ) ; IInvertedIndex diskIndex = new DiskBasedInvertedIndex ( modelOutputDir ) ; CompletionModel completionModel = new CompletionModel ( diskIndex ) ; int zipTotal = zips . size ( ) ; int indexOfNextZipToProcess = 0 ; if ( zips . size ( ) > 0 ) { if ( zips . contains ( lastProcessedZip ) ) { int indexOfLastProcessedZip = zips . indexOf ( lastProcessedZip ) ; indexOfNextZipToProcess = indexOfLastProcessedZip + 1 ; if ( indexOfNextZipToProcess >= zips . size ( ) ) { LOGGER . info ( "No more zips to process." ) ; return ; } LOGGER . info ( "Found previous indexing state. Continuing indexing with zip " + ( indexOfNextZipToProcess + 1 ) + "/" + zipTotal + "(" + zips . get ( indexOfNextZipToProcess ) + ")" ) ; } else { LOGGER . info ( "Did not find previous indexing state. Starting training from beginning " + ( indexOfNextZipToProcess + 1 ) + "/" + zipTotal + "(" + zips . get ( 0 ) + ")" ) ; } List < String > zipsToDo = zips . subList ( indexOfNextZipToProcess , zips . size ( ) ) ; completionModel . startTraining ( ) ; for ( String zip : zipsToDo ) { if ( ! CSCCConfiguration . keepRunning ) break ; int indexOfCurrentZip = zips . indexOf ( zip ) ; if ( CSCCConfiguration . PRINT_PROGRESS ) { double perc = 100 * ( indexOfCurrentZip + 1 ) / ( double ) zipTotal ; System . out . printf ( "## %s, processing %s...\n(%d/%d, %.1f%% done)\n" , new Date ( ) , zip , indexOfCurrentZip + 1 , zipTotal , perc ) ; } try ( IReadingArchive ra = new ReadingArchive ( new File ( zip ) ) ) { while ( ra . hasNext ( ) && CSCCConfiguration . keepRunning ) { if ( CSCCConfiguration . PRINT_PROGRESS_CONTEXTS ) { System . out . printf ( "." ) ; } Context ctx = ra . getNext ( Context . class ) ; completionModel . train ( ctx ) ; } ra . close ( ) ; if ( CSCCConfiguration . PRINT_PROGRESS_CONTEXTS ) { System . out . println ( ) ; } } if ( CSCCConfiguration . keepRunning ) { writeProgressFile ( modelOutputDir , zip ) ; } } completionModel . finishTraining ( ) ; } }
Train the model on the full contexts data set.
Train the model on the full contexts data set .
// Reaching this part of the code means we have to start training from scratch
 // Get the list of zips we still have to process
 // print '.' to indicate that a context is being processed
 // only store progress if current zip file processing has not been interrupted
