private void processSpace() throws IOException {
        LOGGER.info("generating reflective vectors");
        compressedDocumentsWriter.close();
        int numDocuments = documentCounter.get();
        termToIndexVector.clear();
        indexToTerm = new String[termToIndex.size()];
        for (Map.Entry<String,Integer> e : termToIndex.entrySet())
            indexToTerm[e.getValue()] = e.getKey();

        // Read in the compressed version of the corpus, re-processing each
        // document to build up the document vectors
        DataInputStream corpusReader = new DataInputStream(
            new BufferedInputStream(new FileInputStream(compressedDocuments)));

        // Set up the concurrent data structures so we can reprocess the
        // documents concurrently using a work queue
        final BlockingQueue<Runnable> workQueue =
            new LinkedBlockingQueue<Runnable>();
        for (int i = 0; i < Runtime.getRuntime().availableProcessors(); ++i) {
            Thread t = new WorkerThread(workQueue);
            t.start();
        }
        final Semaphore documentsRerocessed = new Semaphore(0);         

        for (int d = 0; d < numDocuments; ++d) {
            final int docId = d;

            // This value already has any filtered tokens taken into account,
            // i.e. in only counts those tokens that remain after filtering
            int tokensInDoc = corpusReader.readInt();
            // Read in the document
            final int[] doc = new int[tokensInDoc];
            for (int i = 0; i < tokensInDoc; ++i)
                doc[i] = corpusReader.readInt();

            workQueue.offer(new Runnable() {
                    public void run() {
                        // This method creates the document vector and then adds
                        // that document vector with the reflective semantic
                        // vector for each word occurring in the document
                        LOGGER.fine("reprocessing doc #" + docId);
                        processIntDocument(docToVector.get(docId), doc);
                        documentsRerocessed.release();
                    }
                });
        }
        corpusReader.close();

        // Wait until all the documents have been processed
        try {
            documentsRerocessed.acquire(numDocuments);
        } catch (InterruptedException ie) {
            throw new Error("interrupted while waiting for documents to " +
                            "finish reprocessing", ie);
        }        
        LOGGER.fine("finished reprocessing all documents");

    }
private void processSpace ( ) throws IOException { LOGGER . info ( "generating reflective vectors" ) ; compressedDocumentsWriter . close ( ) ; int numDocuments = documentCounter . get ( ) ; termToIndexVector . clear ( ) ; indexToTerm = new String [ termToIndex . size ( ) ] ; for ( Map . Entry < String , Integer > e : termToIndex . entrySet ( ) ) indexToTerm [ e . getValue ( ) ] = e . getKey ( ) ; DataInputStream corpusReader = new DataInputStream ( new BufferedInputStream ( new FileInputStream ( compressedDocuments ) ) ) ; final BlockingQueue < Runnable > workQueue = new LinkedBlockingQueue < Runnable > ( ) ; for ( int i = 0 ; i < Runtime . getRuntime ( ) . availableProcessors ( ) ; ++ i ) { Thread t = new WorkerThread ( workQueue ) ; t . start ( ) ; } final Semaphore documentsRerocessed = new Semaphore ( 0 ) ; for ( int d = 0 ; d < numDocuments ; ++ d ) { final int docId = d ; int tokensInDoc = corpusReader . readInt ( ) ; final int [ ] doc = new int [ tokensInDoc ] ; for ( int i = 0 ; i < tokensInDoc ; ++ i ) doc [ i ] = corpusReader . readInt ( ) ; workQueue . offer ( new Runnable ( ) { public void run ( ) { LOGGER . fine ( "reprocessing doc #" + docId ) ; processIntDocument ( docToVector . get ( docId ) , doc ) ; documentsRerocessed . release ( ) ; } } ) ; } corpusReader . close ( ) ; try { documentsRerocessed . acquire ( numDocuments ) ; } catch ( InterruptedException ie ) { throw new Error ( "interrupted while waiting for documents to " + "finish reprocessing" , ie ) ; } LOGGER . fine ( "finished reprocessing all documents" ) ; }
Computes the reflective semantic vectors for word meanings