public void processDocument(BufferedReader document) throws IOException {
        int docIndex = documentCounter.getAndIncrement();

        Iterator<String> documentTokens = 
            IteratorFactory.tokenizeOrdered(document);

        // As we read in the document, generate a compressed version of it,
        // which we will use during the process space method to recompute all of
        // the word vectors' semantics
        ByteArrayOutputStream compressedDocument = 
            new ByteArrayOutputStream(4096);
        DataOutputStream dos = new DataOutputStream(compressedDocument);
        int tokens = 0; // count how many are in this document
        int unfilteredTokens = 0; // how many tokens remained after filtering

        IntegerVector docVector = createVector();
        docToVector.put(docIndex, docVector);

        while (documentTokens.hasNext()) {
            tokens++;
            String focusWord = documentTokens.next();

            // If we are filtering the semantic vectors, check whether this word
            // should have its semantics calculated.  In addition, if there is a
            // filter and it would have excluded the word, do not keep its
            // semantics around
            boolean calculateSemantics =
                semanticFilter.isEmpty() || semanticFilter.contains(focusWord)
                && !focusWord.equals(IteratorFactory.EMPTY_TOKEN);

	    // If the filter does not accept this word, skip the semantic
	    // processing, continue with the next word
            if (!calculateSemantics) {
                // Do not write out any removed tokens to save space
		continue;
	    }

            // Update the occurrences of this token
            unfilteredTokens++;
            add(docVector, getTermIndexVector(focusWord));

            // Update the compress version of the document with the token.
            //
            // NOTE: this call to termToIndex *must* come after the
            // getTermIndexVector() call, which is responsible for adding this
            // mapping if it doesn't already exist.
	    int focusIndex = termToIndex.get(focusWord);

            // write the term index into the compressed for the document for
            // later corpus reprocessing
            dos.writeInt(focusIndex);
        }

        document.close();
        
        dos.close();
        byte[] docAsBytes = compressedDocument.toByteArray();

        // Once the document is finished, write the compressed contents to the
        // corpus stream
        synchronized(compressedDocumentsWriter) {
            // Write how many terms were in this document after filtering
            compressedDocumentsWriter.writeInt(unfilteredTokens);
            compressedDocumentsWriter.write(docAsBytes, 0, docAsBytes.length);
        }
    }
public void processDocument ( BufferedReader document ) throws IOException { int docIndex = documentCounter . getAndIncrement ( ) ; Iterator < String > documentTokens = IteratorFactory . tokenizeOrdered ( document ) ; ByteArrayOutputStream compressedDocument = new ByteArrayOutputStream ( 4096 ) ; DataOutputStream dos = new DataOutputStream ( compressedDocument ) ; int tokens = 0 ; int unfilteredTokens = 0 ; IntegerVector docVector = createVector ( ) ; docToVector . put ( docIndex , docVector ) ; while ( documentTokens . hasNext ( ) ) { tokens ++ ; String focusWord = documentTokens . next ( ) ; boolean calculateSemantics = semanticFilter . isEmpty ( ) || semanticFilter . contains ( focusWord ) && ! focusWord . equals ( IteratorFactory . EMPTY_TOKEN ) ; if ( ! calculateSemantics ) { continue ; } unfilteredTokens ++ ; add ( docVector , getTermIndexVector ( focusWord ) ) ; int focusIndex = termToIndex . get ( focusWord ) ; dos . writeInt ( focusIndex ) ; } document . close ( ) ; dos . close ( ) ; byte [ ] docAsBytes = compressedDocument . toByteArray ( ) ; synchronized ( compressedDocumentsWriter ) { compressedDocumentsWriter . writeInt ( unfilteredTokens ) ; compressedDocumentsWriter . write ( docAsBytes , 0 , docAsBytes . length ) ; } }
Updates the semantic vectors based on the words in the document.

@param document {@inheritDoc}