deduplicator-indexer/src/main/java/is/landsbokasafn/deduplicator/indexer/IndexBuilder.java
/**
     * Writes the contents of a {@link CrawlDataIterator} to this index.
     * <p>
     * This method may be invoked multiple times with different 
     * CrawlDataIterators until {@link #close(boolean)} has been called.
     * 
     * @param dataIt The CrawlDataIterator that provides the data to index.
     * @param mimefilter A regular expression that is used as a filter on the 
     *                   mimetypes to include in the index. 
     * @param blacklist If true then the <code>mimefilter</code> is used
     *                  as a blacklist for mimetypes. If false then the
     *                  <code>mimefilter</code> is treated as a whitelist. 
     * @param verbose If true then progress information will be sent to 
     *                System.out.
     * @return The number of items added to the index.
     * @throws IOException If an error occurs writing the index.
     */
Writes the contents of a  CrawlDataIterator to this index.

This method may be invoked multiple times with different
CrawlDataIterators until  #close(boolean) has been called.

@param dataIt The CrawlDataIterator that provides the data to index.
@param mimefilter A regular expression that is used as a filter on the
mimetypes to include in the index.
@param blacklist If true then the mimefilter is used
as a blacklist for mimetypes. If false then the
mimefilter is treated as a whitelist.
@param verbose If true then progress information will be sent to
System.out.
@return The number of items added to the index.
@throws IOException If an error occurs writing the index.
Writes the contents of a CrawlDataIterator to this index . This method may be invoked multiple times with different CrawlDataIterators until #close ( boolean ) has been called . @param dataIt The CrawlDataIterator that provides the data to index . @param mimefilter A regular expression that is used as a filter on the mimetypes to include in the index . @param blacklist If true then the mimefilter is used as a blacklist for mimetypes . If false then the mimefilter is treated as a whitelist . @param verbose If true then progress information will be sent to System . out . @return The number of items added to the index . @throws IOException If an error occurs writing the index .
public long writeToIndex(
            CrawlDataIterator dataIt, 
            String mimeFilter, 
            boolean blacklist,
            boolean verbose) 
            throws IOException {
        int count = 0;
        int skipped = 0;
        int unresolved = 0;
        FieldType ftIndexed = new FieldType();
        ftIndexed.setIndexed(true);
        ftIndexed.setTokenized(false);
        ftIndexed.setStored(true);
        FieldType ftNotIndexed = new FieldType(ftIndexed);
        ftNotIndexed.setIndexed(false);
        while (dataIt.hasNext()) {
            CrawlDataItem item = dataIt.next();
            if (item.getStatusCode()!=200) {
                skipped++;
            	continue;
            }
            if (item.getMimeType().matches(mimeFilter) == blacklist) {
            	skipped++;
            	continue;
            }
            String url = item.getURL();
            String timestamp = item.getTimestamp();
            if (item.isRevisit()) {
            	if (item.getOriginalURL()==null || item.getOriginalTimestamp()==null) {
        			unresolved++;
            		continue;
            	} else {
            		url = item.getOriginalURL();
            		timestamp = item.getOriginalTimestamp();
            	}
            }
            count++;
            if (verbose && count%10000==0) {
                System.out.println("Indexed " + count + ", unresolved " + unresolved + " - Last URL " +
                		"from " + item.getTimestamp());
            }
            if (url.contains("\"")) {
            	throw new IllegalStateException("Double quotes in URLs should always be properly escaped. " 
            			+ item.getURL());
            }
            Document doc = new Document();
            doc.add(new Field(
                    URL.name(),
                    url,
                    (indexURL ? ftIndexed : ftNotIndexed)));
            if(includeCanonicalizedURL){
                doc.add(new Field(
                        URL_CANONICALIZED.name(),
                        canonicalizer.canonicalize(item.getURL()),
                        (indexURL ? ftIndexed : ftNotIndexed)));
            }
            String digest = item.getContentDigest();
            if (!indexDigestScheme && digest.lastIndexOf(":") >= 0) {
				digest = digest.substring(digest.lastIndexOf(":") + 1);
            }
            doc.add(new Field(
                    DIGEST.name(),
                    digest,
                    ftIndexed));
            doc.add(new Field(
                    DATE.name(),
                    timestamp,
                    ftNotIndexed));
            if(includeEtag && item.getEtag()!=null){
                doc.add(new Field(
                        ETAG.name(),
                        item.getEtag(),
                        ftNotIndexed));
            }
            if (indexURL) {
            	BooleanQuery q = new BooleanQuery();
            	q.add(new TermQuery(new Term(URL.name(), url)), Occur.MUST);
            	q.add(new TermQuery(new Term(DIGEST.name(), item.getContentDigest())), Occur.MUST);
            	index.deleteDocuments(q);
            	index.addDocument(doc);
            } else {
                index.updateDocument(new Term(DIGEST.name()), doc);
            }
        }
        System.out.println("Indexed " + count + " items (unresolved " + unresolved + ", skipped " + skipped + ")");
        return count;
    }
public long writeToIndex ( CrawlDataIterator dataIt , String mimeFilter , boolean blacklist , boolean verbose ) throws IOException { int count = 0 ; int skipped = 0 ; int unresolved = 0 ; FieldType ftIndexed = new FieldType ( ) ; ftIndexed . setIndexed ( true ) ; ftIndexed . setTokenized ( false ) ; ftIndexed . setStored ( true ) ; FieldType ftNotIndexed = new FieldType ( ftIndexed ) ; ftNotIndexed . setIndexed ( false ) ; while ( dataIt . hasNext ( ) ) { CrawlDataItem item = dataIt . next ( ) ; if ( item . getStatusCode ( ) != 200 ) { skipped ++ ; continue ; } if ( item . getMimeType ( ) . matches ( mimeFilter ) == blacklist ) { skipped ++ ; continue ; } String url = item . getURL ( ) ; String timestamp = item . getTimestamp ( ) ; if ( item . isRevisit ( ) ) { if ( item . getOriginalURL ( ) == null || item . getOriginalTimestamp ( ) == null ) { unresolved ++ ; continue ; } else { url = item . getOriginalURL ( ) ; timestamp = item . getOriginalTimestamp ( ) ; } } count ++ ; if ( verbose && count % 10000 == 0 ) { System . out . println ( "Indexed " + count + ", unresolved " + unresolved + " - Last URL " + "from " + item . getTimestamp ( ) ) ; } if ( url . contains ( "\"" ) ) { throw new IllegalStateException ( "Double quotes in URLs should always be properly escaped. " + item . getURL ( ) ) ; } Document doc = new Document ( ) ; doc . add ( new Field ( URL . name ( ) , url , ( indexURL ? ftIndexed : ftNotIndexed ) ) ) ; if ( includeCanonicalizedURL ) { doc . add ( new Field ( URL_CANONICALIZED . name ( ) , canonicalizer . canonicalize ( item . getURL ( ) ) , ( indexURL ? ftIndexed : ftNotIndexed ) ) ) ; } String digest = item . getContentDigest ( ) ; if ( ! indexDigestScheme && digest . lastIndexOf ( ":" ) >= 0 ) { digest = digest . substring ( digest . lastIndexOf ( ":" ) + 1 ) ; } doc . add ( new Field ( DIGEST . name ( ) , digest , ftIndexed ) ) ; doc . add ( new Field ( DATE . name ( ) , timestamp , ftNotIndexed ) ) ; if ( includeEtag && item . getEtag ( ) != null ) { doc . add ( new Field ( ETAG . name ( ) , item . getEtag ( ) , ftNotIndexed ) ) ; } if ( indexURL ) { BooleanQuery q = new BooleanQuery ( ) ; q . add ( new TermQuery ( new Term ( URL . name ( ) , url ) ) , Occur . MUST ) ; q . add ( new TermQuery ( new Term ( DIGEST . name ( ) , item . getContentDigest ( ) ) ) , Occur . MUST ) ; index . deleteDocuments ( q ) ; index . addDocument ( doc ) ; } else { index . updateDocument ( new Term ( DIGEST . name ( ) ) , doc ) ; } } System . out . println ( "Indexed " + count + " items (unresolved " + unresolved + ", skipped " + skipped + ")" ) ; return count ; }
Writes the contents of a {@link CrawlDataIterator} to this index.
Writes the contents of a { @link CrawlDataIterator } to this index .
// Define field types for indexed and non indexed fields. No fields are tokenized
 // Only index items that were crawled without issues
 // TODO: Consider widening to 4XXs at least
 // Can't index without information about the original capture 
 // Ok, we wish to index this URL/Digest
 // TODO: Consider other sanity checks and also option to just log and continue on failed
 //       sanity checks.
 // Add URL to document.
 // Add digest to document
 // The prefix will be terminated by a : which is immediately 
 // followed by the actual digest
 // add timestamp
 // Include etag?
 // Delete any URL+Digest matches from index first
