deduplicator-indexer/src/main/java/is/landsbokasafn/deduplicator/indexer/CrawlLogIterator.java
/** 
     * Parse a line in the crawl log.
     * <p>
     * Override this method to change how individual crawl log
     * items are processed and accepted/rejected.  This method is called from
     * within the loop in prepareNext().
     *
     * @param line A line from the crawl log.  Must not be null.
     * @return A {@link CrawlDataItem} if the next line in the crawl log yielded 
     *         a usable item, null otherwise.
     */
Parse a line in the crawl log.

Override this method to change how individual crawl log
items are processed and accepted/rejected.  This method is called from
within the loop in prepareNext().

@param line A line from the crawl log.  Must not be null.
@return A  CrawlDataItem if the next line in the crawl log yielded
a usable item, null otherwise.
Parse a line in the crawl log . Override this method to change how individual crawl log items are processed and accepted / rejected . This method is called from within the loop in prepareNext () . @param line A line from the crawl log . Must not be null . @return A CrawlDataItem if the next line in the crawl log yielded a usable item null otherwise .
protected CrawlDataItem parseLine(String line) throws IOException {
        if (line != null && line.length() > 42) {
            String[] lineParts = line.split("\\s+",13);
            if(lineParts.length<10){
                log.debug("Ignoring malformed line, lineParts are fewer then 10 in line:\n" + line);
                return null;
            }
            int status = Integer.parseInt(lineParts[1]);
            if (status<=0) {
            	log.debug("Ignoring failed fetch based on status code. Line:\n" + line);
            	return null;
            }
            String url = lineParts[3];
            String mime = lineParts[6];
            String timestamp;
            try {
            	String fetchBegan = lineParts[8];
           		fetchBegan = fetchBegan.substring(0,fetchBegan.indexOf("+")); 
				timestamp = DateUtils.getLog14Date(DateUtils.parse17DigitDate(fetchBegan));
			} catch (ParseException e1) {
				throw new IOException(e1);
			}
            String digest = lineParts[9];
            boolean revisit = false;
        	if (lineParts[11].matches(revisitMatchingRegex)) {
        		revisit=true;
        	}
            String originalURL = null;
            String originalTimestamp = null;
        	String revisitProfile = null;
            if(revisit && lineParts.length==13){
            	JSONObject extraInfo = new JSONObject(lineParts[12]);
            	originalURL = extraInfo.getString(EXTRA_REVISIT_URI);
            	originalTimestamp = extraInfo.getString(EXTRA_REVISIT_DATE);
            	revisitProfile=extraInfo.getString(EXTRA_REVISIT_PROFILE);
            }
            CrawlDataItem cdi = new CrawlDataItem();
            cdi.setURL(url);
            cdi.setTimestamp(timestamp);
            cdi.setStatusCode(status);
            cdi.setContentDigest(digest);
            cdi.setMimeType(mime);
            cdi.setRevisit(revisit);
            cdi.setOriginalURL(originalURL);
            cdi.setOriginalTimestamp(originalTimestamp);
            cdi.setRevisitProfile(revisitProfile);
            return cdi;
        } 
        return null;
    }
protected CrawlDataItem parseLine ( String line ) throws IOException { if ( line != null && line . length ( ) > 42 ) { String [ ] lineParts = line . split ( "\\s+" , 13 ) ; if ( lineParts . length < 10 ) { log . debug ( "Ignoring malformed line, lineParts are fewer then 10 in line:\n" + line ) ; return null ; } int status = Integer . parseInt ( lineParts [ 1 ] ) ; if ( status <= 0 ) { log . debug ( "Ignoring failed fetch based on status code. Line:\n" + line ) ; return null ; } String url = lineParts [ 3 ] ; String mime = lineParts [ 6 ] ; String timestamp ; try { String fetchBegan = lineParts [ 8 ] ; fetchBegan = fetchBegan . substring ( 0 , fetchBegan . indexOf ( "+" ) ) ; timestamp = DateUtils . getLog14Date ( DateUtils . parse17DigitDate ( fetchBegan ) ) ; } catch ( ParseException e1 ) { throw new IOException ( e1 ) ; } String digest = lineParts [ 9 ] ; boolean revisit = false ; if ( lineParts [ 11 ] . matches ( revisitMatchingRegex ) ) { revisit = true ; } String originalURL = null ; String originalTimestamp = null ; String revisitProfile = null ; if ( revisit && lineParts . length == 13 ) { JSONObject extraInfo = new JSONObject ( lineParts [ 12 ] ) ; originalURL = extraInfo . getString ( EXTRA_REVISIT_URI ) ; originalTimestamp = extraInfo . getString ( EXTRA_REVISIT_DATE ) ; revisitProfile = extraInfo . getString ( EXTRA_REVISIT_PROFILE ) ; } CrawlDataItem cdi = new CrawlDataItem ( ) ; cdi . setURL ( url ) ; cdi . setTimestamp ( timestamp ) ; cdi . setStatusCode ( status ) ; cdi . setContentDigest ( digest ) ; cdi . setMimeType ( mime ) ; cdi . setRevisit ( revisit ) ; cdi . setOriginalURL ( originalURL ) ; cdi . setOriginalTimestamp ( originalTimestamp ) ; cdi . setRevisitProfile ( revisitProfile ) ; return cdi ; } return null ; }
Parse a line in the crawl log.
Parse a line in the crawl log .
// Split the line up by whitespaces.
 // Limit to 13 parts. Annotations containing spaces will break this.
 // Index 0: Timestamp, this is the time of log writing and not of interest
 // Index 1: status return code 
 // Index 2: File size (ignore) 
 // Index 3: URL
 // Index 4: Hop path (ignore)
 // Index 5: Parent URL (ignore)
 // Index 6: Mime type
 // Index 7: ToeThread number (ignore)
 // Index 8: ArcTimeAndDuration, this is the time when fetch begin, followed by a plus 
 //          sign and the number of milliseconds the fetch took.
 // Convert from the crawl log dateformat to w3c-iso8601
 // Ignore + fetch duration
 // Index 9: Digest
 // Index 10: Source tag (ignore)
 // Index 11: Annotations 
 // Index 12: Extra info in JSON format. May be missing
 // Got a valid item.
