private void handleCrawlDelay(ParseState state,RobotToken token){
  if (state.isSkipAgents()) {
    return;
  }
  state.setFinishedAgentFields(true);
  if (!state.isAddingRules()) {
    return;
  }
  String delayString=token.getData();
  if (delayString.length() > 0) {
    try {
      if (delayString.indexOf('.') != -1) {
        double delayValue=Double.parseDouble(delayString) * 1000.0;
        state.setCrawlDelay(Math.round(delayValue));
      }
 else {
        long delayValue=Integer.parseInt(delayString) * 1000L;
        state.setCrawlDelay(delayValue);
      }
    }
 catch (    Exception e) {
      reportWarning("Error parsing robots rules - can't decode crawl delay: " + delayString,state.getUrl());
    }
  }
}

private void handleCrawlDelay ( ParseState state , RobotToken token ) { if ( state . isSkipAgents ( ) ) { return ; } state . setFinishedAgentFields ( true ) ; if ( ! state . isAddingRules ( ) ) { return ; } String delayString = token . getData ( ) ; if ( delayString . length ( ) > 0 ) { try { if ( delayString . indexOf ( '.' ) != - 1 ) { double delayValue = Double . parseDouble ( delayString ) * 1000.0 ; state . setCrawlDelay ( Math . round ( delayValue ) ) ; } else { long delayValue = Integer . parseInt ( delayString ) * 1000L ; state . setCrawlDelay ( delayValue ) ; } } catch ( Exception e ) { reportWarning ( "Error parsing robots rules - can't decode crawl delay: " + delayString , state . getUrl ( ) ) ; } } }
handle the crawl - delay : directive
