question: private void handleCrawlDelay ( ParseState state , RobotToken token ) { if ( state . isSkipAgents ( ) ) { return ; } state . setFinishedAgentFields ( true ) ; if ( ! state . isAddingRules ( ) ) { return ; } String delayString = token . getData ( ) ; if ( delayString . length ( ) > 0 ) { try { if ( delayString . indexOf ( '.' ) != - 1 ) { double delayValue = Double . parseDouble ( delayString ) * 1000.0 ; state . setCrawlDelay ( Math . round ( delayValue ) ) ; } else { long delayValue = Integer . parseInt ( delayString ) * 1000L ; state . setCrawlDelay ( delayValue ) ; } } catch ( Exception e ) { reportWarning ( "Error parsing robots rules - can't decode crawl delay: " + delayString , state . getUrl ( ) ) ; } } }
target:  Handle the crawl-delay: directive