private void verifyUrls(Set<CrawlerURL> allUrls, Set<HTMLPageResponse> verifiedUrls,
      Set<HTMLPageResponse> nonWorkingUrls, Map<String, String> requestHeaders) {

    Set<CrawlerURL> urlsThatNeedsVerification = new LinkedHashSet<CrawlerURL>(allUrls);

    urlsThatNeedsVerification.removeAll(verifiedUrls);

    final Set<Callable<HTMLPageResponse>> tasks =
        new HashSet<Callable<HTMLPageResponse>>(urlsThatNeedsVerification.size());

    for (CrawlerURL testURL : urlsThatNeedsVerification) {
      tasks.add(new HTMLPageResponseCallable(testURL, responseFetcher, true, requestHeaders, false));
    }

    try {
      // wait for all urls to verify
      List<Future<HTMLPageResponse>> responses = service.invokeAll(tasks);

      for (Future<HTMLPageResponse> future : responses) {
        if (!future.isCancelled()) {
          HTMLPageResponse response = future.get();
          if (response.getResponseCode() == HttpStatus.SC_OK
              && response.getResponseType().indexOf("html") > 0) {
            // remove, way of catching interrupted / execution e
            urlsThatNeedsVerification.remove(response.getPageUrl());
            verifiedUrls.add(response);
          } else if (response.getResponseCode() == HttpStatus.SC_OK) {
            // it is not HTML
            urlsThatNeedsVerification.remove(response.getPageUrl());
          } else {
            nonWorkingUrls.add(response);
          }
        }
      }

    } catch (InterruptedException e1) {
      // TODO add some logging
      e1.printStackTrace();
    } catch (ExecutionException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }

    // TODO: We can have a delta here if the exception occur

  }
private void verifyUrls ( Set < CrawlerURL > allUrls , Set < HTMLPageResponse > verifiedUrls , Set < HTMLPageResponse > nonWorkingUrls , Map < String , String > requestHeaders ) { Set < CrawlerURL > urlsThatNeedsVerification = new LinkedHashSet < CrawlerURL > ( allUrls ) ; urlsThatNeedsVerification . removeAll ( verifiedUrls ) ; final Set < Callable < HTMLPageResponse > > tasks = new HashSet < Callable < HTMLPageResponse > > ( urlsThatNeedsVerification . size ( ) ) ; for ( CrawlerURL testURL : urlsThatNeedsVerification ) { tasks . add ( new HTMLPageResponseCallable ( testURL , responseFetcher , true , requestHeaders , false ) ) ; } try { List < Future < HTMLPageResponse > > responses = service . invokeAll ( tasks ) ; for ( Future < HTMLPageResponse > future : responses ) { if ( ! future . isCancelled ( ) ) { HTMLPageResponse response = future . get ( ) ; if ( response . getResponseCode ( ) == HttpStatus . SC_OK && response . getResponseType ( ) . indexOf ( "html" ) > 0 ) { urlsThatNeedsVerification . remove ( response . getPageUrl ( ) ) ; verifiedUrls . add ( response ) ; } else if ( response . getResponseCode ( ) == HttpStatus . SC_OK ) { urlsThatNeedsVerification . remove ( response . getPageUrl ( ) ) ; } else { nonWorkingUrls . add ( response ) ; } } } } catch ( InterruptedException e1 ) { e1 . printStackTrace ( ) ; } catch ( ExecutionException e ) { e . printStackTrace ( ) ; } }
Verify that all urls in allUrls returns 200. If not, they will be removed from that set and
instead added to the nonworking list.

@param allUrls all the links that has been fetched
@param nonWorkingUrls links that are not working