src/main/java/webcrawley/crawler/urlchecker/UrlChecker.java
private static boolean validateUrl(final Url url, final CrawlerConfiguration crawlerConfiguration) {
        if (url == null) {
            return false;
        }

        final String completeUrl = url.getCompleteUrl();
        boolean isQualified = false;

        try {

            final Document doc = Jsoup.connect(completeUrl).timeout(crawlerConfiguration.getFetchTimeout() * 1000).get();

            for (Map.Entry<String, Qualifier> configurationEntry : crawlerConfiguration.getCrawlerQualifiers().entrySet()) {
                //Applying each CSS query to the fetched document
                final Elements elements = doc.select(configurationEntry.getKey());

                final String[] keywords = configurationEntry.getValue().getKeywords();
                final String[] attributes = configurationEntry.getValue().getAttributes();

                for (int i = 0; i < elements.size() && !isQualified; i++) {
                    final Element element = elements.get(i);

                    if (attributes.length == 0) {
                        //If attributes to search through are not specified
                        //search for the keywords in the element's inner html
                        isQualified = InnerHtmlElementChecker.isValid(element, keywords);
                    } else {
                        //Search for the keywords through the element's attributes values
                        isQualified = AttributeElementChecker.isValid(element, keywords, attributes);
                    }
                }

                if (isQualified) {
                    break;
                }
            }
        } catch (SocketTimeoutException e) {
            LOGGER.warning("Web page timeout " + completeUrl);
        } catch (UnknownHostException e) {
            LOGGER.warning("Unknown host " + completeUrl);
        } catch (Exception e) {
            LOGGER.warning("Unhandled exception " + completeUrl);
            e.printStackTrace();
        }

        return isQualified;
    }
/**
     * Checks if a URL is valid by applying the crawler configuration.
     *
     * @param url                  URL to check.
     * @param crawlerConfiguration Configuration to qualify a web page.
     * @return True if the URL is qualified, otherwise returns false.
     */
Checks if a URL is valid by applying the crawler configuration.
@param url                  URL to check.
@param crawlerConfiguration Configuration to qualify a web page.
@return True if the URL is qualified, otherwise returns false.
Checks if a URL is valid by applying the crawler configuration . @param url URL to check . @param crawlerConfiguration Configuration to qualify a web page . @return True if the URL is qualified otherwise returns false .
private static boolean validateUrl(final Url url, final CrawlerConfiguration crawlerConfiguration) {
        if (url == null) {
            return false;
        }
        final String completeUrl = url.getCompleteUrl();
        boolean isQualified = false;
        try {
            final Document doc = Jsoup.connect(completeUrl).timeout(crawlerConfiguration.getFetchTimeout() * 1000).get();
            for (Map.Entry<String, Qualifier> configurationEntry : crawlerConfiguration.getCrawlerQualifiers().entrySet()) {
                final Elements elements = doc.select(configurationEntry.getKey());
                final String[] keywords = configurationEntry.getValue().getKeywords();
                final String[] attributes = configurationEntry.getValue().getAttributes();
                for (int i = 0; i < elements.size() && !isQualified; i++) {
                    final Element element = elements.get(i);
                    if (attributes.length == 0) {
                        isQualified = InnerHtmlElementChecker.isValid(element, keywords);
                    } else {
                        isQualified = AttributeElementChecker.isValid(element, keywords, attributes);
                    }
                }
                if (isQualified) {
                    break;
                }
            }
        } catch (SocketTimeoutException e) {
            LOGGER.warning("Web page timeout " + completeUrl);
        } catch (UnknownHostException e) {
            LOGGER.warning("Unknown host " + completeUrl);
        } catch (Exception e) {
            LOGGER.warning("Unhandled exception " + completeUrl);
            e.printStackTrace();
        }
        return isQualified;
    }
private static boolean validateUrl ( final Url url , final CrawlerConfiguration crawlerConfiguration ) { if ( url == null ) { return false ; } final String completeUrl = url . getCompleteUrl ( ) ; boolean isQualified = false ; try { final Document doc = Jsoup . connect ( completeUrl ) . timeout ( crawlerConfiguration . getFetchTimeout ( ) * 1000 ) . get ( ) ; for ( Map . Entry < String , Qualifier > configurationEntry : crawlerConfiguration . getCrawlerQualifiers ( ) . entrySet ( ) ) { final Elements elements = doc . select ( configurationEntry . getKey ( ) ) ; final String [ ] keywords = configurationEntry . getValue ( ) . getKeywords ( ) ; final String [ ] attributes = configurationEntry . getValue ( ) . getAttributes ( ) ; for ( int i = 0 ; i < elements . size ( ) && ! isQualified ; i ++ ) { final Element element = elements . get ( i ) ; if ( attributes . length == 0 ) { isQualified = InnerHtmlElementChecker . isValid ( element , keywords ) ; } else { isQualified = AttributeElementChecker . isValid ( element , keywords , attributes ) ; } } if ( isQualified ) { break ; } } } catch ( SocketTimeoutException e ) { LOGGER . warning ( "Web page timeout " + completeUrl ) ; } catch ( UnknownHostException e ) { LOGGER . warning ( "Unknown host " + completeUrl ) ; } catch ( Exception e ) { LOGGER . warning ( "Unhandled exception " + completeUrl ) ; e . printStackTrace ( ) ; } return isQualified ; }
Checks if a URL is valid by applying the crawler configuration.
Checks if a URL is valid by applying the crawler configuration .
//Applying each CSS query to the fetched document //If attributes to search through are not specified //search for the keywords in the element's inner html //Search for the keywords through the element's attributes values