src/main/java/com/findupon/commons/bot/ListingBot.java
/**
	 * @param baseUrl                  The url to start indexing from.
	 * @param pageQueryParam           The next page url query param, i.e. "page" to produce "?page=x" or "&page=x"
	 * @param portletCssSelector       The css to select all product portlets.
	 * @param noMatchingResultsMessage Many listing bots will still display related results if your search query is too specific,
	 *                                 this will be your site-specific message to short the builder.
	 * @param recursiveUrlReducer      Are there too many results per page? If so, recurse and reduce. Limited to once.
	 *                                 Example: a site will show 5000 results but their serp only traverses to 1000. Return a
	 *                                 new set of urls that have narrowed down the search so each one returns less than 1000.
	 * @param portletBuilder           Builds an product from the portlet selected.
	 * @param hasNextPage              Test the element for a next page condition and enforce a max page value.
	 * @return The number of built products.
	 */
@param baseUrl                  The url to start indexing from.
@param pageQueryParam           The next page url query param, i.e.
@param baseUrl The url to start indexing from . @param pageQueryParam The next page url query param i . e .
protected int abstractSerpBuilder(String baseUrl, String pageQueryParam, String portletCssSelector,
	                                  String noMatchingResultsMessage, final int maxPageNumber,
	                                  BiFunction<Document, String, Set<String>> recursiveUrlReducer,
	                                  Function<Element, P> portletBuilder,
	                                  Predicate<Document> hasNextPage) {
		AtomicInteger builtProduct = new AtomicInteger();
		AtomicInteger pageNumber = new AtomicInteger();
		Document firstPage = download(baseUrl);
		Document currentPage;
		if(firstPage == null) {
			logger.warn(logPre() + "Null document returned at first page, returning. Serp URL [{}]", baseUrl);
			return builtProduct.get();
		}
		if(StringUtils.containsIgnoreCase(firstPage.html(), noMatchingResultsMessage)) {
			logger.debug(logPre() + "No matching results message found at first page URL [{}]", firstPage.location());
			return builtProduct.get();
		}
		Set<String> additionalUrls = recursiveUrlReducer.apply(firstPage, baseUrl);
		if(!additionalUrls.isEmpty()) {
			LongAdder recursiveBuilt = new LongAdder();
			additionalUrls.forEach(u ->
					recursiveBuilt.add(abstractSerpBuilder(baseUrl, pageQueryParam, portletCssSelector, noMatchingResultsMessage,
							maxPageNumber, (r, s) -> new HashSet<>(), portletBuilder, hasNextPage)));
			logger.debug(logPre() + "Recursive built [{}] from base url [{}]", recursiveBuilt, baseUrl);
			return recursiveBuilt.intValue();
		}
		do {
			pageNumber.incrementAndGet();
			if(firstPage != null) {
				currentPage = firstPage;
				firstPage = null;
			} else {
				String pageUrl = ScoutServices.setQueryParamValue(baseUrl, pageQueryParam, pageNumber.toString());
				currentPage = download(pageUrl);
			}
			if(sleep()) {
				break;
			}
			if(currentPage == null) {
				logger.warn(logPre() + "Null document returned at page [{}], returning. Serp URL [{}]", pageNumber.get(), baseUrl);
				return builtProduct.get();
			}
			logger.debug(logPre() + "Building from serp at [{}]", currentPage.location());
			if(StringUtils.containsIgnoreCase(currentPage.html(), noMatchingResultsMessage)) {
				logger.debug(logPre() + "No matching results message found at serp URL [{}]", currentPage.location());
				return builtProduct.get();
			}
			for(Element serpElement : currentPage.select(portletCssSelector)) {
				try {
					P product = portletBuilder.apply(serpElement);
					if(product != null) {
						validateSetMetaAndAdd(product);
						builtProduct.incrementAndGet();
					}
				} catch(Exception e) {
					logger.error(logPre() + "Error building product from serp URL [{}]", baseUrl, e);
				}
			}
			if(products.size() >= ProductDao.productWriteThreshold) {
				logger.debug(logPre() + "Persisting [{}] {}s to the db (over threshold)", currentClassName, products.size());
				persistAndClear();
			}
		} while(hasNextPage.test(currentPage) && pageNumber.get() <= maxPageNumber);
		return builtProduct.get();
	}
protected int abstractSerpBuilder ( String baseUrl , String pageQueryParam , String portletCssSelector , String noMatchingResultsMessage , final int maxPageNumber , BiFunction < Document , String , Set < String > > recursiveUrlReducer , Function < Element , P > portletBuilder , Predicate < Document > hasNextPage ) { AtomicInteger builtProduct = new AtomicInteger ( ) ; AtomicInteger pageNumber = new AtomicInteger ( ) ; Document firstPage = download ( baseUrl ) ; Document currentPage ; if ( firstPage == null ) { logger . warn ( logPre ( ) + "Null document returned at first page, returning. Serp URL [{}]" , baseUrl ) ; return builtProduct . get ( ) ; } if ( StringUtils . containsIgnoreCase ( firstPage . html ( ) , noMatchingResultsMessage ) ) { logger . debug ( logPre ( ) + "No matching results message found at first page URL [{}]" , firstPage . location ( ) ) ; return builtProduct . get ( ) ; } Set < String > additionalUrls = recursiveUrlReducer . apply ( firstPage , baseUrl ) ; if ( ! additionalUrls . isEmpty ( ) ) { LongAdder recursiveBuilt = new LongAdder ( ) ; additionalUrls . forEach ( u -> recursiveBuilt . add ( abstractSerpBuilder ( baseUrl , pageQueryParam , portletCssSelector , noMatchingResultsMessage , maxPageNumber , ( r , s ) -> new HashSet < > ( ) , portletBuilder , hasNextPage ) ) ) ; logger . debug ( logPre ( ) + "Recursive built [{}] from base url [{}]" , recursiveBuilt , baseUrl ) ; return recursiveBuilt . intValue ( ) ; } do { pageNumber . incrementAndGet ( ) ; if ( firstPage != null ) { currentPage = firstPage ; firstPage = null ; } else { String pageUrl = ScoutServices . setQueryParamValue ( baseUrl , pageQueryParam , pageNumber . toString ( ) ) ; currentPage = download ( pageUrl ) ; } if ( sleep ( ) ) { break ; } if ( currentPage == null ) { logger . warn ( logPre ( ) + "Null document returned at page [{}], returning. Serp URL [{}]" , pageNumber . get ( ) , baseUrl ) ; return builtProduct . get ( ) ; } logger . debug ( logPre ( ) + "Building from serp at [{}]" , currentPage . location ( ) ) ; if ( StringUtils . containsIgnoreCase ( currentPage . html ( ) , noMatchingResultsMessage ) ) { logger . debug ( logPre ( ) + "No matching results message found at serp URL [{}]" , currentPage . location ( ) ) ; return builtProduct . get ( ) ; } for ( Element serpElement : currentPage . select ( portletCssSelector ) ) { try { P product = portletBuilder . apply ( serpElement ) ; if ( product != null ) { validateSetMetaAndAdd ( product ) ; builtProduct . incrementAndGet ( ) ; } } catch ( Exception e ) { logger . error ( logPre ( ) + "Error building product from serp URL [{}]" , baseUrl , e ) ; } } if ( products . size ( ) >= ProductDao . productWriteThreshold ) { logger . debug ( logPre ( ) + "Persisting [{}] {}s to the db (over threshold)" , currentClassName , products . size ( ) ) ; persistAndClear ( ) ; } } while ( hasNextPage . test ( currentPage ) && pageNumber . get ( ) <= maxPageNumber ) ; return builtProduct . get ( ) ; }
@param baseUrl                  The url to start indexing from.
@param baseUrl The url to start indexing from .