src/main/java/com/findupon/commons/building/ProductUtils.java
/**
	 * @return {@code true} if the product exists, {@code} false otherwise.
	 */
@return  true if the product exists,  false otherwise.
@return true if the product exists false otherwise .
@SuppressWarnings("unchecked")
	private boolean refreshAggregates(Product product) {
		Objects.requireNonNull(product, "Cannot refresh aggregates on null product!");
		if(product instanceof Automobile) {
			String vin = getAggregationColumnValue(product);
			if(vin == null) {
				return false; 
			}
			List<Automobile> all = automobileDao.findByVin(vin);
			boolean productStillExists = all.stream().anyMatch(a -> {
				if(product.getId() != null) {
					return Objects.equals(product.getId(), a.getId());
				} else {
					return Objects.equals(product.getUrl(), a.getUrl());
				}
			});
			if(!productStillExists) {
				String attr = product.getId() != null ? String.valueOf(product.getId()) : product.getUrl();
				logger.debug("[ProductUtils] - {} [{}] removed, most likely aggregation refresh from another worker. Not refreshing aggregates.",
						product.getClass().getSimpleName(), attr);
				return false;
			}
			List<Automobile> aggregates = all.stream()
					.filter(a -> {
						if(product.getId() != null) {
							return !Objects.equals(product.getId(), a.getId());
						} else {
							return !Objects.equals(product.getUrl(), a.getUrl());
						}
					}).collect(Collectors.toList());
			if(aggregates.isEmpty()) {
				logger.debug("[ProductUtils] - No aggregates to refresh for VIN [{}]", vin);
				return true;
			}
			logger.debug("[ProductUtils] - Refreshing [{}] aggregate product{} for VIN [{}]", aggregates.size(), aggregates.size() > 1 ? "s" : "", vin);
			int numRemoved = 0, numNotTouched = 0;
			List<Automobile> refreshed = new ArrayList<>();
			for(Automobile automobile : aggregates) {
				if(automobile.getVisitedDate() != null && automobile.getVisitedDate().after(refreshThreshold())) {
					numNotTouched++;
					continue;
				}
				if(automobile.getDataSourceId() == null) {
					logger.error("[ProductUtils] - Product missing data source ID! Product ID [{}]", automobile.getId());
				}
				Optional<DataSource> opt = dataSourceRepo.findById(automobile.getDataSourceId());
				DataSource dataSource;
				if(opt.isPresent()) {
					dataSource = opt.get();
					if(dataSource.getDataSourceType() == null) {
						logger.error("[ProductUtils] - Null data source type not allowed. Data source ID: [{}] URL: [{}]", dataSource.getId(), dataSource.getUrl());
						continue;
					}
				} else {
					logger.error("[ProductUtils] - No data source found by ID [{}]. Product ID [{}]", automobile.getDataSourceId(), automobile.getId());
					continue;
				}
				switch(dataSource.getDataSourceType()) {
					case LISTING:
						if(dataSource.getIndexOnly()) {
							logger.trace("[ProductUtils] - You need to implement a generic builder for index only source [{}]", dataSource.getUrl());
							numNotTouched++;
							continue;
						}
						try {
							Class<?> clazz = Class.forName(dataSource.getBotClass());
							if(ListingAutomobileBot.class.isAssignableFrom(clazz)) {
								Object bot = clazz.getConstructor().newInstance();
								SpringUtils.autowire(bot);
								Method method = clazz.getDeclaredMethod("buildProduct", String.class);
								method.setAccessible(true);
								BuiltProduct refresh = (BuiltProduct)method.invoke(bot, automobile.getUrl());
								if(refresh.isMarkedForRemoval()) {
									automobileDao.delete(automobile);
									numRemoved++;
								} else {
									mergeProductFromRefresh(refresh.getProduct(), automobile, dataSource);
									refreshed.add((Automobile)refresh.getProduct());
								}
							} else {
								throw new IllegalAccessException("Bot class for data source is not of type ListingAutomobileBot");
							}
						} catch(Exception e) {
							logger.error("[ProductUtils] - Error running refresh for listing bot [{}]", ExceptionUtils.getRootCauseMessage(e));
						}
						break;
					case GENERIC:
						Document document = ConnectionAgent.INSTANCE.download(automobile.getUrl(), dataSource).getDocument();
						if(document == null) {
							automobileDao.delete(automobile);
							numRemoved++;
							continue;
						}
						Automobile refresh = automotiveGatherer.buildProduct(document);
						if(refresh != null) {
							refresh.setDataSourceId(dataSource.getId()); 
							refresh.setDealerUrl(dataSource.getUrl());
						}
						if(basicInvalidator(refresh)) {
							automobileDao.delete(automobile);
							numRemoved++;
							continue;
						}
						mergeProductFromRefresh(refresh, automobile, dataSource);
						refreshed.add(refresh);
						break;
					case IMPORT_PROCESS:
						break;
					default:
						logger.error("[ProductUtils] - Invalid data source type found [{}]", dataSource.getDataSourceType());
						break;
				}
			}
			automobileDao.saveAll(refreshed);
			logger.debug(ConsoleColors.green("[ProductUtils] - Refreshed: [{}] Removed: [{}] Not touched: [{}] for VIN [{}]"),
					refreshed.size(), numRemoved, numNotTouched, vin);
		}
		if(product instanceof RealEstate) {
			logger.trace("[ProductUtils] - TODO: real estate refresh aggregates");
		}
		if(product instanceof Watercraft) {
			logger.trace("[ProductUtils] - TODO: watercraft refresh aggregates");
		}
		if(product instanceof Aircraft) {
			logger.trace("[ProductUtils] - TODO: aircraft refresh aggregates");
		}
		return true;
	}
@ SuppressWarnings ( "unchecked" ) private boolean refreshAggregates ( Product product ) { Objects . requireNonNull ( product , "Cannot refresh aggregates on null product!" ) ; if ( product instanceof Automobile ) { String vin = getAggregationColumnValue ( product ) ; if ( vin == null ) { return false ; } List < Automobile > all = automobileDao . findByVin ( vin ) ; boolean productStillExists = all . stream ( ) . anyMatch ( a -> { if ( product . getId ( ) != null ) { return Objects . equals ( product . getId ( ) , a . getId ( ) ) ; } else { return Objects . equals ( product . getUrl ( ) , a . getUrl ( ) ) ; } } ) ; if ( ! productStillExists ) { String attr = product . getId ( ) != null ? String . valueOf ( product . getId ( ) ) : product . getUrl ( ) ; logger . debug ( "[ProductUtils] - {} [{}] removed, most likely aggregation refresh from another worker. Not refreshing aggregates." , product . getClass ( ) . getSimpleName ( ) , attr ) ; return false ; } List < Automobile > aggregates = all . stream ( ) . filter ( a -> { if ( product . getId ( ) != null ) { return ! Objects . equals ( product . getId ( ) , a . getId ( ) ) ; } else { return ! Objects . equals ( product . getUrl ( ) , a . getUrl ( ) ) ; } } ) . collect ( Collectors . toList ( ) ) ; if ( aggregates . isEmpty ( ) ) { logger . debug ( "[ProductUtils] - No aggregates to refresh for VIN [{}]" , vin ) ; return true ; } logger . debug ( "[ProductUtils] - Refreshing [{}] aggregate product{} for VIN [{}]" , aggregates . size ( ) , aggregates . size ( ) > 1 ? "s" : "" , vin ) ; int numRemoved = 0 , numNotTouched = 0 ; List < Automobile > refreshed = new ArrayList < > ( ) ; for ( Automobile automobile : aggregates ) { if ( automobile . getVisitedDate ( ) != null && automobile . getVisitedDate ( ) . after ( refreshThreshold ( ) ) ) { numNotTouched ++ ; continue ; } if ( automobile . getDataSourceId ( ) == null ) { logger . error ( "[ProductUtils] - Product missing data source ID! Product ID [{}]" , automobile . getId ( ) ) ; } Optional < DataSource > opt = dataSourceRepo . findById ( automobile . getDataSourceId ( ) ) ; DataSource dataSource ; if ( opt . isPresent ( ) ) { dataSource = opt . get ( ) ; if ( dataSource . getDataSourceType ( ) == null ) { logger . error ( "[ProductUtils] - Null data source type not allowed. Data source ID: [{}] URL: [{}]" , dataSource . getId ( ) , dataSource . getUrl ( ) ) ; continue ; } } else { logger . error ( "[ProductUtils] - No data source found by ID [{}]. Product ID [{}]" , automobile . getDataSourceId ( ) , automobile . getId ( ) ) ; continue ; } switch ( dataSource . getDataSourceType ( ) ) { case LISTING : if ( dataSource . getIndexOnly ( ) ) { logger . trace ( "[ProductUtils] - You need to implement a generic builder for index only source [{}]" , dataSource . getUrl ( ) ) ; numNotTouched ++ ; continue ; } try { Class < ? > clazz = Class . forName ( dataSource . getBotClass ( ) ) ; if ( ListingAutomobileBot . class . isAssignableFrom ( clazz ) ) { Object bot = clazz . getConstructor ( ) . newInstance ( ) ; SpringUtils . autowire ( bot ) ; Method method = clazz . getDeclaredMethod ( "buildProduct" , String . class ) ; method . setAccessible ( true ) ; BuiltProduct refresh = ( BuiltProduct ) method . invoke ( bot , automobile . getUrl ( ) ) ; if ( refresh . isMarkedForRemoval ( ) ) { automobileDao . delete ( automobile ) ; numRemoved ++ ; } else { mergeProductFromRefresh ( refresh . getProduct ( ) , automobile , dataSource ) ; refreshed . add ( ( Automobile ) refresh . getProduct ( ) ) ; } } else { throw new IllegalAccessException ( "Bot class for data source is not of type ListingAutomobileBot" ) ; } } catch ( Exception e ) { logger . error ( "[ProductUtils] - Error running refresh for listing bot [{}]" , ExceptionUtils . getRootCauseMessage ( e ) ) ; } break ; case GENERIC : Document document = ConnectionAgent . INSTANCE . download ( automobile . getUrl ( ) , dataSource ) . getDocument ( ) ; if ( document == null ) { automobileDao . delete ( automobile ) ; numRemoved ++ ; continue ; } Automobile refresh = automotiveGatherer . buildProduct ( document ) ; if ( refresh != null ) { refresh . setDataSourceId ( dataSource . getId ( ) ) ; refresh . setDealerUrl ( dataSource . getUrl ( ) ) ; } if ( basicInvalidator ( refresh ) ) { automobileDao . delete ( automobile ) ; numRemoved ++ ; continue ; } mergeProductFromRefresh ( refresh , automobile , dataSource ) ; refreshed . add ( refresh ) ; break ; case IMPORT_PROCESS : break ; default : logger . error ( "[ProductUtils] - Invalid data source type found [{}]" , dataSource . getDataSourceType ( ) ) ; break ; } } automobileDao . saveAll ( refreshed ) ; logger . debug ( ConsoleColors . green ( "[ProductUtils] - Refreshed: [{}] Removed: [{}] Not touched: [{}] for VIN [{}]" ) , refreshed . size ( ) , numRemoved , numNotTouched , vin ) ; } if ( product instanceof RealEstate ) { logger . trace ( "[ProductUtils] - TODO: real estate refresh aggregates" ) ; } if ( product instanceof Watercraft ) { logger . trace ( "[ProductUtils] - TODO: watercraft refresh aggregates" ) ; } if ( product instanceof Aircraft ) { logger . trace ( "[ProductUtils] - TODO: aircraft refresh aggregates" ) ; } return true ; }
@return {@code true} if the product exists, {@code} false otherwise.
@return { @code true } if the product exists { @code } false otherwise .
// already reported // don't refresh the product passed in. duh. // TODO: // needed for basic invalid check to handle deletion