src/main/java/com/findupon/commons/netops/ConnectionAgent.java
/**
	 * @param proxyMode PUBLIC will use one of our known IPs.
	 * @param agentMode PUBLIC will use our publicly facing user agent.
	 * @implNote MAKE SURE YOU KNOW WHAT YOU ARE DOING AND UNDERSTAND THE EXPOSURE OF USING THIS METHOD AND ITS OVERLOADED CALLERS
	 */
@param proxyMode PUBLIC will use one of our known IPs.
@param agentMode PUBLIC will use our publicly facing user agent.
@param proxyMode PUBLIC will use one of our known IPs . @param agentMode PUBLIC will use our publicly facing user agent .
public AgentResponse download(String url, ProxyMode proxyMode, AgentMode agentMode, boolean retryRequest, Long nodeId) {
		URI uri = ScoutServices.getUriFromString(url);
		if(uri == null && proxyModeOverride) {
			logger.debug("[ConnectionAgent] - Non-production manual URI formation for request [{}]", url);
			try {
				uri = new URI(url);
			} catch(URISyntaxException e) {
				logger.debug("[ConnectionAgent] - Could not manually parse URI from [{}]", url);
			}
		}
		if(uri == null) {
			return new AgentResponse(new AgentDecision(RequestedAction.PROCEED, "Malformed URI", HttpStatusCode.ELSE));
		}
		HttpGet request = new HttpGet(uri);
		RequestMeta requestMeta = setupRequestMeta(request, uri, proxyMode, agentMode);
		HttpClientContext context = ContextOps.create(requestMeta, url, nodeId, retryRequest);
		AgentResponse agentResponse;
		try {
			long start = System.currentTimeMillis();
			agentResponse = client.execute(request, new ChloeResponseHandler(context), context);
			requestLog(context, System.currentTimeMillis() - start);
			boolean switchOverride = false;
			if(requestMeta.proxyUsed()) {
				boolean captcha = StringUtils.containsIgnoreCase(agentResponse.getContent(), "Please verify you're a human");
				boolean openProxy = ProxyMode.ROTATE_OPEN.equals(proxyMode);
				boolean badStatus = HttpStatusCode.isNotSuccessful(agentResponse.getDecision().getStatusCode());
				if(captcha || openProxy && badStatus) {
					requestMeta.switchNextSession(context);
					switchOverride = true;
				}
			}
			if(!switchOverride && isSuccessfulResponse(agentResponse, true)) {
				agentResponse.setContent(ScoutServices.normalize(agentResponse.getContent(), false));
				Document document = Jsoup.parse(agentResponse.getContent(), lastRedirect(uri, context).toString());
				agentResponse.setDocument(document);
			}
		} catch(Exception e) {
			AgentDecision agentDecision = new AgentDecision(RequestedAction.PROCEED, ExceptionUtils.getRootCauseMessage(e), exceptionStatusTranslator(e, uri));
			agentResponse = new AgentResponse(agentDecision);
			if(ProxyMode.ROTATE_OPEN.equals(proxyMode)) {
				requestMeta.switchNextSession(context);
			}
		}
		return agentResponse;
	}
public AgentResponse download ( String url , ProxyMode proxyMode , AgentMode agentMode , boolean retryRequest , Long nodeId ) { URI uri = ScoutServices . getUriFromString ( url ) ; if ( uri == null && proxyModeOverride ) { logger . debug ( "[ConnectionAgent] - Non-production manual URI formation for request [{}]" , url ) ; try { uri = new URI ( url ) ; } catch ( URISyntaxException e ) { logger . debug ( "[ConnectionAgent] - Could not manually parse URI from [{}]" , url ) ; } } if ( uri == null ) { return new AgentResponse ( new AgentDecision ( RequestedAction . PROCEED , "Malformed URI" , HttpStatusCode . ELSE ) ) ; } HttpGet request = new HttpGet ( uri ) ; RequestMeta requestMeta = setupRequestMeta ( request , uri , proxyMode , agentMode ) ; HttpClientContext context = ContextOps . create ( requestMeta , url , nodeId , retryRequest ) ; AgentResponse agentResponse ; try { long start = System . currentTimeMillis ( ) ; agentResponse = client . execute ( request , new ChloeResponseHandler ( context ) , context ) ; requestLog ( context , System . currentTimeMillis ( ) - start ) ; boolean switchOverride = false ; if ( requestMeta . proxyUsed ( ) ) { boolean captcha = StringUtils . containsIgnoreCase ( agentResponse . getContent ( ) , "Please verify you're a human" ) ; boolean openProxy = ProxyMode . ROTATE_OPEN . equals ( proxyMode ) ; boolean badStatus = HttpStatusCode . isNotSuccessful ( agentResponse . getDecision ( ) . getStatusCode ( ) ) ; if ( captcha || openProxy && badStatus ) { requestMeta . switchNextSession ( context ) ; switchOverride = true ; } } if ( ! switchOverride && isSuccessfulResponse ( agentResponse , true ) ) { agentResponse . setContent ( ScoutServices . normalize ( agentResponse . getContent ( ) , false ) ) ; Document document = Jsoup . parse ( agentResponse . getContent ( ) , lastRedirect ( uri , context ) . toString ( ) ) ; agentResponse . setDocument ( document ) ; } } catch ( Exception e ) { AgentDecision agentDecision = new AgentDecision ( RequestedAction . PROCEED , ExceptionUtils . getRootCauseMessage ( e ) , exceptionStatusTranslator ( e , uri ) ) ; agentResponse = new AgentResponse ( agentDecision ) ; if ( ProxyMode . ROTATE_OPEN . equals ( proxyMode ) ) { requestMeta . switchNextSession ( context ) ; } } return agentResponse ; }
@param proxyMode PUBLIC will use one of our known IPs.
@param proxyMode PUBLIC will use one of our known IPs .