library/src/network/searchengines/BP-DuckDuckGoScraper.cc
// Creates a new ddgscrape session out of an existing HTTP_SESSION structure.  Use the functions
// in BP-NetworkWeb.h to create the session before passing it here.  YOU ARE RESPONSIBLE FOR SEARCH INPUT
// FORMATTING IN search_query.
Creates a new ddgscrape session out of an existing HTTP_SESSION structure.  Use the functions
in BP-NetworkWeb.h to create the session before passing it here.  YOU ARE RESPONSIBLE FOR SEARCH INPUT
FORMATTING IN search_query.
Creates a new ddgscrape session out of an existing HTTP_SESSION structure . Use the functions in BP - NetworkWeb . h to create the session before passing it here . YOU ARE RESPONSIBLE FOR SEARCH INPUT FORMATTING IN search_query .
P_BP_DDGSCRAPE_SESSION BP_OpenNewDdgSearch(char *search_query, P_BP_HTTP_SESSION http_session)
{
	if(!search_query)
		return NULL;
	if(!http_session)
		return NULL;
	P_BP_LINKL self_allocator = BP_CreateLinkL("ddg_search", BP_TRUE);
	P_BP_DDGSCRAPE_SESSION session = (P_BP_DDGSCRAPE_SESSION) tq_calloc(self_allocator, sizeof(BP_DDGSCRAPE_SESSION), 1, BPLN);
	session->tq = self_allocator;
	session->http_session = http_session;
	session->ddg_default_search = tq_strdup(session->tq, (char *) bp_ddg_default_search, BPLN);
	session->retrieve_results = 100;
	session->search_query = tq_strdup(session->tq, (char *) search_query, BPLN);
	session->http_session->http_session_timeout = 15;
	session->http_session->http_connect_timeout = 15;
	if(!session->http_session->db_populated_proxy_n)
	{
		session->ddg_retry_with_new_proxy_on_captcha = 1;
		session->ddg_retry_on_bad_connect            = 1;
	}
	else
	{
		session->http_session->request_time_modulo = 30;
	}
	return session;
}
P_BP_DDGSCRAPE_SESSION BP_OpenNewDdgSearch ( char * search_query , P_BP_HTTP_SESSION http_session ) { if ( ! search_query ) return NULL ; if ( ! http_session ) return NULL ; P_BP_LINKL self_allocator = BP_CreateLinkL ( " " , BP_TRUE ) ; P_BP_DDGSCRAPE_SESSION session = ( P_BP_DDGSCRAPE_SESSION ) tq_calloc ( self_allocator , sizeof ( BP_DDGSCRAPE_SESSION ) , 1 , BPLN ) ; session -> tq = self_allocator ; session -> http_session = http_session ; session -> ddg_default_search = tq_strdup ( session -> tq , ( char * ) bp_ddg_default_search , BPLN ) ; session -> retrieve_results = 100 ; session -> search_query = tq_strdup ( session -> tq , ( char * ) search_query , BPLN ) ; session -> http_session -> http_session_timeout = 15 ; session -> http_session -> http_connect_timeout = 15 ; if ( ! session -> http_session -> db_populated_proxy_n ) { session -> ddg_retry_with_new_proxy_on_captcha = 1 ; session -> ddg_retry_on_bad_connect = 1 ; } else { session -> http_session -> request_time_modulo = 30 ; } return session ; }
Creates a new ddgscrape session out of an existing HTTP_SESSION structure.
Creates a new ddgscrape session out of an existing HTTP_SESSION structure .
// check to make sure we have a search pointer // ensure we have a http session pointer // create a new self allocator // create new session // set self referential allocator // create new garbage collector // session->gc = BP_CreateNewGarbageCollector("ddgscrape collector", 1); // --- Fill in Structure Values ------------------- // set the web request pointer in our ddgscrape session // duplicate the default search  search string (used as a basis for a sprintf replace later) // set the default number of results to return per query (100 by default, set manually in the structure later if wanted) // set the search string now // set the session timeout to 3 // set the connect timeout to 2 // if we have a proxy set, override default captcha behavior // set random sleep between 0 and 5 if there are no proxies // being used. // return filled out structure