library/src/network/searchengines/BP-DisconnectMeScraper.cc
// Search for a query for a provided search target.
Search for a query for a provided search target.
Search for a query for a provided search target .
BP_ERROR_T BP_DisconnectMeSearch
(
		P_BP_DISCONNECTME_SCRAPE_SESSION dmss,
		char *search_query,
		BP_DISCONNECT_ME_SEARCH_TARGET search_target,
		size_t max_dupes_before_exit
)
{
	if(!dmss)
		return ERR_FAILURE;
	if(!search_query)
		return ERR_FAILURE;
	if(!BP_StringIsReasonablePrintableString(search_query, bpstrlen(search_query), BP_FALSE, BP_FALSE))
		return ERR_FAILURE;
	if(!search_target)
		return ERR_FAILURE;
	BP_DisconnectMePreGenQueries
	(
			dmss,
			search_query,
			search_target
	);
	char * lookup_table[10];
	lookup_table[0] = "search_queries";
	lookup_table[1] = NULL;
	P_BP_HASH_TABLE_KEYED_ENTRY queries_table = BP_HashRegLookupTableByPath(dmss->system_hreg, (char **) &lookup_table);
	if(!queries_table)
		BP_HASH_TABLE_LEAVE_CRITICAL_SECTION_AND_RETURN_FROM_FUNC(dmss->system_hreg, ERR_FAILURE);
	if(!queries_table->data)
		BP_HASH_TABLE_LEAVE_CRITICAL_SECTION_AND_RETURN_FROM_FUNC(dmss->system_hreg, ERR_FAILURE);
	lookup_table[0] = "search_results";
	lookup_table[1] = NULL;
	P_BP_HASH_TABLE_KEYED_ENTRY query_results_table = BP_HashRegLookupTableByPath(dmss->system_hreg, (char **) &lookup_table);
	if(!query_results_table)
		BP_HASH_TABLE_LEAVE_CRITICAL_SECTION_AND_RETURN_FROM_FUNC(dmss->system_hreg, ERR_FAILURE);
	BP_HTTPAddRequestHeader
	(
			dmss->http_session,
			"User-Agent",
			"Header Descriptionh",
			"Mozilla Firefox 25.0.1 (Build 15r)",
			BP_TRUE
	);
	size_t curr_record_idx = 0;
	BP_BOOL found_end_of_records = BP_FALSE;
	size_t total_dupe_count = 0;
	size_t total_dupes_before_exit = 40;
	if(max_dupes_before_exit)
		total_dupes_before_exit = max_dupes_before_exit;
	BP_HASH_ITER_OPEN((P_BP_HASH_TABLE_KEYED_ENTRY) queries_table->data, query_iter)
	{
		size_t clean_session = 1;
		while(BP_HTTPRequest(dmss->http_session, (char *) query_iter->data, clean_session) != ERR_SUCCESS)
		{
		}
		if(!dmss->http_session->page->page_data)
			continue;
		P_BP_HTML_FULL_EXTRACTION html_extraction = HTMLDecodeBuffer
		(
				dmss->http_session->page->page_data,
				dmss->http_session->page->page_data_size,
				(char *) query_iter->data,
				NULL,
				NULL,
				HTML_PARSE_NOERROR,
				NULL
		);
		if(!html_extraction)
			continue;
		if(!html_extraction->anchors)
			continue;
		size_t iter_n = 0;
		char * static_host_blacklist[] =
		{
				"/",
				"./",
				"https://www.google.com",
				"https://disconnect.me",
				"https://duckduckgo.com",
				"https://books.google.com",
				"http://books.google.com",
				NULL
		};
		printf("\n Displaying anchors here?!!");
		char * anchor_last_preserved = NULL;
		size_t anchor_retrieved_that_passed_filter_count = 0;
		P_BP_HTML_ANCHOR_LIST anchor_iter = html_extraction->anchors->first;
		for
		(
				;
				anchor_iter;
				anchor_iter = anchor_iter->next
		)
		{
			iter_n++;
			if(iter_n < 7)
				continue;
			if(!BP_StringIsReasonablePrintableString(anchor_iter->href, bpstrlen(anchor_iter->href), BP_FALSE, BP_FALSE))
				continue;
			size_t anchor_len = bpstrlen(anchor_iter->href);
			if(anchor_len < 6)
				continue;
			%
			%
			BP_BOOL blacklist_flagged = BP_FALSE;
			size_t blacklist_iter_n = 0;
			for(; static_host_blacklist[blacklist_iter_n]; blacklist_iter_n++)
			{
				if(anchor_len >= bpstrlen(static_host_blacklist[blacklist_iter_n]))
				if(strncmp(anchor_iter->href, static_host_blacklist[blacklist_iter_n], bpstrlen(static_host_blacklist[blacklist_iter_n])) == 0)
				{
					blacklist_flagged = BP_TRUE;
					break;
				}
			}
			if(blacklist_flagged == BP_TRUE)
				continue;
			if(dmss->system_hreg->hive)
			{
				char * lookup_path[10];
				BP_ZERO_STACK(lookup_path);
				lookup_path[0] = "search_results";
				lookup_path[1] = anchor_iter->href;
				lookup_path[2] = NULL;
				P_BP_HASH_TABLE_KEYED_ENTRY lookup_test = BP_HashRegLookupTableEntryByPath(dmss->system_hreg, (char **) &lookup_path);
				if(lookup_test)
				{
					total_dupe_count++;
					if(total_dupe_count >= total_dupes_before_exit)
					{
						found_end_of_records = BP_TRUE;
						break;
					}
				}
			}
			if(found_end_of_records)
				break;
			%%
			%%
			char * add_path[10];
			BP_ZERO_STACK(add_path);
			add_path[0] = "search_results";
			add_path[1] = NULL;
			BP_ERROR_T ret_status = BP_HashRegAddDataToTableByPath
			(
					dmss->system_hreg,
					(char **) &add_path,
					anchor_iter->href,
					BP_HASH_TABLE_KEYED_ENTRY_TYPE_BINARY_BLOB,
					anchor_iter->href,
					bpstrlen(anchor_iter->href)
			);
			if(ret_status)
			{
				printf("\n [+] Added: %u - %s", ret_status, anchor_iter->href);
				anchor_retrieved_that_passed_filter_count++;
			}
			curr_record_idx++;
		}
		if(!anchor_retrieved_that_passed_filter_count)
		{
			found_end_of_records = BP_TRUE;
			break;
		}
		if(!curr_record_idx)
		{
			found_end_of_records = BP_TRUE;
			break;
		}
		if(found_end_of_records)
			break;
	} BP_HASH_ITER_CLOSE;
	BP_HASH_TABLE_LEAVE_CRITICAL_SECTION(dmss->system_hreg);
	return ERR_SUCCESS;
}
BP_ERROR_T BP_DisconnectMeSearch ( P_BP_DISCONNECTME_SCRAPE_SESSION dmss , char * search_query , BP_DISCONNECT_ME_SEARCH_TARGET search_target , size_t max_dupes_before_exit ) { if ( ! dmss ) return ERR_FAILURE ; if ( ! search_query ) return ERR_FAILURE ; if ( ! BP_StringIsReasonablePrintableString ( search_query , bpstrlen ( search_query ) , BP_FALSE , BP_FALSE ) ) return ERR_FAILURE ; if ( ! search_target ) return ERR_FAILURE ; BP_DisconnectMePreGenQueries ( dmss , search_query , search_target ) ; char * lookup_table [ 10 ] ; lookup_table [ 0 ] = " " ; lookup_table [ 1 ] = NULL ; P_BP_HASH_TABLE_KEYED_ENTRY queries_table = BP_HashRegLookupTableByPath ( dmss -> system_hreg , ( char * * ) & lookup_table ) ; if ( ! queries_table ) BP_HASH_TABLE_LEAVE_CRITICAL_SECTION_AND_RETURN_FROM_FUNC ( dmss -> system_hreg , ERR_FAILURE ) ; if ( ! queries_table -> data ) BP_HASH_TABLE_LEAVE_CRITICAL_SECTION_AND_RETURN_FROM_FUNC ( dmss -> system_hreg , ERR_FAILURE ) ; lookup_table [ 0 ] = " " ; lookup_table [ 1 ] = NULL ; P_BP_HASH_TABLE_KEYED_ENTRY query_results_table = BP_HashRegLookupTableByPath ( dmss -> system_hreg , ( char * * ) & lookup_table ) ; if ( ! query_results_table ) BP_HASH_TABLE_LEAVE_CRITICAL_SECTION_AND_RETURN_FROM_FUNC ( dmss -> system_hreg , ERR_FAILURE ) ; BP_HTTPAddRequestHeader ( dmss -> http_session , " " , " " , " " , BP_TRUE ) ; size_t curr_record_idx = 0 ; BP_BOOL found_end_of_records = BP_FALSE ; size_t total_dupe_count = 0 ; size_t total_dupes_before_exit = 40 ; if ( max_dupes_before_exit ) total_dupes_before_exit = max_dupes_before_exit ; BP_HASH_ITER_OPEN ( ( P_BP_HASH_TABLE_KEYED_ENTRY ) queries_table -> data , query_iter ) { size_t clean_session = 1 ; while ( BP_HTTPRequest ( dmss -> http_session , ( char * ) query_iter -> data , clean_session ) != ERR_SUCCESS ) { } if ( ! dmss -> http_session -> page -> page_data ) continue ; P_BP_HTML_FULL_EXTRACTION html_extraction = HTMLDecodeBuffer ( dmss -> http_session -> page -> page_data , dmss -> http_session -> page -> page_data_size , ( char * ) query_iter -> data , NULL , NULL , HTML_PARSE_NOERROR , NULL ) ; if ( ! html_extraction ) continue ; if ( ! html_extraction -> anchors ) continue ; size_t iter_n = 0 ; char * static_host_blacklist [ ] = { " " , " " , " " , " " , " " , " " , " " , NULL } ; printf ( " \n " ) ; char * anchor_last_preserved = NULL ; size_t anchor_retrieved_that_passed_filter_count = 0 ; P_BP_HTML_ANCHOR_LIST anchor_iter = html_extraction -> anchors -> first ; for ( ; anchor_iter ; anchor_iter = anchor_iter -> next ) { iter_n ++ ; if ( iter_n < 7 ) continue ; if ( ! BP_StringIsReasonablePrintableString ( anchor_iter -> href , bpstrlen ( anchor_iter -> href ) , BP_FALSE , BP_FALSE ) ) continue ; size_t anchor_len = bpstrlen ( anchor_iter -> href ) ; if ( anchor_len < 6 ) continue ; BP_BOOL blacklist_flagged = BP_FALSE ; size_t blacklist_iter_n = 0 ; for ( ; static_host_blacklist [ blacklist_iter_n ] ; blacklist_iter_n ++ ) { if ( anchor_len >= bpstrlen ( static_host_blacklist [ blacklist_iter_n ] ) ) if ( strncmp ( anchor_iter -> href , static_host_blacklist [ blacklist_iter_n ] , bpstrlen ( static_host_blacklist [ blacklist_iter_n ] ) ) == 0 ) { blacklist_flagged = BP_TRUE ; break ; } } if ( blacklist_flagged == BP_TRUE ) continue ; if ( dmss -> system_hreg -> hive ) { char * lookup_path [ 10 ] ; BP_ZERO_STACK ( lookup_path ) ; lookup_path [ 0 ] = " " ; lookup_path [ 1 ] = anchor_iter -> href ; lookup_path [ 2 ] = NULL ; P_BP_HASH_TABLE_KEYED_ENTRY lookup_test = BP_HashRegLookupTableEntryByPath ( dmss -> system_hreg , ( char * * ) & lookup_path ) ; if ( lookup_test ) { total_dupe_count ++ ; if ( total_dupe_count >= total_dupes_before_exit ) { found_end_of_records = BP_TRUE ; break ; } } } if ( found_end_of_records ) break ; char * add_path [ 10 ] ; BP_ZERO_STACK ( add_path ) ; add_path [ 0 ] = " " ; add_path [ 1 ] = NULL ; BP_ERROR_T ret_status = BP_HashRegAddDataToTableByPath ( dmss -> system_hreg , ( char * * ) & add_path , anchor_iter -> href , BP_HASH_TABLE_KEYED_ENTRY_TYPE_BINARY_BLOB , anchor_iter -> href , bpstrlen ( anchor_iter -> href ) ) ; if ( ret_status ) { printf ( " \n " , ret_status , anchor_iter -> href ) ; anchor_retrieved_that_passed_filter_count ++ ; } curr_record_idx ++ ; } if ( ! anchor_retrieved_that_passed_filter_count ) { found_end_of_records = BP_TRUE ; break ; } if ( ! curr_record_idx ) { found_end_of_records = BP_TRUE ; break ; } if ( found_end_of_records ) break ; } BP_HASH_ITER_CLOSE ; BP_HASH_TABLE_LEAVE_CRITICAL_SECTION ( dmss -> system_hreg ) ; return ERR_SUCCESS ; }
Search for a query for a provided search target.
Search for a query for a provided search target .
// %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% // %%% Enter Critical Section and Begin Search %%%%%%%%%%%% // %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% // Attempt to pregenerate a stack of queries for retrieval. // create lookup table and get queries table // lookup search queries // overwrite old lookup with new search results lookup // lookup results table // %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% // %%% Run Queries and Store Results %%%%%%%%%%%%%%%%%%%% // %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% // add user agent // current record index // if we found the end of our records, set this here // total number of duplicates encountered // total number of duplicates which can be encountered before we force exit // set max dupes before exit // iterate through queries // printf("\n Search Query: %s - %s", (char *) query_iter->key, (char *) query_iter->data); // declare session clearer // attempt to make the actual request // ensure we have data // Decode Request HTML Data // ensure we have a html extraction // ensure we parsed anchors // print out anchors // HTMLPrintAnchors(html_extraction->anchors, BP_TRUE); // set anchor iterator // static host blacklist applied to anchors // this is used to detect when we run out of results // this is set to the number of records that actually were filtered from search // anchor iterator // increment the iter count // skip first arbitrary 7 anchors (they're always garbage) // only gather reasonable anchors // calculate anchor length // must be at least ftp:// (in case ftp links) // %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% // %%% Run Blacklist Checks %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% // %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% // set if the blacklist was triggered // blacklist checks // skip google links // continue lookp if the blacklist is flagged // %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% // %%% Check for Duplicate Entry %%%%%%%%%%%%%%%%%%%%%%%%%% // %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% // if we have a hive, attempt to lookup // check if already exists // set lookup path // attempt lookup // increment the total dupe count // if the total dupe count matches, force exit // break if we're at the end // %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% // %%% Attempt Record Addition %%%%%%%%%%%%%%%%%%%%%%%%%%%%%% // %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% // set add path // set path to search_results // create index buffer // display added // increment the number of records // increment the current record index only after // all sanity checks. // ensure we have retrieved some anchors that passed checks // if we have no records, we've found the end and can exit // no matter what // exit if we're at the end of records // %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% // %%% Leave Critical Section and Return %%%%%%%%%%%%%%%% // %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% // leave critical section // return indicating success