/// Function to handle crawling a single page.
/// Is Single Threaded.
Function to handle crawling a single page.
Is Single Threaded.
Function to handle crawling a single page . Is Single Threaded .
async fn crawl_page(
    url: Url,
    client: reqwest::Client,
    tx: mpsc::Sender<Link>,
    tx_selenium: mpsc::Sender<String>,
    limit: usize,
    resolver: trust_dns_resolver::TokioAsyncResolver,
    word_list: Arc<HashSet<String>>,
) {
    let mut link = links::Link::new_from_url(&url);
    let resp = match get_page(url.as_str(), &client).await {
        Ok(x) => x,
        Err(_) => {
            link.crawled = true;
            let _ = tx.send(link.clone()).await;
            return;
        }
    };
    link.update_from_response(&resp);
    if let Some(host) = &link.host {
        let host = host.to_string();
        let ipv4 = links::resolve_ipv4(&resolver, &host).await;
        let ipv6 = links::resolve_ipv6(&resolver, &host).await;
        link.update_dns(ipv4, ipv6);
    };
    let is_html = link.check_mime_from_list(&[mime::TEXT_HTML, mime::TEXT_HTML_UTF_8]);
    if is_html {
        let html = match resp.text().await {
            Ok(x) => x,
            Err(_) => {
                return;
            }
        };
        if links::check_words_html(&html, word_list) {
            link.contains_words = true;
            let _ = tx_selenium.send(link.url.to_string()).await;
        }
        let links = links::get_links_from_html(&html, url.as_str());
        let tx_ref = &tx;
        stream::iter(links)
            .for_each_concurrent(limit, |x| async move {
                let _ = tx_ref.send(x).await;
            })
            .await;
    }
    if let Err(_) = tx.send(link).await {
        return;
    }
}
async fn crawl_page ( url : Url , client : reqwest :: Client , tx : mpsc :: Sender < Link > , tx_selenium : mpsc :: Sender < String > , limit : usize , resolver : trust_dns_resolver :: TokioAsyncResolver , word_list : Arc < HashSet < String > > , ) { let mut link = links :: Link :: new_from_url ( & url ) ; let resp = match get_page ( url . as_str ( ) , & client ) . await { Ok ( x ) => x , Err ( _ ) => { link . crawled = true ; let _ = tx . send ( link . clone ( ) ) . await ; return ; } } ; link . update_from_response ( & resp ) ; if let Some ( host ) = & link . host { let host = host . to_string ( ) ; let ipv4 = links :: resolve_ipv4 ( & resolver , & host ) . await ; let ipv6 = links :: resolve_ipv6 ( & resolver , & host ) . await ; link . update_dns ( ipv4 , ipv6 ) ; } ; let is_html = link . check_mime_from_list ( & [ mime :: TEXT_HTML , mime :: TEXT_HTML_UTF_8 ] ) ; if is_html { let html = match resp . text ( ) . await { Ok ( x ) => x , Err ( _ ) => { return ; } } ; if links :: check_words_html ( & html , word_list ) { link . contains_words = true ; let _ = tx_selenium . send ( link . url . to_string ( ) ) . await ; } let links = links :: get_links_from_html ( & html , url . as_str ( ) ) ; let tx_ref = & tx ; stream :: iter ( links ) . for_each_concurrent ( limit , | x | async move { let _ = tx_ref . send ( x ) . await ; } ) . await ; } if let Err ( _ ) = tx . send ( link ) . await { return ; } }
Function to handle crawling a single page.
Function to handle crawling a single page .
url
Url
client
reqwest::Client
tx
mpsc::Sender<Link>
tx_selenium
mpsc::Sender<String>
limit
usize
resolver
trust_dns_resolver::TokioAsyncResolver
word_list
Arc<HashSet<String>>