/// Function to crawl a single sitemap.
/// Currently only supports text sitemap
Function to crawl a single sitemap.
Currently only supports text sitemap
Function to crawl a single sitemap . Currently only supports text sitemap
async fn crawl_sitemap(url: Url, tx: mpsc::Sender<Link>, limit: usize, client: reqwest::Client) {
    let mut link = links::Link::new_from_url(&url);
    let resp = match get_page(url.as_str(), &client).await {
        Ok(x) => x,
        Err(_) => return,
    };
    link.update_from_response(&resp);
    let text = match resp.text().await {
        Ok(x) => x,
        Err(_) => return,
    };
    let links = match link.content_type {
        Some(x) => match (x.type_(), x.subtype()) {
            (mime::TEXT, mime::PLAIN) => links::get_links_from_text(&text, url.as_str()),
            _ => return,
        },
        None => return,
    };
    let tx_ref = &tx;
    stream::iter(links)
        .for_each_concurrent(limit, |x| async {
            let _ = tx_ref.send(x).await;
        })
        .await;
}
async fn crawl_sitemap ( url : Url , tx : mpsc :: Sender < Link > , limit : usize , client : reqwest :: Client ) { let mut link = links :: Link :: new_from_url ( & url ) ; let resp = match get_page ( url . as_str ( ) , & client ) . await { Ok ( x ) => x , Err ( _ ) => return , } ; link . update_from_response ( & resp ) ; let text = match resp . text ( ) . await { Ok ( x ) => x , Err ( _ ) => return , } ; let links = match link . content_type { Some ( x ) => match ( x . type_ ( ) , x . subtype ( ) ) { ( mime :: TEXT , mime :: PLAIN ) => links :: get_links_from_text ( & text , url . as_str ( ) ) , _ => return , } , None => return , } ; let tx_ref = & tx ; stream :: iter ( links ) . for_each_concurrent ( limit , | x | async { let _ = tx_ref . send ( x ) . await ; } ) . await ; }
Function to crawl a single sitemap.
Function to crawl a single sitemap .
url
Url
tx
mpsc::Sender<Link>
limit
usize
client
reqwest::Client