/// Checks if a url should be crawled.
/// Checks the whitelist and blacklist
Checks if a url should be crawled.
Checks the whitelist and blacklist
Checks if a url should be crawled . Checks the whitelist and blacklist
pub fn should_crawl(
        &self,
        whitelist_host: &Option<HashSet<url::Host>>,
        blacklist_host: &Option<HashSet<url::Host>>,
    ) -> bool {
        if let Some(x) = whitelist_host {
            return self.check_host(x, false);
        }
        if let Some(x) = blacklist_host {
            return !self.check_host(x, true);
        }
        false
    }
pub fn should_crawl ( & self , whitelist_host : & Option < HashSet < url :: Host > > , blacklist_host : & Option < HashSet < url :: Host > > , ) -> bool { if let Some ( x ) = whitelist_host { return self . check_host ( x , false ) ; } if let Some ( x ) = blacklist_host { return ! self . check_host ( x , true ) ; } false }
Checks if a url should be crawled.
Checks if a url should be crawled .
&self
whitelist_host
&Option<HashSet<url::Host>>
blacklist_host
&Option<HashSet<url::Host>>