/// Funtion to launch the crawler. Fires off the correct crawler method depending on the arguments.
Funtion to launch the crawler. Fires off the correct crawler method depending on the arguments.
Funtion to launch the crawler . Fires off the correct crawler method depending on the arguments .
async fn launch_crawler(
    origin_url: String,
    depth: Option<usize>,
    task_limit: usize,
    tx_output: mpsc::Sender<Link>,
    tx_selenium: mpsc::Sender<String>,
    whitelist: Option<PathBuf>,
    blacklist: Option<PathBuf>,
    search_words: Option<PathBuf>,
    timeout: u64,
) -> Result<(), String> {
    let origin_url = match Link::new_from_str(origin_url.as_str()) {
        Some(x) => x,
        None => return Err("Invalid Url".to_string()),
    };
    let whitelist = match whitelist {
        Some(x) => match file_handler::read_hosts(x).await {
            Ok(y) => Some(y),
            Err(_) => return Err("Error in reading Whitelist".to_string()),
        },
        None => None,
    };
    let blacklist = match blacklist {
        Some(x) => match file_handler::read_hosts(x).await {
            Ok(y) => Some(y),
            Err(_) => return Err("Error in reading Blacklist".to_string()),
        },
        None => None,
    };
    let word_list = match search_words {
        Some(x) => match file_handler::read_words(x).await {
            Ok(x) => x,
            Err(_) => return Err("Error in reading Word List".to_string()),
        },
        None => HashSet::new(),
    };
    let handler = match depth {
        None => {
            crate::crawler::crawl_no_depth(
                origin_url,
                whitelist,
                blacklist,
                word_list,
                tx_output,
                tx_selenium,
                task_limit,
                timeout,
            )
            .await
        }
        Some(x) => {
            crate::crawler::crawl_with_depth(
                origin_url,
                x,
                whitelist,
                blacklist,
                word_list,
                tx_output,
                tx_selenium,
                task_limit,
                timeout,
            )
            .await
        }
    };
    match handler {
        Ok(_) => Ok(()),
        Err(_) => Err("Something went wrong in the Crawler".to_string()),
    }
}
async fn launch_crawler ( origin_url : String , depth : Option < usize > , task_limit : usize , tx_output : mpsc :: Sender < Link > , tx_selenium : mpsc :: Sender < String > , whitelist : Option < PathBuf > , blacklist : Option < PathBuf > , search_words : Option < PathBuf > , timeout : u64 , ) -> Result < ( ) , String > { let origin_url = match Link :: new_from_str ( origin_url . as_str ( ) ) { Some ( x ) => x , None => return Err ( " " . to_string ( ) ) , } ; let whitelist = match whitelist { Some ( x ) => match file_handler :: read_hosts ( x ) . await { Ok ( y ) => Some ( y ) , Err ( _ ) => return Err ( " " . to_string ( ) ) , } , None => None , } ; let blacklist = match blacklist { Some ( x ) => match file_handler :: read_hosts ( x ) . await { Ok ( y ) => Some ( y ) , Err ( _ ) => return Err ( " " . to_string ( ) ) , } , None => None , } ; let word_list = match search_words { Some ( x ) => match file_handler :: read_words ( x ) . await { Ok ( x ) => x , Err ( _ ) => return Err ( " " . to_string ( ) ) , } , None => HashSet :: new ( ) , } ; let handler = match depth { None => { crate :: crawler :: crawl_no_depth ( origin_url , whitelist , blacklist , word_list , tx_output , tx_selenium , task_limit , timeout , ) . await } Some ( x ) => { crate :: crawler :: crawl_with_depth ( origin_url , x , whitelist , blacklist , word_list , tx_output , tx_selenium , task_limit , timeout , ) . await } } ; match handler { Ok ( _ ) => Ok ( ( ) ) , Err ( _ ) => Err ( " " . to_string ( ) ) , } }
Funtion to launch the crawler.
Funtion to launch the crawler .
origin_url
String
depth
Option<usize>
task_limit
usize
tx_output
mpsc::Sender<Link>
tx_selenium
mpsc::Sender<String>
whitelist
Option<PathBuf>
blacklist
Option<PathBuf>
search_words
Option<PathBuf>
timeout
u64