/* sorts lines of a file through external bash commands
       1) separates file into subfiles; according to available RAM and disk
       2) sort each subfile into a copy and move the copy back to subfile
       3) merge-sort subfiles into the original file */
sorts lines of a file through external bash commands
1) separates file into subfiles; according to available RAM and disk
2) sort each subfile into a copy and move the copy back to subfile
3) merge-sort subfiles into the original file
sorts lines of a file through external bash commands 1 ) separates file into subfiles ; according to available RAM and disk 2 ) sort each subfile into a copy and move the copy back to subfile 3 ) merge - sort subfiles into the original file
pub fn sort_lines(&self,filename: &str, outdir: &str) -> bool {
        // list the current directory and see available memory
        let current_dir = env::current_dir().unwrap();
        let free_disk_space = fs2::free_space(current_dir).unwrap() as f64;
        // check size of the file
        let file_size = fs::metadata(filename).unwrap().len() as f64;
        // if file takes more than 1/3 of disk, it may clogg up the disk
        let ratio = (free_disk_space / file_size).ceil() as u32;
        if ratio < (4.0 as u32) {panic!("Not enough disk space for sorting!");}
        // check available RAM, take half of it
        let mut avail_ram = ((mem_info().unwrap().total * 1024) / 2) as f64;
        // if available RAM is more than 20 GB, limit to that amount
        // the idea is that sorting very large files takes too much time
        // and it goes wrong easier
        // it is better to sort many smaller files than few large ones
        if avail_ram > 21474836480. as f64 {avail_ram = 21474836480. as f64;}
        let ram_ratio = (avail_ram / file_size) as f64;
        // how many lines will be in sub-files for sorting
        let mut num_of_lines: String = "-l ".to_owned();
        num_of_lines.push_str(&((ram_ratio * (self.numrec as f64)).ceil() as u64).to_string());
        let mut tmp_filename: String = "".to_owned();
        tmp_filename.push_str(filename);
        tmp_filename.push_str("_");
        // use bash to split original file into subfiles for sorting
        let _split_file = Command::new("split").args(&["-a 10", &num_of_lines])
            .arg(filename).arg(tmp_filename.clone())
            .status().expect("Error in splitting file");
        // list created temporary files, it returns vector of ASCII u8
        let list_tmp = Command::new("ls").arg(outdir).output().expect("error!");
        let tmp_files: Vec<_> = list_tmp.stdout.split(|i| *i == 10u8).collect();
        // create a vector of temporary file names
        let mut tmp_filename_list = Vec::<String>::new();
        // loop over temporary files
        for tmp_file in tmp_files {
            let mut tmp_filename = outdir.to_string();
            tmp_filename.push_str("/");
            tmp_filename.push_str(&String::from_utf8(tmp_file.to_vec()).unwrap());
            // sort only subfiles containing "tmp_"
            if tmp_filename.contains("tmp_") {
                let mut tmp_tmp_filename: String = "-o".to_owned();
                tmp_tmp_filename.push_str(&tmp_filename);
                tmp_tmp_filename.push_str(".cp");
                tmp_filename_list.push(tmp_filename.clone());
                // sort the subfiles, send output to tmp_tmp_filename
                let _sort_tmp = Command::new("sort").arg(tmp_filename.clone()).arg(tmp_tmp_filename.clone()).output().expect("Error in sorting.");
                tmp_tmp_filename.replace_range(..2,"");
                // overwrite original temporary filenames with sorted ones
                let _overwrite = Command::new("mv").arg(&tmp_tmp_filename).arg(&tmp_filename).status().expect("Error in overwriting!");
            }
        }
        assert!(tmp_filename_list.len() > 0);
        // end_filename should be equal to the original input-file (with -o)
        let mut end_filename: String = "-o".to_owned();
        end_filename.push_str(filename);
        // merge sorted subfiles into the original file
        let _sort_tmp = Command::new("sort").arg("-m")
            .args(&tmp_filename_list.clone()).arg(end_filename)
            .status().expect("Error in merging!");
        // delete the subfiles
        let _delete = Command::new("rm").args(&tmp_filename_list).status().expect("Error in deleting!");
        true
    }
pub fn sort_lines ( & self , filename : & str , outdir : & str ) -> bool { // list the current directory and see available memory let current_dir = env :: current_dir ( ) . unwrap ( ) ; let free_disk_space = fs2 :: free_space ( current_dir ) . unwrap ( ) as f64 ; // check size of the file let file_size = fs :: metadata ( filename ) . unwrap ( ) . len ( ) as f64 ; // if file takes more than 1/3 of disk, it may clogg up the disk let ratio = ( free_disk_space / file_size ) . ceil ( ) as u32 ; if ratio < ( 4.0 as u32 ) { panic ! ( " " ) ; } // check available RAM, take half of it let mut avail_ram = ( ( mem_info ( ) . unwrap ( ) . total * 1024 ) / 2 ) as f64 ; // if available RAM is more than 20 GB, limit to that amount // the idea is that sorting very large files takes too much time // and it goes wrong easier // it is better to sort many smaller files than few large ones if avail_ram > 21474836480. as f64 { avail_ram = 21474836480. as f64 ; } let ram_ratio = ( avail_ram / file_size ) as f64 ; // how many lines will be in sub-files for sorting let mut num_of_lines : String = " " . to_owned ( ) ; num_of_lines . push_str ( & ( ( ram_ratio * ( self . numrec as f64 ) ) . ceil ( ) as u64 ) . to_string ( ) ) ; let mut tmp_filename : String = " " . to_owned ( ) ; tmp_filename . push_str ( filename ) ; tmp_filename . push_str ( " " ) ; // use bash to split original file into subfiles for sorting let _split_file = Command :: new ( " " ) . args ( & [ " " , & num_of_lines ] ) . arg ( filename ) . arg ( tmp_filename . clone ( ) ) . status ( ) . expect ( " " ) ; // list created temporary files, it returns vector of ASCII u8 let list_tmp = Command :: new ( " " ) . arg ( outdir ) . output ( ) . expect ( " " ) ; let tmp_files : Vec < _ > = list_tmp . stdout . split ( | i | * i == 10u8 ) . collect ( ) ; // create a vector of temporary file names let mut tmp_filename_list = Vec :: < String > :: new ( ) ; // loop over temporary files for tmp_file in tmp_files { let mut tmp_filename = outdir . to_string ( ) ; tmp_filename . push_str ( " " ) ; tmp_filename . push_str ( & String :: from_utf8 ( tmp_file . to_vec ( ) ) . unwrap ( ) ) ; // sort only subfiles containing "tmp_" if tmp_filename . contains ( " " ) { let mut tmp_tmp_filename : String = " " . to_owned ( ) ; tmp_tmp_filename . push_str ( & tmp_filename ) ; tmp_tmp_filename . push_str ( " " ) ; tmp_filename_list . push ( tmp_filename . clone ( ) ) ; // sort the subfiles, send output to tmp_tmp_filename let _sort_tmp = Command :: new ( " " ) . arg ( tmp_filename . clone ( ) ) . arg ( tmp_tmp_filename . clone ( ) ) . output ( ) . expect ( " " ) ; tmp_tmp_filename . replace_range ( .. 2 , " " ) ; // overwrite original temporary filenames with sorted ones let _overwrite = Command :: new ( " " ) . arg ( & tmp_tmp_filename ) . arg ( & tmp_filename ) . status ( ) . expect ( " " ) ; } } assert ! ( tmp_filename_list len ( ) 0 ) ; // end_filename should be equal to the original input-file (with -o) let mut end_filename : String = " " . to_owned ( ) ; end_filename . push_str ( filename ) ; // merge sorted subfiles into the original file let _sort_tmp = Command :: new ( " " ) . arg ( " " ) . args ( & tmp_filename_list . clone ( ) ) . arg ( end_filename ) . status ( ) . expect ( " " ) ; // delete the subfiles let _delete = Command :: new ( " " ) . args ( & tmp_filename_list ) . status ( ) . expect ( " " ) ; true }
sorts lines of a file through external bash commands
1) separates file into subfiles; according to available RAM and disk
2) sort each subfile into a copy and move the copy back to subfile
3) merge-sort subfiles into the original file
sorts lines of a file through external bash commands 1 ) separates file into subfiles ; according to available RAM and disk 2 ) sort each subfile into a copy and move the copy back to subfile 3 ) merge - sort subfiles into the original file
&self
filename
&str
outdir
&str