/// Map over lines in the associated files, writing to a sink for each file.
///
/// A (cloneable) one-pass iterator is provided over each line's words
/// is passed per `apply` invocation. You should write out just the contents
/// and any newlines you'd like to add yourself.
///
/// Creates a new file, one for each input path in this `SvmScanner`, in the
/// same directory as the input files, with a different suffix. I.e., if we
/// are scanning over files "f1.svm" and "f2.svm" then the output of this
/// command will be "f1.<suffix>" and "f2.<suffix>".
///
/// It's on the user to make sure this doesn't overwrite existing files.
/// If an input file has an extension and it's equivalent to the suffix, shit
/// will break.
///
/// The `scratch` argument can be used to generate mutable scratch state
/// that is passed to the apply function.
Map over lines in the associated files, writing to a sink for each file.
A (cloneable) one-pass iterator is provided over each line's words
is passed per `apply` invocation. You should write out just the contents
and any newlines you'd like to add yourself.

Creates a new file, one for each input path in this `SvmScanner`, in the
same directory as the input files, with a different suffix.

It's on the user to make sure this doesn't overwrite existing files.
If an input file has an extension and it's equivalent to the suffix, shit
will break.

The `scratch` argument can be used to generate mutable scratch state
that is passed to the apply function.
Map over lines in the associated files writing to a sink for each file . A ( cloneable ) one - pass iterator is provided over each line ' s words is passed per ` apply ` invocation . You should write out just the contents and any newlines you ' d like to add yourself . Creates a new file one for each input path in this ` SvmScanner ` in the same directory as the input files with a different suffix . It ' s on the user to make sure this doesn ' t overwrite existing files . If an input file has an extension and it ' s equivalent to the suffix shit will break . The ` scratch ` argument can be used to generate mutable scratch state that is passed to the apply function .
pub(crate) fn for_each_sink<Apply, Scratch, T>(
        &self,
        apply: Apply,
        suffix: &str,
        scratch: Scratch,
    ) where
        Apply: Fn(DelimIter<'_>, &mut BufWriter<File>, &mut T) + Send + Sync,
        Scratch: Fn() -> T + Sync + Send,
    {
        // This works by writing out a file for each byte block, then cat-ing them.
        //
        // Suppose we have input files f1.svm and f2.svm
        //
        // We might have byte blocks 0, 1, 2 associated with f1 and 3, 4 with f2.
        // So we'll first create files f1.<suffix>.0 f1.<suffix>.1 f1.<suffix>.2
        // and f2.<suffix>.3 f2.<suffix>.4 in parallel.
        let pad_width = format!("{}", (self.blocks.len() - 1).max(0)).len();
        let grouped_blocks = self
            .blocks
            .iter()
            .enumerate()
            .collect::<Vec<_>>()
            .par_iter()
            // for each block, write it out to its file
            .map(|(i, block)| {
                let block_suffix = format!("{}.{:0width$}", suffix, i, width = pad_width);
                let (block_path, block_file) =
                    replace_extension(&self.paths[block.mmap_index], &block_suffix);
                // 64KB buffer chosen experimentally based on my laptop
                let mut writer = BufWriter::with_capacity(64 * 1024, block_file);
                let mut scratch_state = scratch();
                for line in DelimIter::new(block.bytes(&self), b'\n') {
                    apply(
                        DelimIter::new(line, self.delimiter),
                        &mut writer,
                        &mut scratch_state,
                    );
                }
                writer.flush().unwrap();
                (block.mmap_index, block_path)
            })
            .collect::<Vec<_>>()
            .into_iter()
            .group_by(|(mmap_index, _)| *mmap_index)
            .into_iter()
            .map(|(mmap_index, gby)| (mmap_index, gby.map(|(_, path)| path).collect::<Vec<_>>()))
            .collect::<Vec<_>>();
        grouped_blocks
            .into_par_iter()
            .for_each(|(mmap_index, block_files)| {
                let (_, out_file) = replace_extension(&self.paths[mmap_index], suffix);
                assert!(Command::new("cat")
                    .args(&block_files)
                    .stdout(out_file)
                    .status()
                    .expect("cat failure")
                    .success());
                assert!(Command::new("rm")
                    .args(&block_files)
                    .status()
                    .expect("rm failure")
                    .success());
            });
    }
pub ( crate ) fn for_each_sink < Apply , Scratch , T > ( & self , apply : Apply , suffix : & str , scratch : Scratch , ) where Apply : Fn ( DelimIter < ' _ > , & mut BufWriter < File > , & mut T ) + Send + Sync , Scratch : Fn ( ) -> T + Sync + Send , { // This works by writing out a file for each byte block, then cat-ing them. // // Suppose we have input files f1.svm and f2.svm // // We might have byte blocks 0, 1, 2 associated with f1 and 3, 4 with f2. // So we'll first create files f1.<suffix>.0 f1.<suffix>.1 f1.<suffix>.2 // and f2.<suffix>.3 f2.<suffix>.4 in parallel. let pad_width = format ! ( " " ( self blocks len ( ) 1 ) max ( 0 ) ) . len ( ) ; let grouped_blocks = self . blocks . iter ( ) . enumerate ( ) . collect :: < Vec < _ > > ( ) . par_iter ( ) // for each block, write it out to its file . map ( | ( i , block ) | { let block_suffix = format ! ( " " suffix i width pad_width ) ; let ( block_path , block_file ) = replace_extension ( & self . paths [ block . mmap_index ] , & block_suffix ) ; // 64KB buffer chosen experimentally based on my laptop let mut writer = BufWriter :: with_capacity ( 64 * 1024 , block_file ) ; let mut scratch_state = scratch ( ) ; for line in DelimIter :: new ( block . bytes ( & self ) , b'\n' ) { apply ( DelimIter :: new ( line , self . delimiter ) , & mut writer , & mut scratch_state , ) ; } writer . flush ( ) . unwrap ( ) ; ( block . mmap_index , block_path ) } ) . collect :: < Vec < _ > > ( ) . into_iter ( ) . group_by ( | ( mmap_index , _ ) | * mmap_index ) . into_iter ( ) . map ( | ( mmap_index , gby ) | ( mmap_index , gby . map ( | ( _ , path ) | path ) . collect :: < Vec < _ > > ( ) ) ) . collect :: < Vec < _ > > ( ) ; grouped_blocks . into_par_iter ( ) . for_each ( | ( mmap_index , block_files ) | { let ( _ , out_file ) = replace_extension ( & self . paths [ mmap_index ] , suffix ) ; assert ! ( Command new ( " " ) args ( block_files ) stdout ( out_file ) status ( ) expect ( " " ) success ( ) ) ; assert ! ( Command new ( " " ) args ( block_files ) status ( ) expect ( " " ) success ( ) ) ; } ) ; }
Map over lines in the associated files, writing to a sink for each file.
Map over lines in the associated files writing to a sink for each file .
&self
apply
Apply
suffix
&str
scratch
Scratch