/// Map reduce to collect per-feature statistics over lines.
Map reduce to collect per-feature statistics over lines.
Map reduce to collect per - feature statistics over lines .
pub(crate) fn map_reduce_sketch<S, F>(
    train: &SvmScanner,
    featurizer: &Featurizer,
    filter: F,
) -> (S, Vec<S>)
where
    S: Sketch,
    F: for<'a> Fn(DelimIter<'a>) -> bool + Sync + Send,
{
    // add in an extra "global" feature at the end for overall numbers.
    let global = featurizer.nsparse();
    let (_, stats) = train.fold_reduce(
        || (0usize, vec![LineSketch::<S>::default(); global + 1]),
        |(mut line_num, mut stats), mut word_iter| {
            if !filter(word_iter.clone()) {
                return (line_num, stats);
            }
            line_num += 1;
            // SAFETY: caller promised the first word is f64 in ascii
            // in SvmScanner guarantees
            let target: &str =
                unsafe { std::str::from_utf8_unchecked(word_iter.next().expect("target")) };
            let target: f64 = target.parse().expect("float format");
            for h in word_iter
                .enumerate()
                .flat_map(|(i, word)| featurizer.sparse(i, word))
            {
                let stat = &mut stats[h as usize];
                stat.update(line_num, target);
            }
            let stat = &mut stats[global];
            stat.update(line_num, target);
            (line_num, stats)
        },
        |(_, mut l_stats), (_, r_stats)| {
            for (l_stat, r_stat) in l_stats.iter_mut().zip(r_stats.iter()) {
                l_stat.merge(r_stat);
            }
            (0, l_stats)
        },
    );
    let mut all: Vec<_> = stats
        .into_iter()
        .map(|line_sketch| line_sketch.sketch)
        .collect();
    let global = all.pop().unwrap();
    (global, all)
}
pub ( crate ) fn map_reduce_sketch < S , F > ( train : & SvmScanner , featurizer : & Featurizer , filter : F , ) -> ( S , Vec < S > ) where S : Sketch , F : for < ' a > Fn ( DelimIter < ' a > ) -> bool + Sync + Send , { // add in an extra "global" feature at the end for overall numbers. let global = featurizer . nsparse ( ) ; let ( _ , stats ) = train . fold_reduce ( | | ( 0usize , vec ! [ LineSketch S default ( ) global 1 ] ) , | ( mut line_num , mut stats ) , mut word_iter | { if ! filter ( word_iter . clone ( ) ) { return ( line_num , stats ) ; } line_num += 1 ; // SAFETY: caller promised the first word is f64 in ascii // in SvmScanner guarantees let target : & str = unsafe { std :: str :: from_utf8_unchecked ( word_iter . next ( ) . expect ( " " ) ) } ; let target : f64 = target . parse ( ) . expect ( " " ) ; for h in word_iter . enumerate ( ) . flat_map ( | ( i , word ) | featurizer . sparse ( i , word ) ) { let stat = & mut stats [ h as usize ] ; stat . update ( line_num , target ) ; } let stat = & mut stats [ global ] ; stat . update ( line_num , target ) ; ( line_num , stats ) } , | ( _ , mut l_stats ) , ( _ , r_stats ) | { for ( l_stat , r_stat ) in l_stats . iter_mut ( ) . zip ( r_stats . iter ( ) ) { l_stat . merge ( r_stat ) ; } ( 0 , l_stats ) } , ) ; let mut all : Vec < _ > = stats . into_iter ( ) . map ( | line_sketch | line_sketch . sketch ) . collect ( ) ; let global = all . pop ( ) . unwrap ( ) ; ( global , all ) }
Map reduce to collect per-feature statistics over lines.
Map reduce to collect per - feature statistics over lines .
train
&SvmScanner
featurizer
&Featurizer
filter
F