/// Takes batches of data and the columns that make up the primary key. Computes the number of
/// distinct values for each primary key column across all batches, also known as "cardinality".
/// Used to determine sort order.
Takes batches of data and the columns that make up the primary key. Computes the number of
distinct values for each primary key column across all batches, also known as "cardinality".
Used to determine sort order.
Takes batches of data and the columns that make up the primary key . Computes the number of distinct values for each primary key column across all batches also known as " cardinality " . Used to determine sort order .
fn distinct_counts<'a>(
    batches: impl Iterator<Item = &'a RecordBatch>,
    primary_key: &[&str],
) -> HashMap<String, u64> {
    let mut distinct_values_across_batches = HashMap::with_capacity(primary_key.len());
    for batch in batches {
        for (column, distinct_values) in distinct_values(batch, primary_key) {
            let set = distinct_values_across_batches
                .entry(column)
                .or_insert_with(HashSet::new);
            set.extend(distinct_values.into_iter());
        }
    }
    distinct_values_across_batches
        .into_iter()
        .map(|(column, distinct_values)| {
            let count = distinct_values
                .len()
                .try_into()
                .expect("usize -> u64 overflow");
            (column, count)
        })
        .collect()
}
fn distinct_counts < ' a > ( batches : impl Iterator < Item = & ' a RecordBatch > , primary_key : & [ & str ] , ) -> HashMap < String , u64 > { let mut distinct_values_across_batches = HashMap :: with_capacity ( primary_key . len ( ) ) ; for batch in batches { for ( column , distinct_values ) in distinct_values ( batch , primary_key ) { let set = distinct_values_across_batches . entry ( column ) . or_insert_with ( HashSet :: new ) ; set . extend ( distinct_values . into_iter ( ) ) ; } } distinct_values_across_batches . into_iter ( ) . map ( | ( column , distinct_values ) | { let count = distinct_values . len ( ) . try_into ( ) . expect ( " " ) ; ( column , count ) } ) . collect ( ) }
Takes batches of data and the columns that make up the primary key.
Takes batches of data and the columns that make up the primary key .
batches
impl Iterator<Item = &'a RecordBatch>
primary_key
&[&str]