/// Optimizes the dictionaries for a column
Optimizes the dictionaries for a column
Optimizes the dictionaries for a column
fn optimize_dict_col(
    col: &ArrayRef,
    key_type: &DataType,
    value_type: &DataType,
) -> Result<ArrayRef> {
    if key_type != &DataType::Int32 {
        return Err(ArrowError::NotYetImplemented(format!(
            "truncating non-Int32 dictionaries not supported: {}",
            key_type
        )));
    }
    if value_type != &DataType::Utf8 {
        return Err(ArrowError::NotYetImplemented(format!(
            "truncating non-string dictionaries not supported: {}",
            value_type
        )));
    }
    let col = col
        .as_any()
        .downcast_ref::<DictionaryArray<Int32Type>>()
        .expect("unexpected datatype");
    let keys = col.keys();
    let values = col.values();
    let values = values
        .as_any()
        .downcast_ref::<StringArray>()
        .expect("unexpected datatype");
    // The total length of the resulting values array
    let mut values_len = 0_usize;
    // Keys that appear in the values array
    // Use a BTreeSet to preserve the order of the dictionary
    let mut used_keys = BTreeSet::new();
    for key in keys.iter().flatten() {
        if used_keys.insert(key) {
            values_len += values.value_length(key as usize) as usize;
        }
    }
    // Then perform deduplication
    let mut new_dictionary = StringDictionary::with_capacity(used_keys.len(), values_len);
    let mut old_to_new_idx: HashMap<i32, i32> = HashMap::with_capacity(used_keys.len());
    for key in used_keys {
        let new_key = new_dictionary.lookup_value_or_insert(values.value(key as usize));
        old_to_new_idx.insert(key, new_key);
    }
    let new_keys = keys.iter().map(|x| match x {
        Some(x) => *old_to_new_idx.get(&x).expect("no mapping found"),
        None => -1,
    });
    let offset = keys.data().offset();
    let nulls = keys
        .data()
        .null_buffer()
        .map(|buffer| buffer.bit_slice(offset, keys.len()));
    Ok(Arc::new(new_dictionary.to_arrow(new_keys, nulls)))
}
fn optimize_dict_col ( col : & ArrayRef , key_type : & DataType , value_type : & DataType , ) -> Result < ArrayRef > { if key_type != & DataType :: Int32 { return Err ( ArrowError :: NotYetImplemented ( format ! ( " " key_type ) ) ) ; } if value_type != & DataType :: Utf8 { return Err ( ArrowError :: NotYetImplemented ( format ! ( " " value_type ) ) ) ; } let col = col . as_any ( ) . downcast_ref :: < DictionaryArray < Int32Type > > ( ) . expect ( " " ) ; let keys = col . keys ( ) ; let values = col . values ( ) ; let values = values . as_any ( ) . downcast_ref :: < StringArray > ( ) . expect ( " " ) ; // The total length of the resulting values array let mut values_len = 0_usize ; // Keys that appear in the values array // Use a BTreeSet to preserve the order of the dictionary let mut used_keys = BTreeSet :: new ( ) ; for key in keys . iter ( ) . flatten ( ) { if used_keys . insert ( key ) { values_len += values . value_length ( key as usize ) as usize ; } } // Then perform deduplication let mut new_dictionary = StringDictionary :: with_capacity ( used_keys . len ( ) , values_len ) ; let mut old_to_new_idx : HashMap < i32 , i32 > = HashMap :: with_capacity ( used_keys . len ( ) ) ; for key in used_keys { let new_key = new_dictionary . lookup_value_or_insert ( values . value ( key as usize ) ) ; old_to_new_idx . insert ( key , new_key ) ; } let new_keys = keys . iter ( ) . map ( | x | match x { Some ( x ) => * old_to_new_idx . get ( & x ) . expect ( " " ) , None => - 1 , } ) ; let offset = keys . data ( ) . offset ( ) ; let nulls = keys . data ( ) . null_buffer ( ) . map ( | buffer | buffer . bit_slice ( offset , keys . len ( ) ) ) ; Ok ( Arc :: new ( new_dictionary . to_arrow ( new_keys , nulls ) ) ) }
Optimizes the dictionaries for a column
Optimizes the dictionaries for a column
col
&ArrayRef
key_type
&DataType
value_type
&DataType