/// Infer the fields of a JSON file by reading all items from the JSON Value Iterator.
Infer the fields of a JSON file by reading all items from the JSON Value Iterator.
Infer the fields of a JSON file by reading all items from the JSON Value Iterator .
pub fn infer_json_schema_from_iterator<I>(value_iter: I) -> Result<Schema>
where
    I: Iterator<Item = Result<Value>>,
{
    let mut values: HashMap<String, HashSet<DataType>> = HashMap::new();
    for record in value_iter {
        match record? {
            Value::Object(map) => {
                let res = map.iter().try_for_each(|(k, v)| {
                    match v {
                        Value::Array(a) => {
                            // collect the data types in array
                            let types: Result<Vec<Option<&DataType>>> = a
                                .iter()
                                .map(|a| match a {
                                    Value::Null => Ok(None),
                                    Value::Number(n) => {
                                        if n.is_i64() {
                                            Ok(Some(&DataType::Int64))
                                        } else {
                                            Ok(Some(&DataType::Float64))
                                        }
                                    }
                                    Value::Bool(_) => Ok(Some(&DataType::Boolean)),
                                    Value::String(_) => Ok(Some(&DataType::Utf8)),
                                    Value::Array(_) | Value::Object(_) => {
                                        Err(ArrowError::NotYetImplemented(
                                            "Nested lists and structs not supported".to_string(),
                                        ))
                                    }
                                })
                                .collect();
                            match types {
                                Ok(types) => {
                                    // unwrap the Option and discard None values (from
                                    // JSON nulls)
                                    let mut types: Vec<&DataType> =
                                        types.into_iter().flatten().collect();
                                    types.dedup();
                                    // if a record contains only nulls, it is not
                                    // added to values
                                    if !types.is_empty() {
                                        let dt = coerce_data_type(&types);
                                        if values.contains_key(k) {
                                            let x = values.get_mut(k).unwrap();
                                            x.insert(DataType::List(Box::new(Field::new(
                                                "item", dt, true,
                                            ))));
                                        } else {
                                            // create hashset and add value type
                                            let mut hs = HashSet::new();
                                            hs.insert(DataType::List(Box::new(Field::new(
                                                "item", dt, true,
                                            ))));
                                            values.insert(k.to_string(), hs);
                                        }
                                    }
                                    Ok(())
                                }
                                Err(e) => Err(e),
                            }
                        }
                        Value::Bool(_) => {
                            if values.contains_key(k) {
                                let x = values.get_mut(k).unwrap();
                                x.insert(DataType::Boolean);
                            } else {
                                // create hashset and add value type
                                let mut hs = HashSet::new();
                                hs.insert(DataType::Boolean);
                                values.insert(k.to_string(), hs);
                            }
                            Ok(())
                        }
                        Value::Null => {
                            // do nothing, we treat json as nullable by default when
                            // inferring
                            Ok(())
                        }
                        Value::Number(n) => {
                            if n.is_f64() {
                                if values.contains_key(k) {
                                    let x = values.get_mut(k).unwrap();
                                    x.insert(DataType::Float64);
                                } else {
                                    // create hashset and add value type
                                    let mut hs = HashSet::new();
                                    hs.insert(DataType::Float64);
                                    values.insert(k.to_string(), hs);
                                }
                            } else {
                                // default to i64
                                if values.contains_key(k) {
                                    let x = values.get_mut(k).unwrap();
                                    x.insert(DataType::Int64);
                                } else {
                                    // create hashset and add value type
                                    let mut hs = HashSet::new();
                                    hs.insert(DataType::Int64);
                                    values.insert(k.to_string(), hs);
                                }
                            }
                            Ok(())
                        }
                        Value::String(_) => {
                            if values.contains_key(k) {
                                let x = values.get_mut(k).unwrap();
                                x.insert(DataType::Utf8);
                            } else {
                                // create hashset and add value type
                                let mut hs = HashSet::new();
                                hs.insert(DataType::Utf8);
                                values.insert(k.to_string(), hs);
                            }
                            Ok(())
                        }
                        Value::Object(_) => Err(ArrowError::NotYetImplemented(
                            "Inferring schema from nested JSON structs currently not supported"
                                .to_string(),
                        )),
                    }
                });
                match res {
                    Ok(()) => {}
                    Err(e) => return Err(e),
                }
            }
            value => {
                return Err(ArrowError::Other(format!(
                    "Expected JSON record to be an object, found {:?}",
                    value
                )));
            }
        };
    }
    Ok(generate_schema(values))
}
pub fn infer_json_schema_from_iterator < I > ( value_iter : I ) -> Result < Schema > where I : Iterator < Item = Result < Value > > , { let mut values : HashMap < String , HashSet < DataType > > = HashMap :: new ( ) ; for record in value_iter { match record ? { Value :: Object ( map ) => { let res = map . iter ( ) . try_for_each ( | ( k , v ) | { match v { Value :: Array ( a ) => { // collect the data types in array let types : Result < Vec < Option < & DataType > > > = a . iter ( ) . map ( | a | match a { Value :: Null => Ok ( None ) , Value :: Number ( n ) => { if n . is_i64 ( ) { Ok ( Some ( & DataType :: Int64 ) ) } else { Ok ( Some ( & DataType :: Float64 ) ) } } Value :: Bool ( _ ) => Ok ( Some ( & DataType :: Boolean ) ) , Value :: String ( _ ) => Ok ( Some ( & DataType :: Utf8 ) ) , Value :: Array ( _ ) | Value :: Object ( _ ) => { Err ( ArrowError :: NotYetImplemented ( " " . to_string ( ) , ) ) } } ) . collect ( ) ; match types { Ok ( types ) => { // unwrap the Option and discard None values (from // JSON nulls) let mut types : Vec < & DataType > = types . into_iter ( ) . flatten ( ) . collect ( ) ; types . dedup ( ) ; // if a record contains only nulls, it is not // added to values if ! types . is_empty ( ) { let dt = coerce_data_type ( & types ) ; if values . contains_key ( k ) { let x = values . get_mut ( k ) . unwrap ( ) ; x . insert ( DataType :: List ( Box :: new ( Field :: new ( " " , dt , true , ) ) ) ) ; } else { // create hashset and add value type let mut hs = HashSet :: new ( ) ; hs . insert ( DataType :: List ( Box :: new ( Field :: new ( " " , dt , true , ) ) ) ) ; values . insert ( k . to_string ( ) , hs ) ; } } Ok ( ( ) ) } Err ( e ) => Err ( e ) , } } Value :: Bool ( _ ) => { if values . contains_key ( k ) { let x = values . get_mut ( k ) . unwrap ( ) ; x . insert ( DataType :: Boolean ) ; } else { // create hashset and add value type let mut hs = HashSet :: new ( ) ; hs . insert ( DataType :: Boolean ) ; values . insert ( k . to_string ( ) , hs ) ; } Ok ( ( ) ) } Value :: Null => { // do nothing, we treat json as nullable by default when // inferring Ok ( ( ) ) } Value :: Number ( n ) => { if n . is_f64 ( ) { if values . contains_key ( k ) { let x = values . get_mut ( k ) . unwrap ( ) ; x . insert ( DataType :: Float64 ) ; } else { // create hashset and add value type let mut hs = HashSet :: new ( ) ; hs . insert ( DataType :: Float64 ) ; values . insert ( k . to_string ( ) , hs ) ; } } else { // default to i64 if values . contains_key ( k ) { let x = values . get_mut ( k ) . unwrap ( ) ; x . insert ( DataType :: Int64 ) ; } else { // create hashset and add value type let mut hs = HashSet :: new ( ) ; hs . insert ( DataType :: Int64 ) ; values . insert ( k . to_string ( ) , hs ) ; } } Ok ( ( ) ) } Value :: String ( _ ) => { if values . contains_key ( k ) { let x = values . get_mut ( k ) . unwrap ( ) ; x . insert ( DataType :: Utf8 ) ; } else { // create hashset and add value type let mut hs = HashSet :: new ( ) ; hs . insert ( DataType :: Utf8 ) ; values . insert ( k . to_string ( ) , hs ) ; } Ok ( ( ) ) } Value :: Object ( _ ) => Err ( ArrowError :: NotYetImplemented ( " " . to_string ( ) , ) ) , } } ) ; match res { Ok ( ( ) ) => { } Err ( e ) => return Err ( e ) , } } value => { return Err ( ArrowError :: Other ( format ! ( " " value ) ) ) ; } } ; } Ok ( generate_schema ( values ) ) }
Infer the fields of a JSON file by reading all items from the JSON Value Iterator.
Infer the fields of a JSON file by reading all items from the JSON Value Iterator .
value_iter
I