polars
polars copied to clipboard
improving the 0.29.0 JsonReader error message for the polars-core crate when required feature flags are missing during schema inference
Hello!
Thanks for this awesome repo and the great community!
This pr shows the user what may be missing within their Cargo.toml with a helpful suggestion after hitting errors with the new 0.29.0 JsonReader support. Without the additional polars-core feature flags (mentioned in the logged error message), the new 0.29.0 infer schema support crashes rust binaries that are not compiled with these polars-core feature flags on these lines:
- https://github.com/pola-rs/polars/blame/6334057879948d372a2bc718d4af0223c216edc7/polars/polars-core/src/datatypes/field.rs#L142
- https://github.com/pola-rs/polars/pull/8411/files#diff-586ff9ed1f826ead771508d6aacd9a25988747372a47eff779390f05f54ec7f2R215
Without the logged feature flags (I think only dtype-struct is actually required now), I was seeing this error with my previously-stable 0.28.0 JsonReader schema inference:
thread 'main' panicked at 'Arrow datatype Struct([Field { name: "date", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "date_utc_str", data_type: Utf8, is_nullable: true, metadata: {} }, Field { name: "ds", data_type: Utf8, is_nullable: true, metadata: {} }, Field { name: "q.last", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "q.change", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "q.v", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "q.o", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "q.h", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "q.l", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "q.c", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "q.b", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "q.a", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "q.chg_p", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "q.avg_vlm", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "q.last_vlm", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "q.trade_date", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "q.pc", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "q.w52h", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "q.w52l", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "q.b_s", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "q.b_date", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "q.a_s", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "q.a_date", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "atm_d", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "strike", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "change", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "volume", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "open", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "high", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "low", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "close", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "bid", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "ask", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "last", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "cp", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "a_vol", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "l_vol", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "pc", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "bid_s", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "ask_s", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "oi", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "ct_size", data_type: Int64, is_nullable: true, metadata: {} }, Field { name: "expdate", data_type: Utf8, is_nullable: true, metadata: {} }, Field { name: "ot", data_type: Utf8, is_nullable: true, metadata: {} }, Field { name: "delta", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "gamma", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "theta", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "vega", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "rho", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "phi", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "bid_iv", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "mid_iv", data_type: Float64, is_nullable: true, metadata: {} }, Field { name: "ask_iv", data_type: Float64, is_nullable: true, metadata: {} }]) not supported by Polars. You probably need to activate that data-type feature.', /home/jay/.cargo/registry/src/github.com-1ecc6299db9ec823/polars-core-0.29.0/src/datatypes/field.rs:156:19 stack backtrace: 0: rust_begin_unwind at /rustc/2c8cc343237b8f7d5a3c3703e3a87f2eb2c54a74/library/std/src/panicking.rs:575:5 1: core::panicking::panic_fmt at /rustc/2c8cc343237b8f7d5a3c3703e3a87f2eb2c54a74/library/core/src/panicking.rs:64:14 2: polars_core::datatypes::field::<impl core::convert::From<&arrow2::datatypes::DataType> for polars_core::datatypes::dtype::DataType>::from at /home/jay/.cargo/registry/src/github.com-1ecc6299db9ec823/polars-core-0.29.0/src/datatypes/field.rs:156:19 3: <polars_io::json::JsonReader<R> as polars_io::SerReader<R>>::finish::{{closure}}::{{closure}} at /home/jay/.cargo/registry/src/github.com-1ecc6299db9ec823/polars-io-0.29.0/src/json.rs:215:43 4: core::result::Result<T,E>::map at /rustc/2c8cc343237b8f7d5a3c3703e3a87f2eb2c54a74/library/core/src/result.rs:774:25 5: <polars_io::json::JsonReader<R> as polars_io::SerReader<R>>::finish::{{closure}} at /home/jay/.cargo/registry/src/github.com-1ecc6299db9ec823/polars-io-0.29.0/src/json.rs:213:29
Thanks again for this project!