Daft
Daft copied to clipboard
Eventual-Inc
[1] 2941192 IOT instruction (core dumped)
Describe the bug
if show return result size is big , will produce core dump，
To Reproduce
No response
Expected behavior
No response
Component(s)
Other
Additional context
No response
if show return result size is big , will produce core dump，
Could you elaborate a little here - what's in the show result? Maybe a .schema() would be useful
@desmondcheongzx Thank you very much for your reply. I have debugged the coredump file. It seems to be related to the fact that I am reading lance. I'm already trying to fix it.
gdb statck
0 __pthread_kill_implementation (threadid=<optimized out>, signo=signo@entry=6, no_tid=no_tid@entry=0) at ./nptl/pthread_kill.c:44
#1 0x00007fd794d80f4f in __pthread_kill_internal (signo=6, threadid=<optimized out>) at ./nptl/pthread_kill.c:78
#2 0x00007fd794d31fb2 in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26
#3 0x00007fd794d1c472 in __GI_abort () at ./stdlib/abort.c:79
#4 0x000000000049f959 in fatal_error_exit (status=-1) at /usr/local/src/conda/python-3.9.19/Python/pylifecycle.c:2201
#5 fatal_error (stream=stream@entry=0x7fd794ec9680 <_IO_2_1_stderr_>, header=header@entry=0, prefix=prefix@entry=0x0, msg=msg@entry=0x0, status=status@entry=-1)
at /usr/local/src/conda/python-3.9.19/Python/pylifecycle.c:2219
#6 0x00000000004a0223 in _Py_FatalErrorFormat (func=func@entry=0x68af50 <__func__.1.lto_priv.10> "PyGILState_Release",
format=format@entry=0x61e1c0 "thread state %p must be current when releasing") at /usr/local/src/conda/python-3.9.19/Python/pylifecycle.c:2335
#7 0x000000000046538d in PyGILState_Release (oldstate=PyGILState_UNLOCKED) at /usr/local/src/conda/python-3.9.19/Python/pystate.c:1418
#8 0x00007fd6e49f4ad4 in pyo3::gil::{impl#1}::drop (self=0x7fd743df6edc) at src/gil.rs:242
#9 0x00007fd6e49d9f97 in core::ptr::drop_in_place<pyo3::gil::GILGuard> ()
at /root/.rustup/toolchains/1.86.0-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:523
#10 0x00007fd6dc24aa46 in pyo3::marker::Python::with_gil<lance::executor::{impl#0}::spawn::{closure_env#1}<lance::reader::{impl#1}::next::{async_block_env#0}>, core::result::Result<core::option::Option<core::result::Result<arrow_array::record_batch::RecordBatch, lance_core::error::Error>>, pyo3::err::PyErr>> (f=...)
at /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/pyo3-0.24.2/src/marker.rs:412
#11 0x00007fd6dc315f1b in lance::executor::BackgroundExecutor::spawn<lance::reader::{impl#1}::next::{async_block_env#0}> (
self=0x7fd6e8cad488 <lance::RT::h85a125318f69bcbc>, py=..., task=...) at src/executor.rs:62
#12 0x00007fd6dc5c3535 in lance::reader::{impl#1}::next (self=0x7fd5e8006b60) at src/reader.rs:61
#13 0x00007fd6e5f9599e in alloc::boxed::iter::{impl#0}::next<(dyn arrow_array::record_batch::RecordBatchReader<Item=core::result::Result<arrow_array::record_batch::RecordBatch, arrow_schema::error::ArrowError>> + core::marker::Send), alloc::alloc::Global> (self=0x7fd64402bc20)
at /root/.rustup/toolchains/1.86.0-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed/iter.rs:21
#14 0x00007fd6e5f16ec4 in arrow_array::ffi_stream::ExportedArrayStream::get_next (self=0x7fd743df7480, out=0x7fd743df74f0) at src/ffi_stream.rs:240
#15 0x00007fd6e5f1670d in arrow_array::ffi_stream::get_next (stream=0x7fd5e8008608, array=0x7fd743df74f0) at src/ffi_stream.rs:138
#16 0x00007fd77d2a5649 in arrow::(anonymous namespace)::ArrayStreamBatchReader<false>::ReadNext(std::shared_ptr<arrow::RecordBatch>*) ()
from /data00/code/emr/Daft/.venv/lib/python3.9/site-packages/pyarrow/libarrow.so.1900
#17 0x00007fd7863423a9 in __pyx_pw_7pyarrow_3lib_17RecordBatchReader_7read_next_batch(_object*, _object* const*, long, _object*) ()
from /data00/code/emr/Daft/.venv/lib/python3.9/site-packages/pyarrow/lib.cpython-39-x86_64-linux-gnu.so
#18 0x00007fd786297858 in __pyx_pf_7pyarrow_3lib_17RecordBatchReader_4__next__(__pyx_obj_7pyarrow_3lib_RecordBatchReader*) ()
from /data00/code/emr/Daft/.venv/lib/python3.9/site-packages/pyarrow/lib.cpython-39-x86_64-linux-gnu.so
#19 0x00000000004e9493 in _PyEval_EvalFrameDefault (tstate=<optimized out>, f=0x7fd74bd75380, throwflag=<optimized out>)
at /usr/local/src/conda/python-3.9.19/Python/ceval.c:2206
#20 0x000000000050bb1c in _PyEval_EvalFrame (throwflag=<optimized out>, f=0x7fd74bd75380, tstate=0x7fd5e8001ab0)
at /usr/local/src/conda/python-3.9.19/Include/internal/pycore_ceval.h:40
--Type <RET> for more, q to quit, c to continue without paging--
#21 gen_send_ex (gen=0x7fd76cb01cf0, arg=<optimized out>, exc=<optimized out>, closing=<optimized out>) at /usr/local/src/conda/python-3.9.19/Objects/genobject.c:215
#22 0x00000000004e8232 in _PyEval_EvalFrameDefault (tstate=<optimized out>, f=0x7fd74bd74580, throwflag=<optimized out>)
at /usr/local/src/conda/python-3.9.19/Python/ceval.c:3308
#23 0x000000000050bb1c in _PyEval_EvalFrame (throwflag=<optimized out>, f=0x7fd74bd74580, tstate=0x7fd5e8001ab0)
at /usr/local/src/conda/python-3.9.19/Include/internal/pycore_ceval.h:40
#24 gen_send_ex (gen=0x7fd76cb01e40, arg=<optimized out>, exc=<optimized out>, closing=<optimized out>) at /usr/local/src/conda/python-3.9.19/Objects/genobject.c:215
#25 0x000000000050a5ee in PyIter_Next (iter=<optimized out>, iter=<optimized out>) at /usr/local/src/conda/python-3.9.19/Objects/abstract.c:2702
#26 0x00007fd7932f9a0e in pyo3::types::iterator::<impl pyo3::instance::Borrowed<pyo3::types::iterator::PyIterator>>::next::h4dc2846ee6af49de ()
at src/types/iterator.rs:104
#27 0x00007fd78d53694d in pyo3::types::iterator::<impl core::iter::traits::iterator::Iterator for pyo3::instance::Bound<pyo3::types::iterator::PyIterator>>::next::had949c810ba81738 () at /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/pyo3-0.25.1/src/types/iterator.rs:88
#28 0x00007fd78d52e19e in daft_micropartition::python::read_pyfunc_into_table_iter::{{closure}}::{{closure}}::{{closure}}::h2deec8bcabd7d530 ()
at src/daft-micropartition/src/python.rs:1141
#29 0x00007fd78d56f17e in pyo3::marker::Python::with_gil::hccaa57cf163c2250 ()
at /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/pyo3-0.25.1/src/marker.rs:410
#30 0x00007fd78d52e125 in daft_micropartition::python::read_pyfunc_into_table_iter::{{closure}}::{{closure}}::h9647ab1ab3d381dd ()
at src/daft-micropartition/src/python.rs:1140
#31 0x00007fd78d60594e in <core::iter::sources::from_fn::FromFn<F> as core::iter::traits::iterator::Iterator>::next::h19a837c52b2c66c7 ()
a
reproduce script
tm0 = time.time()
default_scan_options={
"with_row_id": True
}
dataset = daft.read_lance(url="s3://xxx/ds_4coewnmpfe59tgkklbol.lance/", io_config=io_config, default_scan_options=default_scan_options)
df = daft.sql("select * from dataset limit 20")
df.show(6)
tm1 = time.time()
print('cost：', tm1 - tm0)
