dask
"TypeError: h5py objects cannot be pickled" when applying map_blocks to dask array constructed from HDF5 dataset
Describe the issue:
Applying map_blocks to a Dask Array constructed by from_array from HDF5 dataset raises TypeError: h5py objects cannot be pickled. This only occurs when using distributed.
This maybe related to #860 and #861. In fact, the below minimal verifiable example is borrowed from test_h5py_serialize mentioned in #860.
Minimal Complete Verifiable Example:
# from dask.utils import SerializableLock
import distributed
import h5py
import dask.array as da
client = distributed.Client() # Error does not occur if this is commented out
fn = 'temporary_file_for_test.h5'
lock = True # lock = SerializableLock('hdf5') and lock = False also leads to an error.
with h5py.File(fn, mode='w') as f:
x = f.create_dataset('/group/x', shape=(4,), dtype='i4',
chunks=(2,))
x[:] = [1, 2, 3, 4]
with h5py.File(fn, mode='r') as f:
dset = f['/group/x']
x = da.from_array(dset, chunks=dset.chunks, lock=lock)
def func(array):
return array**2
res = da.map_blocks(func, x).compute()
Anything else we need to know?:
Traceback on Jupyter Lab looks like following:
2025-02-13 22:00:12,875 - distributed.protocol.pickle - ERROR - Failed to serialize <ToPickle: HighLevelGraph with 1 layers.
<dask.highlevelgraph.HighLevelGraph object at 0x7fc9a461f350>
0. 140504024525248
>.
Traceback (most recent call last):
File "[/home/sm69/.conda/envs/pyathena/lib/python3.13/site-packages/distributed/protocol/pickle.py", line 73](http://127.0.0.1:20201/home/sm69/.conda/envs/pyathena/lib/python3.13/site-packages/distributed/protocol/pickle.py#line=72), in dumps
result = cloudpickle.dumps(x, **dump_kwargs)
File "[/home/sm69/.conda/envs/pyathena/lib/python3.13/site-packages/cloudpickle/cloudpickle.py", line 1537](http://127.0.0.1:20201/home/sm69/.conda/envs/pyathena/lib/python3.13/site-packages/cloudpickle/cloudpickle.py#line=1536), in dumps
cp.dump(obj)
~~~~~~~^^^^^
File "[/home/sm69/.conda/envs/pyathena/lib/python3.13/site-packages/cloudpickle/cloudpickle.py", line 1303](http://127.0.0.1:20201/home/sm69/.conda/envs/pyathena/lib/python3.13/site-packages/cloudpickle/cloudpickle.py#line=1302), in dump
return super().dump(obj)
~~~~~~~~~~~~^^^^^
File "[/home/sm69/.conda/envs/pyathena/lib/python3.13/site-packages/h5py/_hl/base.py", line 366](http://127.0.0.1:20201/home/sm69/.conda/envs/pyathena/lib/python3.13/site-packages/h5py/_hl/base.py#line=365), in __getnewargs__
raise TypeError("h5py objects cannot be pickled")
TypeError: h5py objects cannot be pickled
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "[/home/sm69/.conda/envs/pyathena/lib/python3.13/site-packages/distributed/protocol/pickle.py", line 77](http://127.0.0.1:20201/home/sm69/.conda/envs/pyathena/lib/python3.13/site-packages/distributed/protocol/pickle.py#line=76), in dumps
result = cloudpickle.dumps(x, **dump_kwargs)
File "[/home/sm69/.conda/envs/pyathena/lib/python3.13/site-packages/cloudpickle/cloudpickle.py", line 1537](http://127.0.0.1:20201/home/sm69/.conda/envs/pyathena/lib/python3.13/site-packages/cloudpickle/cloudpickle.py#line=1536), in dumps
cp.dump(obj)
~~~~~~~^^^^^
File "[/home/sm69/.conda/envs/pyathena/lib/python3.13/site-packages/cloudpickle/cloudpickle.py", line 1303](http://127.0.0.1:20201/home/sm69/.conda/envs/pyathena/lib/python3.13/site-packages/cloudpickle/cloudpickle.py#line=1302), in dump
return super().dump(obj)
~~~~~~~~~~~~^^^^^
File "[/home/sm69/.conda/envs/pyathena/lib/python3.13/site-packages/h5py/_hl/base.py", line 366](http://127.0.0.1:20201/home/sm69/.conda/envs/pyathena/lib/python3.13/site-packages/h5py/_hl/base.py#line=365), in __getnewargs__
raise TypeError("h5py objects cannot be pickled")
TypeError: h5py objects cannot be pickled
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
File [~/.conda/envs/pyathena/lib/python3.13/site-packages/distributed/protocol/pickle.py:73](http://127.0.0.1:20201/~/.conda/envs/pyathena/lib/python3.13/site-packages/distributed/protocol/pickle.py#line=72), in dumps(x, buffer_callback, protocol)
72 buffers.clear()
---> 73 result = cloudpickle.dumps(x, **dump_kwargs)
74 except Exception:
File [~/.conda/envs/pyathena/lib/python3.13/site-packages/cloudpickle/cloudpickle.py:1537](http://127.0.0.1:20201/~/.conda/envs/pyathena/lib/python3.13/site-packages/cloudpickle/cloudpickle.py#line=1536), in dumps(obj, protocol, buffer_callback)
1536 cp = Pickler(file, protocol=protocol, buffer_callback=buffer_callback)
-> 1537 cp.dump(obj)
1538 return file.getvalue()
File [~/.conda/envs/pyathena/lib/python3.13/site-packages/cloudpickle/cloudpickle.py:1303](http://127.0.0.1:20201/~/.conda/envs/pyathena/lib/python3.13/site-packages/cloudpickle/cloudpickle.py#line=1302), in Pickler.dump(self, obj)
1302 try:
-> 1303 return super().dump(obj)
1304 except RuntimeError as e:
File [~/.conda/envs/pyathena/lib/python3.13/site-packages/h5py/_hl/base.py:366](http://127.0.0.1:20201/~/.conda/envs/pyathena/lib/python3.13/site-packages/h5py/_hl/base.py#line=365), in HLObject.__getnewargs__(self)
357 """Disable pickle.
358
359 Handles for HDF5 objects can't be reliably deserialised, because the
(...)
364 limitations, look at the h5pickle project on PyPI.
365 """
--> 366 raise TypeError("h5py objects cannot be pickled")
TypeError: h5py objects cannot be pickled
During handling of the above exception, another exception occurred:
TypeError Traceback (most recent call last)
File [~/.conda/envs/pyathena/lib/python3.13/site-packages/distributed/protocol/serialize.py:366](http://127.0.0.1:20201/~/.conda/envs/pyathena/lib/python3.13/site-packages/distributed/protocol/serialize.py#line=365), in serialize(x, serializers, on_error, context, iterate_collection)
365 try:
--> 366 header, frames = dumps(x, context=context) if wants_context else dumps(x)
367 header["serializer"] = name
File [~/.conda/envs/pyathena/lib/python3.13/site-packages/distributed/protocol/serialize.py:78](http://127.0.0.1:20201/~/.conda/envs/pyathena/lib/python3.13/site-packages/distributed/protocol/serialize.py#line=77), in pickle_dumps(x, context)
76 writeable.append(not f.readonly)
---> 78 frames[0] = pickle.dumps(
79 x,
80 buffer_callback=buffer_callback,
81 protocol=context.get("pickle-protocol", None) if context else None,
82 )
83 header = {
84 "serializer": "pickle",
85 "writeable": tuple(writeable),
86 }
File [~/.conda/envs/pyathena/lib/python3.13/site-packages/distributed/protocol/pickle.py:77](http://127.0.0.1:20201/~/.conda/envs/pyathena/lib/python3.13/site-packages/distributed/protocol/pickle.py#line=76), in dumps(x, buffer_callback, protocol)
76 buffers.clear()
---> 77 result = cloudpickle.dumps(x, **dump_kwargs)
78 except Exception:
File [~/.conda/envs/pyathena/lib/python3.13/site-packages/cloudpickle/cloudpickle.py:1537](http://127.0.0.1:20201/~/.conda/envs/pyathena/lib/python3.13/site-packages/cloudpickle/cloudpickle.py#line=1536), in dumps(obj, protocol, buffer_callback)
1536 cp = Pickler(file, protocol=protocol, buffer_callback=buffer_callback)
-> 1537 cp.dump(obj)
1538 return file.getvalue()
File [~/.conda/envs/pyathena/lib/python3.13/site-packages/cloudpickle/cloudpickle.py:1303](http://127.0.0.1:20201/~/.conda/envs/pyathena/lib/python3.13/site-packages/cloudpickle/cloudpickle.py#line=1302), in Pickler.dump(self, obj)
1302 try:
-> 1303 return super().dump(obj)
1304 except RuntimeError as e:
File [~/.conda/envs/pyathena/lib/python3.13/site-packages/h5py/_hl/base.py:366](http://127.0.0.1:20201/~/.conda/envs/pyathena/lib/python3.13/site-packages/h5py/_hl/base.py#line=365), in HLObject.__getnewargs__(self)
357 """Disable pickle.
358
359 Handles for HDF5 objects can't be reliably deserialised, because the
(...)
364 limitations, look at the h5pickle project on PyPI.
365 """
--> 366 raise TypeError("h5py objects cannot be pickled")
TypeError: h5py objects cannot be pickled
The above exception was the direct cause of the following exception:
TypeError Traceback (most recent call last)
Cell In[1], line 19
17 def func(array):
18 return array**2
---> 19 res = da.map_blocks(func, x).compute()
File [~/.conda/envs/pyathena/lib/python3.13/site-packages/dask/base.py:372](http://127.0.0.1:20201/~/.conda/envs/pyathena/lib/python3.13/site-packages/dask/base.py#line=371), in DaskMethodsMixin.compute(self, **kwargs)
348 def compute(self, **kwargs):
349 """Compute this dask collection
350
351 This turns a lazy Dask collection into its in-memory equivalent.
(...)
370 dask.compute
371 """
--> 372 (result,) = compute(self, traverse=False, **kwargs)
373 return result
File [~/.conda/envs/pyathena/lib/python3.13/site-packages/dask/base.py:660](http://127.0.0.1:20201/~/.conda/envs/pyathena/lib/python3.13/site-packages/dask/base.py#line=659), in compute(traverse, optimize_graph, scheduler, get, *args, **kwargs)
657 postcomputes.append(x.__dask_postcompute__())
659 with shorten_traceback():
--> 660 results = schedule(dsk, keys, **kwargs)
662 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
File [~/.conda/envs/pyathena/lib/python3.13/site-packages/distributed/protocol/serialize.py:392](http://127.0.0.1:20201/~/.conda/envs/pyathena/lib/python3.13/site-packages/distributed/protocol/serialize.py#line=391), in serialize(x, serializers, on_error, context, iterate_collection)
390 except Exception:
391 raise TypeError(msg) from exc
--> 392 raise TypeError(msg, str_x) from exc
393 else: # pragma: nocover
394 raise ValueError(f"{on_error=}; expected 'message' or 'raise'")
TypeError: ('Could not serialize object of type HighLevelGraph', '<ToPickle: HighLevelGraph with 1 layers.\n<dask.highlevelgraph.HighLevelGraph object at 0x7fc9a461f350>\n 0. 140504024525248\n>')
Environment:
- Dask version: 2024.12.1
- Python version: 3.13.1
- Operating System: linux
- Install method (conda, pip, source): conda (mamba, actually)
The issues you are referring to are 8 years old, so I would be surprised if they are related.
This is likely a regression caused by some of the serialisation changes that have happened lately. cc @fjetter
Here is another trigger for (I believe) the same bug, and maybe a clue. The maybe clue is that the following code works as expected with xarray<2025.8.0. With a newer xarray the result is ultimately the same TypeError (see below). I see that @sanghyukmoon does not import xarray, but is it in your environment?
import h5netcdf
import fsspec
import xarray as xr
from dask.distributed import Client
client = Client()
fs = fsspec.filesystem("memory")
values = list(range(32))
path = "foo"
with fs.open(path, "wb") as g:
with h5netcdf.File(g, "w") as f:
f.dimensions = {"x": len(values)}
f.create_variable("data", ("x",), dtype="i4")
f.variables["data"][:] = values
f = fs.open(path)
future = client.submit(xr.open_dataset, f)
ds = future.result()
ds
Output
2025-09-10 12:15:28,210 - distributed.protocol.pickle - ERROR - Failed to serializeSize: 128B Dimensions: (x: 32) Dimensions without coordinates: x Data variables: data (x) int32 128B 0 1 2 3 4 5 6 7 8 9 ... 23 24 25 26 27 28 29 30 31. Traceback (most recent call last): File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/pickle.py", line 63, in dumps result = pickle.dumps(x, **dump_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ TypeError: cannot pickle 'module' object During handling of the above exception, another exception occurred:
Traceback (most recent call last): File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/pickle.py", line 68, in dumps pickler.dump(x) TypeError: cannot pickle 'module' object
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/pickle.py", line 80, in dumps result = cloudpickle.dumps(x, **dump_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/cloudpickle/cloudpickle.py", line 1537, in dumps cp.dump(obj) File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/cloudpickle/cloudpickle.py", line 1303, in dump return super().dump(obj) ^^^^^^^^^^^^^^^^^ File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/h5py/_hl/base.py", line 369, in getnewargs raise TypeError("h5py objects cannot be pickled") TypeError: h5py objects cannot be pickled 2025-09-10 12:15:28,213 - distributed.protocol.core - CRITICAL - Failed to deserialize Traceback (most recent call last): File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/core.py", line 175, in loads return msgpack.loads( ^^^^^^^^^^^^^^ File "msgpack/_unpacker.pyx", line 194, in msgpack._cmsgpack.unpackb File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/core.py", line 159, in _decode_default return merge_and_deserialize( ^^^^^^^^^^^^^^^^^^^^^^ File "/Users/icarroll/.local/share/uv/python/cpython-3.12.10-macos-aarch64-none/lib/python3.12/contextlib.py", line 81, in inner return func(*args, **kwds) ^^^^^^^^^^^^^^^^^^^ File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/serialize.py", line 525, in merge_and_deserialize return deserialize(header, merged_frames, deserializers=deserializers) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/serialize.py", line 452, in deserialize return loads(header, frames) ^^^^^^^^^^^^^^^^^^^^^ File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/serialize.py", line 195, in serialization_error_loads raise TypeError(msg) TypeError: Could not serialize object of type Dataset Traceback (most recent call last): File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/pickle.py", line 63, in dumps result = pickle.dumps(x, **dump_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ TypeError: cannot pickle 'module' object
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/pickle.py", line 68, in dumps pickler.dump(x) TypeError: cannot pickle 'module' object
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/serialize.py", line 366, in serialize header, frames = dumps(x, context=context) if wants_context else dumps(x) ^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/serialize.py", line 78, in pickle_dumps frames[0] = pickle.dumps( ^^^^^^^^^^^^^ File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/pickle.py", line 80, in dumps result = cloudpickle.dumps(x, **dump_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/cloudpickle/cloudpickle.py", line 1537, in dumps cp.dump(obj) File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/cloudpickle/cloudpickle.py", line 1303, in dump return super().dump(obj) ^^^^^^^^^^^^^^^^^ File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/h5py/_hl/base.py", line 369, in getnewargs raise TypeError("h5py objects cannot be pickled") TypeError: h5py objects cannot be pickled
TypeError Traceback (most recent call last) Cell In[1], line 19 17 f = fs.open(path) 18 future = client.submit(xr.open_dataset, f) ---> 19 ds = future.result() 20 ds
File ~/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/client.py:395, in Future.result(self, timeout) 393 self._verify_initialized() 394 with shorten_traceback(): --> 395 return self.client.sync(self._result, callback_timeout=timeout)
File ~/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/utils_comm.py:416, in retry_operation(coro, operation, *args, **kwargs) 410 retry_delay_min = parse_timedelta( 411 dask.config.get("distributed.comm.retry.delay.min"), default="s" 412 ) 413 retry_delay_max = parse_timedelta( 414 dask.config.get("distributed.comm.retry.delay.max"), default="s" 415 ) --> 416 return await retry( 417 partial(coro, *args, **kwargs), 418 count=retry_count, 419 delay_min=retry_delay_min, 420 delay_max=retry_delay_max, 421 operation=operation, 422 )
File ~/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/utils_comm.py:395, in retry(coro, count, delay_min, delay_max, jitter_fraction, retry_on_exceptions, operation) 393 delay *= 1 + random.random() * jitter_fraction 394 await asyncio.sleep(delay) --> 395 return await coro()
File ~/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/core.py:1259, in PooledRPCCall.getattr.
.send_recv_from_rpc(**kwargs) 1257 prev_name, comm.name = comm.name, "ConnectionPool." + key 1258 try: -> 1259 return await send_recv(comm=comm, op=key, **kwargs) 1260 finally: 1261 self.pool.reuse(self.addr, comm) File ~/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/core.py:1018, in send_recv(comm, reply, serializers, deserializers, **kwargs) 1016 await comm.write(msg, serializers=serializers, on_error="raise") 1017 if reply: -> 1018 response = await comm.read(deserializers=deserializers) 1019 else: 1020 response = None
File ~/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/comm/tcp.py:248, in TCP.read(self, deserializers) 246 else: 247 try: --> 248 msg = await from_frames( 249 frames, 250 deserialize=self.deserialize, 251 deserializers=deserializers, 252 allow_offload=self.allow_offload, 253 ) 254 except EOFError: 255 # Frames possibly garbled or truncated by communication error 256 self.abort()
File ~/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/comm/utils.py:78, in from_frames(frames, deserialize, deserializers, allow_offload) 76 res = await offload(_from_frames) 77 else: ---> 78 res = _from_frames() 80 return res
File ~/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/comm/utils.py:61, in from_frames.
._from_frames() 59 def _from_frames(): 60 try: ---> 61 return protocol.loads( 62 frames, deserialize=deserialize, deserializers=deserializers 63 ) 64 except EOFError: 65 if size > 1000: File ~/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/core.py:175, in loads(frames, deserialize, deserializers) 172 return pickle.loads(sub_header["pickled-obj"], buffers=sub_frames) 173 return msgpack_decode_default(obj) --> 175 return msgpack.loads( 176 frames[0], object_hook=_decode_default, use_list=False, **msgpack_opts 177 ) 179 except Exception: 180 logger.critical("Failed to deserialize", exc_info=True)
File msgpack/_unpacker.pyx:194, in msgpack._cmsgpack.unpackb()
File ~/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/core.py:159, in loads.
._decode_default(obj) 157 if "compression" in sub_header: 158 sub_frames = decompress(sub_header, sub_frames) --> 159 return merge_and_deserialize( 160 sub_header, sub_frames, deserializers=deserializers 161 ) 162 else: 163 return Serialized(sub_header, sub_frames) File ~/.local/share/uv/python/cpython-3.12.10-macos-aarch64-none/lib/python3.12/contextlib.py:81, in ContextDecorator.call.
.inner(*args, **kwds) 78 @wraps(func) 79 def inner(*args, **kwds): 80 with self._recreate_cm(): ---> 81 return func(*args, **kwds) File ~/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/serialize.py:525, in merge_and_deserialize(header, frames, deserializers) 521 merged = host_array_from_buffers(subframes) 523 merged_frames.append(merged) --> 525 return deserialize(header, merged_frames, deserializers=deserializers)
File ~/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/serialize.py:452, in deserialize(header, frames, deserializers) 447 raise TypeError( 448 "Data serialized with %s but only able to deserialize " 449 "data with %s" % (name, str(list(deserializers))) 450 ) 451 dumps, loads, wants_context = families[name] --> 452 return loads(header, frames)
File ~/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/serialize.py:195, in serialization_error_loads(header, frames) 193 def serialization_error_loads(header, frames): 194 msg = "\n".join([codecs.decode(frame, "utf8") for frame in frames]) --> 195 raise TypeError(msg)
TypeError: Could not serialize object of type Dataset Traceback (most recent call last): File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/pickle.py", line 63, in dumps result = pickle.dumps(x, **dump_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ TypeError: cannot pickle 'module' object
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/pickle.py", line 68, in dumps pickler.dump(x) TypeError: cannot pickle 'module' object
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/serialize.py", line 366, in serialize header, frames = dumps(x, context=context) if wants_context else dumps(x) ^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/serialize.py", line 78, in pickle_dumps frames[0] = pickle.dumps( ^^^^^^^^^^^^^ File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/distributed/protocol/pickle.py", line 80, in dumps result = cloudpickle.dumps(x, **dump_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/cloudpickle/cloudpickle.py", line 1537, in dumps cp.dump(obj) File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/cloudpickle/cloudpickle.py", line 1303, in dump return super().dump(obj) ^^^^^^^^^^^^^^^^^ File "/Users/icarroll/tmp/h5pickly/.venv/lib/python3.12/site-packages/h5py/_hl/base.py", line 369, in getnewargs raise TypeError("h5py objects cannot be pickled") TypeError: h5py objects cannot be pickled
Environment
% uv pip list Package Version ---------------- ----------- click 8.2.1 cloudpickle 3.1.1 dask 2025.7.0 distributed 2025.7.0 fsspec 2025.9.0 h5netcdf 1.6.4 h5py 3.14.0 jinja2 3.1.6 locket 1.0.0 markupsafe 3.0.2 msgpack 1.1.1 numpy 2.3.3 packaging 25.0 pandas 2.3.2 partd 1.4.2 psutil 7.0.0 python-dateutil 2.9.0.post0 pytz 2025.2 pyyaml 6.0.2 six 1.17.0 sortedcontainers 2.4.0 tblib 3.1.0 toolz 1.0.0 tornado 6.5.2 tzdata 2025.2 urllib3 2.5.0 xarray 2025.9.0 zict 3.0.0
