python-blosc2 icon indicating copy to clipboard operation
python-blosc2 copied to clipboard

blosc2 freezes when used in a ProcessPoolExecutor -- fine in a ThreadPoolExecutor

Open tazr opened this issue 1 year ago • 0 comments

from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor

import blosc2
import numpy as np


def test(arr):
    print("test begins")
    blosc2.pack_array(arr)
    print("test ends")


def main():
    arrs = [np.random.randint(255, size=(1024, 1024), dtype=np.uint8) for _ in range(20)]

    with ThreadPoolExecutor(max_workers=2) as pool:
        for arr in arrs:
            pool.submit(test, arr=arr)

    print("threadpool done")
    with ProcessPoolExecutor(max_workers=2) as pool:
        for arr in arrs:
            pool.submit(test, arr=arr)
    print("processpool done")  # never printed


if __name__ == "__main__":
    main()

I tried to set blosc2.ncores/nthreads to 1 but to no avail.

tazr avatar Aug 16 '24 17:08 tazr