hamilton
hamilton copied to clipboard
Parallelizable hangs in certain cases
Short description explaining the high-level reason for the new issue.
Current behavior
Hangs on this:
import logging
from hamilton import driver
from hamilton.execution.executors import SynchronousLocalTaskExecutor
from hamilton.htypes import Parallelizable, Collect
def initial() -> int:
return 0
def parallelize_over(initial: int) -> Parallelizable[int]:
for i in range(2):
yield i
def internal(parallelize_over: int, initial: int) -> int:
return parallelize_over + 1 + initial
def collect(internal: Collect[int], initial: int) -> int:
return sum(internal) + 1
def final(collect: int) -> int:
return collect + 1
if __name__ == "__main__":
import __main__
from hamilton.log_setup import setup_logging
setup_logging(log_level=logging.DEBUG)
dr = (
driver
.Builder()
.with_modules(__main__)
.enable_dynamic_execution(allow_experimental_mode=True)
.with_local_executor(SynchronousLocalTaskExecutor())
.build()
)
dr.visualize_execution(final_vars=["final"], inputs={}, output_file_path="parallelism.png")
dr.execute(["final"])