hamilton icon indicating copy to clipboard operation
hamilton copied to clipboard

Parallelizable hangs in certain cases

Open elijahbenizzy opened this issue 1 year ago • 0 comments

Short description explaining the high-level reason for the new issue.

Current behavior

Hangs on this:

import logging

from hamilton import driver
from hamilton.execution.executors import SynchronousLocalTaskExecutor
from hamilton.htypes import Parallelizable, Collect

def initial() -> int:
    return 0


def parallelize_over(initial: int) -> Parallelizable[int]:
    for i in range(2):
        yield i


def internal(parallelize_over: int, initial: int) -> int:
    return parallelize_over + 1 + initial


def collect(internal: Collect[int], initial: int) -> int:
    return sum(internal) + 1


def final(collect: int) -> int:
    return collect + 1


if __name__ == "__main__":
    import __main__

    from hamilton.log_setup import setup_logging
    setup_logging(log_level=logging.DEBUG)

    dr = (
        driver
        .Builder()
        .with_modules(__main__)
        .enable_dynamic_execution(allow_experimental_mode=True)
        .with_local_executor(SynchronousLocalTaskExecutor())
        .build()
    )
    dr.visualize_execution(final_vars=["final"], inputs={}, output_file_path="parallelism.png")
    dr.execute(["final"])

elijahbenizzy avatar Mar 05 '24 19:03 elijahbenizzy