Repeated calls to Overlay, Curve, and .opts() causes StoreOptions to fail

[dycw]

ALL software version info

Python 3.9, with a simple Makefile:

.ONESHELL:
SHELL=/bin/bash
.PHONY: env
env:
        [ -d .venv ] && rm -rf .venv/
        PYENV_VERSION=3.9.13 python -m venv --clear .venv
        ! [ -L activate ] && ln -s .venv/bin/activate
        source activate
        pip install --upgrade numpy holoviews hypothesis pip pre-commit pytest

Run with make env.

Description of expected behavior and the observed behavior

Not to crash. Instead, hypothesis marks this as a flaky test: one that fails initially, but then succeeded upon shrinking.

Complete, minimal, self-contained example code that reproduces the issue

# test_plots.py

from holoviews import Curve, Overlay
from holoviews.plotting import bokeh  # type: ignore # noqa
from hypothesis import given, settings
from hypothesis.extra.numpy import arrays
from hypothesis.strategies import data, integers, lists


@given(data=data())
@settings(deadline=None, max_examples=10000)
def test_main(data):
    n = data.draw(integers(1, 5))
    arrs = data.draw(lists(arrays(float, n), min_size=1, max_size=5))

    _ = Overlay(items=[Curve(arr) for arr in arrs])  # this will succeed

    if True:  # this will fail
        _ = Overlay(items=[Curve(arr).opts(show_grid=True) for arr in arrs])

    if False:  # this will fail
        _ = Overlay(items=[Curve(arr).opts(tools=["hover"]) for arr in arrs])

Stack traceback and/or browser JavaScript console output

Run with pytest test_plots.py:

E           hypothesis.errors.Flaky: Hypothesis test_main(data=data(...)) produces unreliable results: Falsified on the first call but did not on a subsequent one
E           Falsifying example: test_main(
E               data=data(...),
E           )
E           Draw 1: 3
E           Draw 2: [array([6.10351562e-05, 6.10351562e-05, 6.10351562e-05])]
E           Failed to reproduce exception. Expected:
E           data = data(...)
E
E               @given(data=data())
E               @settings(max_examples=1000)
E               def test_main(data):
E                   n = data.draw(integers(1, 5))
E                   arrs = data.draw(
E                       lists(
E                           arrays(float, n, elements=floats(-1.0, 1.0)),
E                           min_size=1,
E                           max_size=5,
E                       )
E                   )
E                   if False:  # this will succeed
E                       _ = Overlay(items=[Curve(arr).opts() for arr in arrs])
E
E                   if False:  # this will succeed
E                       _ = Overlay(items=[Curve(arr).opts(show_grid=True) for arr in arrs])
E
E                   if True:  # this will fail
E           >           _ = Overlay(items=[Curve(arr).opts(tools=["hover"]) for arr in arrs])
E
E           test_plots.py:26:
E           _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
E           test_plots.py:26: in <listcomp>
E               _ = Overlay(items=[Curve(arr).opts(tools=["hover"]) for arr in arrs])
E           .venv/lib/python3.9/site-packages/holoviews/core/accessors.py:45: in pipelined_call
E               result = __call__(*args, **kwargs)
E           .venv/lib/python3.9/site-packages/holoviews/core/accessors.py:571: in __call__
E               return self._dispatch_opts( *args, **kwargs)
E           .venv/lib/python3.9/site-packages/holoviews/core/accessors.py:575: in _dispatch_opts
E               return self._base_opts(*args, **kwargs)
E           .venv/lib/python3.9/site-packages/holoviews/core/accessors.py:654: in _base_opts
E               return self._obj.options(*new_args, **kwargs)
E           .venv/lib/python3.9/site-packages/holoviews/core/data/__init__.py:204: in pipelined_fn
E               result = method_fn(*args, **kwargs)
E           .venv/lib/python3.9/site-packages/holoviews/core/data/__init__.py:1212: in options
E               return super(Dataset, self).options(*args, **kwargs)
E           .venv/lib/python3.9/site-packages/holoviews/core/dimension.py:1283: in options
E               obj = obj.opts._dispatch_opts(expanded, backend=backend, clone=clone)
E           .venv/lib/python3.9/site-packages/holoviews/core/accessors.py:575: in _dispatch_opts
E               return self._base_opts(*args, **kwargs)
E           .venv/lib/python3.9/site-packages/holoviews/core/accessors.py:651: in _base_opts
E               return opts.apply_groups(self._obj, **dict(kwargs, **new_kwargs))
E           .venv/lib/python3.9/site-packages/holoviews/util/__init__.py:225: in apply_groups
E               obj = cls._apply_groups_to_backend(obj, backend_opts, backend, clone)
E           .venv/lib/python3.9/site-packages/holoviews/util/__init__.py:150: in _apply_groups_to_backend
E               return StoreOptions.set_options(obj_handle, options, backend=backend)
E           .venv/lib/python3.9/site-packages/holoviews/core/options.py:1858: in set_options
E               custom_trees, id_mapping = cls.create_custom_trees(obj, spec, backend=backend)
E           .venv/lib/python3.9/site-packages/holoviews/core/options.py:1648: in create_custom_trees
E               offset = cls.id_offset()
E           _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
E
E           cls = <class 'holoviews.core.options.StoreOptions'>
E
E               @classmethod
E               def id_offset(cls):
E                   """
E                   Compute an appropriate offset for future id values given the set
E                   of ids currently defined across backends.
E                   """
E                   max_ids = []
E                   for backend in Store.renderers.keys():
E                       store_ids = Store.custom_options(backend=backend).keys()
E           >           max_id = max(store_ids)+1 if len(store_ids) > 0 else 0
E           E           ValueError: max() arg is an empty sequence
E
E           .venv/lib/python3.9/site-packages/holoviews/core/options.py:1783: ValueError

[hoxbro]

This seems to happen because of a race condition happening between len(store_ids) and max(store_ids). It could be solved by converting store_ids to a list in the above line. Thoughts @jlstevens?

https://github.com/holoviz/holoviews/blob/6c0dafb8dc0b2c059b0cfbf49908c51f243c309c/holoviews/core/options.py#L1782-L1783

I'm curious about why you need to run this test with so many iterations.

[dycw]

Hi @Hoxbro , hypothesis defaults to 100 examples, but for this MCRE, I raised the example count higher so it fails with near certainty.