pytest-xdist
pytest-xdist copied to clipboard
Published
20 hours ago •
pytest-dev
pytest-dev
Very long strings may be generated when I test a large file with "- n NUM" parameter. And It causes an Error: the string is too long.
My python version is 3.10.12 pytest version : 8.2.1 execnet Version: 2.1.1 The versions are both the lastest.
When I run the command: pytest -k mykey -n 16, I got the error as follows:
INTERNALERROR> E raise DumpError(error)
INTERNALERROR> E execnet.gateway_base.DumpError: string is too long
INTERNALERROR> E assert False
INTERNALERROR>
I checked the source code in gateway_base.py and find that when the byte_ > FOUR_BYTE_INT_MAX , there will raise a DumpError.
def _write_int4(
self, i: int, error: str = "int must be less than %i" % (FOUR_BYTE_INT_MAX,)
) -> None:
if i > FOUR_BYTE_INT_MAX:
raise DumpError(error)
self._write(struct.pack("!i", i))
I tried to add the int8 bytecodes for long string, but it doesn't work.
How should I fix this bug? Another idea is that I do not need to print so much string information. Can I simplify the printout or disable this feature directly?
The value origin is unclear, is there a minimal reproducer?
This is a big project, I will try to make a minimal reproducer later.
This are complete error messages, and hopefully this will help.
INTERNALERROR> def worker_internal_error(
INTERNALERROR> self, node: WorkerController, formatted_error: str
INTERNALERROR> ) -> None:
INTERNALERROR> """
INTERNALERROR> pytest_internalerror() was called on the worker.
INTERNALERROR>
INTERNALERROR> pytest_internalerror() arguments are an excinfo and an excrepr, which can't
INTERNALERROR> be serialized, so we go with a poor man's solution of raising an exception
INTERNALERROR> here ourselves using the formatted message.
INTERNALERROR> """
INTERNALERROR> self._active_nodes.remove(node)
INTERNALERROR> try:
INTERNALERROR> > assert False, formatted_error
INTERNALERROR> E AssertionError: Traceback (most recent call last):
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/_pytest/main.py", line 285, in wrap_session
INTERNALERROR> E session.exitstatus = doit(config, session) or 0
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/_pytest/main.py", line 339, in _main
INTERNALERROR> E config.hook.pytest_runtestloop(session=session)
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_hooks.py", line 513, in __call__
INTERNALERROR> E return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_manager.py", line 120, in _hookexec
INTERNALERROR> E return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 139, in _multicall
INTERNALERROR> E raise exception.with_traceback(exception.__traceback__)
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 122, in _multicall
INTERNALERROR> E teardown.throw(exception) # type: ignore[union-attr]
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/_pytest/logging.py", line 807, in pytest_runtestloop
INTERNALERROR> E return (yield) # Run all the tests.
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 103, in _multicall
INTERNALERROR> E res = hook_impl.function(*args)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/xdist/remote.py", line 174, in pytest_runtestloop
INTERNALERROR> E self.run_one_test()
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/xdist/remote.py", line 195, in run_one_test
INTERNALERROR> E self.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_hooks.py", line 513, in __call__
INTERNALERROR> E return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_manager.py", line 120, in _hookexec
INTERNALERROR> E return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 182, in _multicall
INTERNALERROR> E return outcome.get_result()
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_result.py", line 100, in get_result
INTERNALERROR> E raise exc.with_traceback(exc.__traceback__)
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 167, in _multicall
INTERNALERROR> E teardown.throw(outcome._exception)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/_pytest/warnings.py", line 111, in pytest_runtest_protocol
INTERNALERROR> E return (yield)
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 167, in _multicall
INTERNALERROR> E teardown.throw(outcome._exception)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/_pytest/assertion/__init__.py", line 176, in pytest_runtest_protocol
INTERNALERROR> E return (yield)
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 167, in _multicall
INTERNALERROR> E teardown.throw(outcome._exception)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/_pytest/unittest.py", line 421, in pytest_runtest_protocol
INTERNALERROR> E res = yield
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 167, in _multicall
INTERNALERROR> E teardown.throw(outcome._exception)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/_pytest/faulthandler.py", line 85, in pytest_runtest_protocol
INTERNALERROR> E return (yield)
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 103, in _multicall
INTERNALERROR> E res = hook_impl.function(*args)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/_pytest/runner.py", line 116, in pytest_runtest_protocol
INTERNALERROR> E runtestprotocol(item, nextitem=nextitem)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/_pytest/runner.py", line 135, in runtestprotocol
INTERNALERROR> E reports.append(call_and_report(item, "call", log))
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/_pytest/runner.py", line 245, in call_and_report
INTERNALERROR> E ihook.pytest_runtest_logreport(report=report)
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_hooks.py", line 513, in __call__
INTERNALERROR> E return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_manager.py", line 120, in _hookexec
INTERNALERROR> E return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 139, in _multicall
INTERNALERROR> E raise exception.with_traceback(exception.__traceback__)
INTERNALERROR> E File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 103, in _multicall
INTERNALERROR> E res = hook_impl.function(*args)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/xdist/remote.py", line 255, in pytest_runtest_logreport
INTERNALERROR> E self.sendevent("testreport", data=data)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/xdist/remote.py", line 98, in sendevent
INTERNALERROR> E self.channel.send((name, kwargs))
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 912, in send
INTERNALERROR> E self.gateway._send(Message.CHANNEL_DATA, self.id, dumps_internal(item))
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1636, in dumps_internal
INTERNALERROR> E return _Serializer().save(obj) # type: ignore[return-value]
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1654, in save
INTERNALERROR> E self._save(obj)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1674, in _save
INTERNALERROR> E dispatch(self, obj)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1768, in save_tuple
INTERNALERROR> E self._save(item)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1674, in _save
INTERNALERROR> E dispatch(self, obj)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1764, in save_dict
INTERNALERROR> E self._write_setitem(key, value)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1758, in _write_setitem
INTERNALERROR> E self._save(value)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1674, in _save
INTERNALERROR> E dispatch(self, obj)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1764, in save_dict
INTERNALERROR> E self._write_setitem(key, value)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1758, in _write_setitem
INTERNALERROR> E self._save(value)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1674, in _save
INTERNALERROR> E dispatch(self, obj)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1754, in save_list
INTERNALERROR> E self._write_setitem(i, item)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1758, in _write_setitem
INTERNALERROR> E self._save(value)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1674, in _save
INTERNALERROR> E dispatch(self, obj)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1768, in save_tuple
INTERNALERROR> E self._save(item)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1674, in _save
INTERNALERROR> E dispatch(self, obj)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1696, in save_str
INTERNALERROR> E self._write_unicode_string(s)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1703, in _write_unicode_string
INTERNALERROR> E self._write_byte_sequence(as_bytes)
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1706, in _write_byte_sequence
INTERNALERROR> E self._write_int4(len(bytes_), "string is too long")
INTERNALERROR> E File "/home/jasper.zeng/.local/lib/python3.10/site-packages/execnet/gateway_base.py", line 1740, in _write_int4
INTERNALERROR> E raise DumpError(error)
INTERNALERROR> E execnet.gateway_base.DumpError: string is too long
INTERNALERROR> E assert False
INTERNALERROR>
INTERNALERROR> /home/jasper.zeng/.local/lib/python3.10/site-packages/xdist/dsession.py:232: AssertionError
INTERNALERROR> Traceback (most recent call last):
INTERNALERROR> File "/home/jasper.zeng/.local/lib/python3.10/site-packages/_pytest/main.py", line 285, in wrap_session
INTERNALERROR> session.exitstatus = doit(config, session) or 0
INTERNALERROR> File "/home/jasper.zeng/.local/lib/python3.10/site-packages/_pytest/main.py", line 339, in _main
INTERNALERROR> config.hook.pytest_runtestloop(session=session)
INTERNALERROR> File "/usr/local/lib/python3.10/dist-packages/pluggy/_hooks.py", line 513, in __call__
INTERNALERROR> return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
INTERNALERROR> File "/usr/local/lib/python3.10/dist-packages/pluggy/_manager.py", line 120, in _hookexec
INTERNALERROR> return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
INTERNALERROR> File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 139, in _multicall
INTERNALERROR> raise exception.with_traceback(exception.__traceback__)
INTERNALERROR> File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 122, in _multicall
INTERNALERROR> teardown.throw(exception) # type: ignore[union-attr]
INTERNALERROR> File "/home/jasper.zeng/.local/lib/python3.10/site-packages/_pytest/logging.py", line 807, in pytest_runtestloop
INTERNALERROR> return (yield) # Run all the tests.
INTERNALERROR> File "/usr/local/lib/python3.10/dist-packages/pluggy/_callers.py", line 103, in _multicall
INTERNALERROR> res = hook_impl.function(*args)
INTERNALERROR> File "/home/jasper.zeng/.local/lib/python3.10/site-packages/xdist/dsession.py", line 138, in pytest_runtestloop
INTERNALERROR> self.loop_once()
INTERNALERROR> File "/home/jasper.zeng/.local/lib/python3.10/site-packages/xdist/dsession.py", line 163, in loop_once
INTERNALERROR> call(**kwargs)
INTERNALERROR> File "/home/jasper.zeng/.local/lib/python3.10/site-packages/xdist/dsession.py", line 217, in worker_workerfinished
INTERNALERROR> assert not crashitem, (crashitem, node)
INTERNALERROR> AssertionError: ('tests/my_case.py[case_info=xxxx.yaml/linear_1-4096-16384-5120]', <WorkerController gw14>)
INTERNALERROR> assert not 'tests/my_case.py[case_info=xxxx.yaml/linear_1-4096-16384-5120]'
Thank you!
This looks like log or io capture of your test accumulated massive amounts of data
Try disabling log capture, or using subtests