aio-libs
RuntimeError: Site <...> is not registered in runner when using aiohttp_client fixture
🐞 Describe the bug
If your test finishes before exhausting any async generators then the test appears to try and deregister twice, causing an error. Or something.
This is using pytest and aiohttp_client fixture.
python 3.7.7(on osx)pytestversions:5.3.5and5.4.1pytest-asyncio-0.10.0pytest-aiohttp-0.3.0aiohttp-3.6.2
💡 To Reproduce
The following is a distilled version of my code that I found the problem in. Search for TOGGLE near the end of the file for various ways to "fix" the problem. All the fixes are essentially the same.
#!/usr/bin/env python
import asyncio
import pytest
import aiohttp
import inspect
from functools import partial
from functools import wraps
@pytest.fixture
def my_client(loop, aiohttp_client):
async def page(request):
return aiohttp.web.Response(text="this is the page")
app = aiohttp.web.Application()
app.router.add_get('/', page)
return loop.run_until_complete(aiohttp_client(app))
async def get_next(generator):
try:
item = await generator.__anext__()
return item
except StopAsyncIteration:
print("get_next stopped")
return None
async def fetch(session, url):
try:
async with session.get(url) as resp:
resp.text = await resp.text() # set coro with value, this is allowed
resp.close()
return resp
except (aiohttp.ClientResponseError, aiohttp.client_exceptions.ClientError) as e:
print("url: %s: error: %s", url, e)
return None
async def convert_to_generator(callback, response):
print("calling convert_to_generator")
yield await callback(response)
async def scrape(client, url, callback):
loop = asyncio.get_event_loop()
async with client as session:
task = loop.create_task(
fetch(session, url)
)
resp = await task
# if the callback is not a generator, then convert it
# to one so that we can use it in the loop below
# ie: comment in/out the yield in parse()
if not inspect.isasyncgenfunction(callback):
callback = partial(convert_to_generator, callback)
async for x in callback(resp):
print("pre yield in scrape")
yield x
print("post yield in scrape")
# never got this to work properly, would be happy if somebody did though
def exhaust_generator(func):
@wraps
async def inner(*args, **kwargs):
gen = func(*args, **kwargs)
if gen is None:
return
async for _ in gen:
pass
inner.__name__ = func.__name__
return inner
# @pytest.mark.asyncio
async def test_cause_error(my_client):
"""
tl;dr call parse() with the Response and yield 'foo' back
through scrape() to test body. If the StopIteration aren't triggered
and gets everything to clean up a strange error occurs.
"""
called = False
async def parse(response):
nonlocal called
assert 'page' in response.text
print("pre yield in parse")
called = True
yield 'foo' # this line was what started this investigation
print("post yield in parse")
if False: # TOGGLE
async for x in scrape(my_client, '/', parse):
print(f"x={x}")
else:
gen = scrape(my_client, '/', parse)
x = await get_next(gen)
print(f"x={x}")
# await get_next(gen) # TOGGLE THIS TO FIX "BUG"
# or do the following
# async for _ in gen:
# pass
assert called
💡 Expected behaviour
Tests should pass and exit cleanly. Note the 1 passed, 1 error even though there is only one test.
📋 Logs/tracebacks
pytest x_test_scrape.py
========================================== test session starts ==========================================
platform darwin -- Python 3.7.7, pytest-5.4.1, py-1.8.1, pluggy-0.13.1
rootdir: /Users/kneufeld/src/iterweb, inifile: pytest.ini
plugins: asyncio-0.10.0, aiohttp-0.3.0
collected 1 item
x_test_scrape.py .E [100%]
================================================ ERRORS =================================================
_____________________________ ERROR at teardown of test_cause_error[pyloop] _____________________________
loop = <_UnixSelectorEventLoop running=False closed=True debug=False>
@pytest.fixture
def aiohttp_client(loop): # type: ignore
"""Factory to create a TestClient instance.
aiohttp_client(app, **kwargs)
aiohttp_client(server, **kwargs)
aiohttp_client(raw_server, **kwargs)
"""
clients = []
async def go(__param, *args, server_kwargs=None, **kwargs): # type: ignore
if (isinstance(__param, Callable) and # type: ignore
not isinstance(__param, (Application, BaseTestServer))):
__param = __param(loop, *args, **kwargs)
kwargs = {}
else:
assert not args, "args should be empty"
if isinstance(__param, Application):
server_kwargs = server_kwargs or {}
server = TestServer(__param, loop=loop, **server_kwargs)
client = TestClient(server, loop=loop, **kwargs)
elif isinstance(__param, BaseTestServer):
client = TestClient(__param, loop=loop, **kwargs)
else:
raise ValueError("Unknown argument type: %r" % type(__param))
await client.start_server()
clients.append(client)
return client
yield go
async def finalize(): # type: ignore
while clients:
await clients.pop().close()
> loop.run_until_complete(finalize())
../../.virtualenvs/iterweb/lib/python3.7/site-packages/aiohttp/pytest_plugin.py:345:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/base_events.py:587: in run_until_complete
return future.result()
../../.virtualenvs/iterweb/lib/python3.7/site-packages/aiohttp/pytest_plugin.py:343: in finalize
await clients.pop().close()
../../.virtualenvs/iterweb/lib/python3.7/site-packages/aiohttp/test_utils.py:388: in close
await self._server.close()
../../.virtualenvs/iterweb/lib/python3.7/site-packages/aiohttp/test_utils.py:171: in close
await self.runner.cleanup()
../../.virtualenvs/iterweb/lib/python3.7/site-packages/aiohttp/web_runner.py:250: in cleanup
await site.stop()
../../.virtualenvs/iterweb/lib/python3.7/site-packages/aiohttp/web_runner.py:67: in stop
self._runner._unreg_site(self)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <aiohttp.web_runner.AppRunner object at 0x10d317f50>
site = <aiohttp.web_runner.SockSite object at 0x10d326130>
def _unreg_site(self, site: BaseSite) -> None:
if site not in self._sites:
raise RuntimeError("Site {} is not registered in runner {}"
> .format(site, self))
E RuntimeError: Site <aiohttp.web_runner.SockSite object at 0x10d326130> is not registered in runner <aiohttp.web_runner.AppRunner object at 0x10d317f50>
../../.virtualenvs/iterweb/lib/python3.7/site-packages/aiohttp/web_runner.py:283: RuntimeError
----------------------------------------- Captured stdout call ------------------------------------------
pre yield in parse
pre yield in scrape
x=foo
======================================== short test summary info ========================================
ERROR x_test_scrape.py::test_cause_error[pyloop] - RuntimeError: Site <aiohttp.web_runner.SockSite obj...
====================================== 1 passed, 1 error in 0.17s =======================================
📋 Your version of the Python
$ python --version
Python 3.7.7
📋 Your version of the aiohttp/yarl/multidict distributions
$ python -m pip show aiohttp
Name: aiohttp
Version: 3.6.2
$ python -m pip show multidict
Name: multidict
Version: 4.7.5
$ python -m pip show yarl
Name: yarl
Version: 1.4.2
📋 Additional context
client I think
I am getting the same error when trying to use the aiohttp test_client to consume a single message from a mocked websocket.
I notice pytest-asyncio in the original issue. Does everything work if pytest-asyncio is uninstalled?
Uninstalling pytest-asyncio made no difference for me.
I am essentially doing this (distilled to bare essentials):
# Mocked websocket server handler added to mock server via 'create_app()'
async def mock_websocket(request: web.Request) -> WebSocketResponse:
ws = web.WebSocketResponse()
await ws.prepare(request)
try:
await ws.send_json(SOME_JSON)
return ws
finally:
if not ws.closed:
await ws.close()
async def test_subscribe_to_thing(test_client):
try:
test_session = await test_client(create_app)
my_client: MyClient = await _get_my_client(test_session)
async for result in my_client.subscribe_to_thing():
if result:
logger.debug(result)
assert True
return
except ClientResponseError as e:
raise pytest.fail(str(e))
MyClient.subscribe_to_thing() has this section which errors when it triggers the __aexit__ and closes out the test_client session:
async with self.session as session: # This raises the RuntimeError on `__aexit__`
async with session.ws_connect(url, headers={}, timeout=5.0) as ws:
async for msg in ws:
content = json.loads(msg.data)
yield content
And I get the same error as OP:
RuntimeError: Site <aiohttp.web_runner.SockSite object at 0x...> is not registered in runner <aiohttp.web_runner.AppRunner object at 0x...>
I am working around this issue for now by catching the RuntimeError in my source code with a comment that this is to catch a pytest bug...
Ah! I'm a dummy. I've changed the code from:
async with self.session as session: # This raises the RuntimeError on `__aexit__`
async with session.ws_connect(url, headers={}, timeout=5.0) as ws:
async for msg in ws:
content = json.loads(msg.data)
yield content
to:
async with self.session.ws_connect(url, headers={}, timeout=5.0) as ws:
async for msg in ws:
content = json.loads(msg.data)
yield content
I was using a context manager for the cached session instance when I should not be - I want the ClientSession to hang around
I'm not clear what's surprising about the original issue. You've created a ClientSession inside a generator function and then not executed the generator to completion in order for the ClientSession to close and clean up. I would expect something to mess up there..