httpx
httpx copied to clipboard
Published
20 hours ago •
encode
encode
deque mutated / dictionary changed during iteration
trafficstars
Discussions #3190 #3279
Errors occurred while downloading a lot of small (~2KB) files with 8 threads on both secure and clear text HTTP/2 but not HTTP/1.1. The following statistics of downloading from Azure Storage give some ideas of the rarity of errors.
1200 of 30597 failed
Count: Exception
11: RuntimeError('deque mutated during iteration')
2: RuntimeError('dictionary changed size during iteration')
1: RuntimeError('dictionary keys changed during iteration')
25: KeyError
34: LocalProtocolError('Invalid input ConnectionInputs.RECV_DATA in state ConnectionState.CLOSED')
15: LocalProtocolError('Invalid input ConnectionInputs.RECV_HEADERS in state ConnectionState.CLOSED')
15: LocalProtocolError('Invalid input StreamInputs.SEND_HEADERS in state 5')
1: LocalProtocolError('StreamIDTooLowError: 19 is lower than 31')
1: LocalProtocolError('StreamIDTooLowError: 399 is lower than 409')
1: LocalProtocolError('StreamIDTooLowError: 781 is lower than 793')
1: LocalProtocolError('StreamIDTooLowError: 95 is lower than 101')
248: RemoteProtocolError('<ConnectionTerminated error_code:1>')
40: RemoteProtocolError('<ConnectionTerminated error_code:9>')
774: RemoteProtocolError('Server disconnected')
31: WriteError('EOF occurred in violation of protocol (_ssl.c:2427)')
Traceback
RuntimeError: deque mutated during iteration
Traceback (most recent call last):
File "R:\httpxdltest.py", line 26, in download
with contextlib.closing(client.send(req, stream=True)) as response:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpx\_client.py", line 914, in send
response = self._send_handling_auth(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpx\_client.py", line 942, in _send_handling_auth
response = self._send_handling_redirects(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpx\_client.py", line 979, in _send_handling_redirects
response = self._send_single_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpx\_client.py", line 1014, in _send_single_request
response = transport.handle_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpx\_transports\default.py", line 250, in handle_request
resp = self._pool.handle_request(req)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\connection_pool.py", line 256, in handle_request
raise exc from None
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\connection_pool.py", line 236, in handle_request
response = connection.handle_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\connection.py", line 103, in handle_request
return self._connection.handle_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\http2.py", line 187, in handle_request
raise exc
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\http2.py", line 144, in handle_request
self._send_request_headers(request=request, stream_id=stream_id)
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\http2.py", line 249, in _send_request_headers
self._h2_state.send_headers(stream_id, headers, end_stream=end_stream)
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\h2\connection.py", line 806, in send_headers
frames.extend(stream.send_headers(
^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\h2\stream.py", line 894, in send_headers
frames = self._build_headers_frames(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\h2\stream.py", line 1298, in _build_headers_frames
encoded_headers = encoder.encode(headers)
^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\hpack\hpack.py", line 276, in encode
header_block.append(self.add(new_header, sensitive, huffman))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\hpack\hpack.py", line 301, in add
match = self.header_table.search(name, value)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\hpack\table.py", line 186, in search
for (i, (n, v)) in enumerate(self.dynamic_entries):
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: deque mutated during iteration
RuntimeError: dictionary changed size during iteration
Traceback (most recent call last):
File "R:\httpxdltest.py", line 26, in download
with contextlib.closing(client.send(req, stream=True)) as response:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpx\_client.py", line 914, in send
response = self._send_handling_auth(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpx\_client.py", line 942, in _send_handling_auth
response = self._send_handling_redirects(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpx\_client.py", line 979, in _send_handling_redirects
response = self._send_single_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpx\_client.py", line 1014, in _send_single_request
response = transport.handle_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpx\_transports\default.py", line 250, in handle_request
resp = self._pool.handle_request(req)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\connection_pool.py", line 256, in handle_request
raise exc from None
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\connection_pool.py", line 236, in handle_request
response = connection.handle_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\connection.py", line 103, in handle_request
return self._connection.handle_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\http2.py", line 187, in handle_request
raise exc
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\http2.py", line 144, in handle_request
self._send_request_headers(request=request, stream_id=stream_id)
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\http2.py", line 249, in _send_request_headers
self._h2_state.send_headers(stream_id, headers, end_stream=end_stream)
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\h2\connection.py", line 796, in send_headers
if (self.open_outbound_streams + 1) > max_open_streams:
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\h2\connection.py", line 451, in open_outbound_streams
return self._open_streams(outbound_numbers)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\h2\connection.py", line 433, in _open_streams
for stream_id, stream in self.streams.items():
^^^^^^^^^^^^^^^^^^^^
RuntimeError: dictionary changed size during iteration
RuntimeError: dictionary keys changed during iteration
Traceback (most recent call last):
File "R:\httpxdltest.py", line 26, in download
with contextlib.closing(client.send(req, stream=True)) as response:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpx\_client.py", line 914, in send
response = self._send_handling_auth(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpx\_client.py", line 942, in _send_handling_auth
response = self._send_handling_redirects(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpx\_client.py", line 979, in _send_handling_redirects
response = self._send_single_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpx\_client.py", line 1014, in _send_single_request
response = transport.handle_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpx\_transports\default.py", line 250, in handle_request
resp = self._pool.handle_request(req)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\connection_pool.py", line 256, in handle_request
raise exc from None
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\connection_pool.py", line 236, in handle_request
response = connection.handle_request(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\connection.py", line 103, in handle_request
return self._connection.handle_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\http2.py", line 187, in handle_request
raise exc
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\http2.py", line 144, in handle_request
self._send_request_headers(request=request, stream_id=stream_id)
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\httpcore\_sync\http2.py", line 249, in _send_request_headers
self._h2_state.send_headers(stream_id, headers, end_stream=end_stream)
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\h2\connection.py", line 796, in send_headers
if (self.open_outbound_streams + 1) > max_open_streams:
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\h2\connection.py", line 451, in open_outbound_streams
return self._open_streams(outbound_numbers)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\User\AppData\Roaming\Python\Python312\site-packages\h2\connection.py", line 433, in _open_streams
for stream_id, stream in self.streams.items():
^^^^^^^^^^^^^^^^^^^^
RuntimeError: dictionary keys changed during iteration
They are reproducible in the following environment.
Server
Azure Storage x-ms-version 2009-09-19 / fastapi 0.115.12 on hypercorn 0.17.3
fastapi's "Hello world" code run on hypercorn
from typing import Union
from fastapi import FastAPI
app = FastAPI()
@app.get("/items/{item_id}")
async def read_item(item_id: int, q: Union[str, None] = None):
return {"item_id": item_id, "q": q}
Client
Windows 7 / 10 Python 3.8.10 / 3.12.10 httpx 0.28.1 httpcore 1.0.9 h2 4.2.0
Test
#! /usr/bin/env python3
import os
import logging
import contextlib
import concurrent.futures
import httpx
logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())
traceback_logger = logging.getLogger(__name__ + '_traceback')
traceback_logger.addHandler(logging.NullHandler())
def download(client, url, timeout=30):
try:
req = client.build_request('GET', url, timeout=timeout)
with contextlib.closing(client.send(req, stream=True)) as response:
for chunk in response.iter_bytes():
pass
except BaseException as e:
logger.error('{!r} at "{}"'.format(e, url))
traceback_logger.error('{!r} at "{}"'.format(e, url), exc_info=e)
raise
def test(urls, max_workers=os.cpu_count() or 1, timeout=30):
with httpx.Client(http1=False, http2=True, timeout=timeout) as client,\
concurrent.futures.ThreadPoolExecutor(max_workers) as executor:
not_done = set()
for url in urls:
not_done.add(executor.submit(download, client, url, timeout))
done, not_done = concurrent.futures.wait(not_done)
errors = {}
for future in done:
if future.exception() is not None:
errors[repr(future.exception())] = errors.get(repr(future.exception()), 0) + 1
stat = ['{} of {} failed'.format(sum(errors.values()), len(done))]
if errors:
stat.append('Count: Exception')
for k in sorted(errors):
stat.append('{: >5}: {}'.format(errors[k], k))
logger.info('\n'.join(stat))
if __name__ == '__main__':
import sys
if len(sys.argv) != 2:
print('Usage: {} <concurrent count>'.format(sys.argv[0]))
sys.exit(1)
logger.setLevel(logging.DEBUG)
traceback_logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('{asctime}: TID {thread}: {levelname}: {message}', style='{')
handler = logging.StreamHandler(sys.stderr)
handler.setLevel(logging.DEBUG)
handler.setFormatter(formatter)
logger.addHandler(handler)
handler = logging.FileHandler(os.path.splitext(sys.argv[0])[0] + '.log', encoding='utf-8')
handler.setLevel(logging.DEBUG)
handler.setFormatter(formatter)
logger.addHandler(handler)
handler = logging.FileHandler(os.path.splitext(sys.argv[0])[0] + '_traceback.log', encoding='utf-8')
handler.setLevel(logging.DEBUG)
handler.setFormatter(formatter)
traceback_logger.addHandler(handler)
urls = ('http://127.0.0.1:8000/items/' + str(id) for id in range(100000))
test(urls, int(sys.argv[1]) or os.cpu_count() or 1)
