Misleading IncompleteRead exception, if a HTTP response contains unexpected content #3261
Description
Subject
When interacting with poorly implemented servers or when paying little attention during mocking of HTTP requests, one can be faced e.g. with a 204 response, which contains an unexpected body.
So far so bad. urllib3 only raises an IncompleteRead exception, which provides no insight into the reason (e.g. a 204 response with body).
Environment
- Python 3.10
- urllib3 2.1.0
Steps to Reproduce
pip install requests responses# scratch.py
import unittest
import responses
import requests
class MyTest(unittest.TestCase):
url = "http://example.com"
def test_requestsmock(self):
rmock = responses.RequestsMock()
rmock.start()
rmock.post(url=self.url,
status=204,
json={"status": "ok"},
auto_calculate_content_length=True)
response = requests.post(url=self.url, data="Foo Bar!")
@responses.activate
def test_decorator(self):
responses.add(method=responses.POST,
url=self.url,
status=204,
json={"status": "ok"},
auto_calculate_content_length=True)
response = requests.post(url=self.url, data="Foo Bar!")
unittest.main()Expected Behavior
If an empty response body is expected, the raised exception should complain about the presence of an unexpected body.
Also, the reason for expecting an empty body would be helpful and speed up debugging.
Actual Behavior
Instead, IncompleteRead(16 bytes read, -16 more expected) is raised:
python3.10 scratch.py
EE
======================================================================
ERROR: test_decorator (__main__.MyTest)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/urllib3/response.py", line 712, in _error_catcher
yield
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/urllib3/response.py", line 833, in _raw_read
raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
urllib3.exceptions.IncompleteRead: IncompleteRead(16 bytes read, -16 more expected)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/requests/models.py", line 816, in generate
yield from self.raw.stream(chunk_size, decode_content=True)
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/urllib3/response.py", line 934, in stream
data = self.read(amt=amt, decode_content=decode_content)
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/urllib3/response.py", line 905, in read
data = self._raw_read(amt)
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/urllib3/response.py", line 811, in _raw_read
with self._error_catcher():
File "/usr/lib/python3.10/contextlib.py", line 153, in __exit__
self.gen.throw(typ, value, traceback)
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/urllib3/response.py", line 729, in _error_catcher
raise ProtocolError(f"Connection broken: {e!r}", e) from e
urllib3.exceptions.ProtocolError: ('Connection broken: IncompleteRead(16 bytes read, -16 more expected)', IncompleteRead(16 bytes read, -16 more expected))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/responses/__init__.py", line 232, in wrapper
return func(*args, **kwargs)
File "/home/andi/.config/JetBrains/PyCharm2023.2/scratches/scratch.py", line 28, in test_decorator
response = requests.post(url=self.url, data="Foo Bar!")
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/requests/api.py", line 115, in post
return request("post", url, data=data, json=json, **kwargs)
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/requests/sessions.py", line 747, in send
r.content
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/requests/models.py", line 899, in content
self._content = b"".join(self.iter_content(CONTENT_CHUNK_SIZE)) or b""
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/requests/models.py", line 818, in generate
raise ChunkedEncodingError(e)
requests.exceptions.ChunkedEncodingError: ('Connection broken: IncompleteRead(16 bytes read, -16 more expected)', IncompleteRead(16 bytes read, -16 more expected))
======================================================================
ERROR: test_requestsmock (__main__.MyTest)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/urllib3/response.py", line 712, in _error_catcher
yield
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/urllib3/response.py", line 833, in _raw_read
raise IncompleteRead(self._fp_bytes_read, self.length_remaining)
urllib3.exceptions.IncompleteRead: IncompleteRead(16 bytes read, -16 more expected)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/requests/models.py", line 816, in generate
yield from self.raw.stream(chunk_size, decode_content=True)
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/urllib3/response.py", line 934, in stream
data = self.read(amt=amt, decode_content=decode_content)
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/urllib3/response.py", line 905, in read
data = self._raw_read(amt)
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/urllib3/response.py", line 811, in _raw_read
with self._error_catcher():
File "/usr/lib/python3.10/contextlib.py", line 153, in __exit__
self.gen.throw(typ, value, traceback)
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/urllib3/response.py", line 729, in _error_catcher
raise ProtocolError(f"Connection broken: {e!r}", e) from e
urllib3.exceptions.ProtocolError: ('Connection broken: IncompleteRead(16 bytes read, -16 more expected)', IncompleteRead(16 bytes read, -16 more expected))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/andi/.config/JetBrains/PyCharm2023.2/scratches/scratch.py", line 18, in test_requestsmock
response = requests.post(url=self.url, data="Foo Bar!")
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/requests/api.py", line 115, in post
return request("post", url, data=data, json=json, **kwargs)
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/requests/sessions.py", line 747, in send
r.content
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/requests/models.py", line 899, in content
self._content = b"".join(self.iter_content(CONTENT_CHUNK_SIZE)) or b""
File "/home/andi/virtualenvs/checkers/lib/python3.10/site-packages/requests/models.py", line 818, in generate
raise ChunkedEncodingError(e)
requests.exceptions.ChunkedEncodingError: ('Connection broken: IncompleteRead(16 bytes read, -16 more expected)', IncompleteRead(16 bytes read, -16 more expected))
----------------------------------------------------------------------
Ran 2 tests in 0.005s
FAILED (errors=2)