Use lenient headers for response parser (#7490)

CHANGES/7490.feature

@@ -0,0 +1 @@
+Enabled lenient headers for more flexible parsing in the client. -- by :user:`Dreamsorcerer`

aiohttp/_http_parser.pyx

@@ -20,6 +20,7 @@ from multidict import CIMultiDict as _CIMultiDict, CIMultiDictProxy as _CIMultiD
 from yarl import URL as _URL
 
 from aiohttp import hdrs
+from aiohttp.helpers import DEBUG
 
 from .http_exceptions import (
     BadHttpMessage,
@@ -648,6 +649,9 @@ cdef class HttpResponseParser(HttpParser):
                    max_line_size, max_headers, max_field_size,
                    payload_exception, response_with_body, read_until_eof,
                    auto_decompress)
+        # Use strict parsing on dev mode, so users are warned about broken servers.
+        if not DEBUG:
+            cparser.llhttp_set_lenient_headers(self._cparser, 1)
 
     cdef object _on_status_complete(self):
         if self._buf:

docs/index.rst

@@ -118,6 +118,18 @@ Server example:
 For more information please visit :ref:`aiohttp-client` and
 :ref:`aiohttp-web` pages.
 
+Development mode
+================
+
+When writing your code, we recommend enabling Python's
+`development mode <https://docs.python.org/3/library/devmode.html>`_
+(``python -X dev``). In addition to the extra features enabled for asyncio, aiohttp
+will:
+
+- Use a strict parser in the client code (which can help detect malformed responses
+  from a server).
+- Enable some additional checks (resulting in warnings in certain situations).
+
 What's new in aiohttp 3?
 ========================
 

setup.cfg

@@ -130,6 +130,9 @@ addopts =
     # `pytest-cov`:
     --cov=aiohttp
     --cov=tests/
+
+    # run tests that are not marked with dev_mode
+    -m "not dev_mode"
 filterwarnings =
     error
     ignore:module 'ssl' has no attribute 'OP_NO_COMPRESSION'. The Python interpreter is compiled against OpenSSL < 1.0.0. Ref. https.//docs.python.org/3/library/ssl.html#ssl.OP_NO_COMPRESSION:UserWarning
@@ -153,3 +156,5 @@ minversion = 3.8.2
 testpaths = tests/
 junit_family=xunit2
 xfail_strict = true
+markers =
+    dev_mode: mark test to run in dev mode.

tests/test_http_parser.py

@@ -721,6 +721,23 @@
     assert msg.reason == ""
 
 
+def test_http_response_parser_lenient_headers(response) -> None:
+    messages, upgrade, tail = response.feed_data(
+        b"HTTP/1.1 200 test\r\nFoo: abc\x01def\r\n\r\n"
+    )
+    msg = messages[0][0]
+
+    assert msg.headers["Foo"] == "abc\x01def"
+
+
+@pytest.mark.dev_mode
+def test_http_response_parser_strict_headers(response) -> None:
+    if isinstance(response, HttpResponseParserPy):
+        pytest.xfail("Py parser is lenient. May update py-parser later.")
+    with pytest.raises(http_exceptions.BadHttpMessage):
+        response.feed_data(b"HTTP/1.1 200 test\r\nFoo: abc\x01def\r\n\r\n")
+
+
 def test_http_response_parser_bad(response) -> None:
     with pytest.raises(http_exceptions.BadHttpMessage):
         response.feed_data(b"HTT/1\r\n\r\n")