Use lenient headers for response parser (#7490)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
(cherry picked from commit 6396531)

.github/workflows/ci-cd.yml

@@ -222,6 +222,13 @@ jobs:
       run: >-  # `exit 1` makes sure that the job remains red with flaky runs
         pytest --no-cov -vvvvv --lf && exit 1
       shell: bash
+    - name: Run dev_mode tests
+      env:
+        COLOR: yes
+        AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }}
+        PIP_USER: 1
+      run: python -X dev -m pytest -m dev_mode --cov-append
+      shell: bash
     - name: Turn coverage into xml
       env:
         COLOR: 'yes'

CHANGES/7490.feature

@@ -0,0 +1 @@
+Enabled lenient headers for more flexible parsing in the client. -- by :user:`Dreamsorcerer`

aiohttp/_http_parser.pyx

@@ -20,6 +20,7 @@ from multidict import CIMultiDict as _CIMultiDict, CIMultiDictProxy as _CIMultiD
 from yarl import URL as _URL
 
 from aiohttp import hdrs
+from aiohttp.helpers import DEBUG
 
 from .http_exceptions import (
     BadHttpMessage,
@@ -648,6 +649,9 @@ cdef class HttpResponseParser(HttpParser):
                    max_line_size, max_headers, max_field_size,
                    payload_exception, response_with_body, read_until_eof,
                    auto_decompress)
+        # Use strict parsing on dev mode, so users are warned about broken servers.
+        if not DEBUG:
+            cparser.llhttp_set_lenient_headers(self._cparser, 1)
 
     cdef object _on_status_complete(self):
         if self._buf:

docs/index.rst

@@ -118,6 +118,18 @@ Server example:
 For more information please visit :ref:`aiohttp-client` and
 :ref:`aiohttp-web` pages.
 
+Development mode
+================
+
+When writing your code, we recommend enabling Python's
+`development mode <https://docs.python.org/3/library/devmode.html>`_
+(``python -X dev``). In addition to the extra features enabled for asyncio, aiohttp
+will:
+
+- Use a strict parser in the client code (which can help detect malformed responses
+  from a server).
+- Enable some additional checks (resulting in warnings in certain situations).
+
 What's new in aiohttp 3?
 ========================
 

tests/test_http_parser.py

@@ -729,6 +729,23 @@ def test_http_response_parser_no_reason(response) -> None:
     assert msg.reason == ""
 
 
+def test_http_response_parser_lenient_headers(response) -> None:
+    messages, upgrade, tail = response.feed_data(
+        b"HTTP/1.1 200 test\r\nFoo: abc\x01def\r\n\r\n"
+    )
+    msg = messages[0][0]
+
+    assert msg.headers["Foo"] == "abc\x01def"
+
+
+@pytest.mark.dev_mode
+def test_http_response_parser_strict_headers(response) -> None:
+    if isinstance(response, HttpResponseParserPy):
+        pytest.xfail("Py parser is lenient. May update py-parser later.")
+    with pytest.raises(http_exceptions.BadHttpMessage):
+        response.feed_data(b"HTTP/1.1 200 test\r\nFoo: abc\x01def\r\n\r\n")
+
+
 def test_http_response_parser_bad(response) -> None:
     with pytest.raises(http_exceptions.BadHttpMessage):
         response.feed_data(b"HTT/1\r\n\r\n")