From ecd9c72697ca1529e7cae4693775612987a2a6d2 Mon Sep 17 00:00:00 2001 From: jorop Date: Tue, 25 Apr 2023 19:55:25 +0200 Subject: [PATCH] Support setting response header size limits (#6559) ## What do these changes do? Allow the user to set the response header limits. The properties max_line_size, max_headers and max_field_size can be set on ClientSession creation. Default values are: `max_line_size=8190` `max_headers=32768` and `max_field_size=8190`. Thus the user can set and - when needed - receive bigger response headers. ## Are there changes in behavior for the user? No ## Related issue number #2304 ## Checklist - [x] I think the code is well written - [x] Unit tests for the changes exist - [ ] Documentation reflects the changes - [ ] If you provide code modification, please add yourself to `CONTRIBUTORS.txt` * The format is <Name> <Surname>. * Please keep alphabetical order, the file is sorted by names. - [x] Add a new news fragment into the `CHANGES` folder * name it `.` for example (588.bugfix) * if you don't have an `issue_id` change it to the pr id after creating the pr * ensure type is one of the following: * `.feature`: Signifying a new feature. * `.bugfix`: Signifying a bug fix. * `.doc`: Signifying a documentation improvement. * `.removal`: Signifying a deprecation or removal of public API. * `.misc`: A ticket has been closed, but it is not of interest to users. * Make sure to use full sentences with correct case and punctuation, for example: "Fix issue with non-ascii contents in doctest text files." --------- Co-authored-by: Josef Prenner Co-authored-by: Sam Bull --- CHANGES/2304.feature | 1 + aiohttp/client.py | 20 +++++++ aiohttp/client_proto.py | 4 ++ tests/test_client_functional.py | 102 ++++++++++++++++++++++++++++++++ 4 files changed, 127 insertions(+) create mode 100644 CHANGES/2304.feature diff --git a/CHANGES/2304.feature b/CHANGES/2304.feature new file mode 100644 index 00000000000..c89b812cba2 --- /dev/null +++ b/CHANGES/2304.feature @@ -0,0 +1 @@ +Support setting response header parameters max_line_size and max_field_size. diff --git a/aiohttp/client.py b/aiohttp/client.py index 7132811cd36..0edfb81df10 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -186,6 +186,8 @@ class ClientSession: "_ws_response_class", "_trace_configs", "_read_bufsize", + "_max_line_size", + "_max_field_size", ) def __init__( @@ -213,6 +215,8 @@ def __init__( requote_redirect_url: bool = True, trace_configs: Optional[List[TraceConfig]] = None, read_bufsize: int = 2**16, + max_line_size: int = 8190, + max_field_size: int = 8190, ) -> None: if base_url is None or isinstance(base_url, URL): self._base_url: Optional[URL] = base_url @@ -261,6 +265,8 @@ def __init__( self._trust_env = trust_env self._requote_redirect_url = requote_redirect_url self._read_bufsize = read_bufsize + self._max_line_size = max_line_size + self._max_field_size = max_field_size # Convert to list of tuples if headers: @@ -342,6 +348,8 @@ async def _request( trace_request_ctx: Optional[SimpleNamespace] = None, read_bufsize: Optional[int] = None, auto_decompress: Optional[bool] = None, + max_line_size: Optional[int] = None, + max_field_size: Optional[int] = None, ) -> ClientResponse: # NOTE: timeout clamps existing connect and read timeouts. We cannot # set the default to None because we need to detect if the user wants @@ -404,6 +412,12 @@ async def _request( if auto_decompress is None: auto_decompress = self._auto_decompress + if max_line_size is None: + max_line_size = self._max_line_size + + if max_field_size is None: + max_field_size = self._max_field_size + traces = [ Trace( self, @@ -510,6 +524,8 @@ async def _request( read_timeout=real_timeout.sock_read, read_bufsize=read_bufsize, timeout_ceil_threshold=self._connector._timeout_ceil_threshold, + max_line_size=max_line_size, + max_field_size=max_field_size, ) try: @@ -1186,6 +1202,8 @@ def request( version: HttpVersion = http.HttpVersion11, connector: Optional[BaseConnector] = None, read_bufsize: Optional[int] = None, + max_line_size: int = 8190, + max_field_size: int = 8190, ) -> _SessionRequestContextManager: """Constructs and sends a request. @@ -1256,6 +1274,8 @@ def request( proxy=proxy, proxy_auth=proxy_auth, read_bufsize=read_bufsize, + max_line_size=max_line_size, + max_field_size=max_field_size, ), session, ) diff --git a/aiohttp/client_proto.py b/aiohttp/client_proto.py index 877c5dfeb99..bfa9ea84a97 100644 --- a/aiohttp/client_proto.py +++ b/aiohttp/client_proto.py @@ -153,6 +153,8 @@ def set_response_params( read_timeout: Optional[float] = None, read_bufsize: int = 2**16, timeout_ceil_threshold: float = 5, + max_line_size: int = 8190, + max_field_size: int = 8190, ) -> None: self._skip_payload = skip_payload @@ -169,6 +171,8 @@ def set_response_params( response_with_body=not skip_payload, read_until_eof=read_until_eof, auto_decompress=auto_decompress, + max_line_size=max_line_size, + max_field_size=max_field_size, ) if self._tail: diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index cefd1c83333..84ee9cb8c3a 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -3065,3 +3065,105 @@ async def handler(request): assert resp.status == 200 assert await resp.text() == "ok" assert resp.headers["Content-Type"] == "text/plain; charset=utf-8" + + +async def test_max_field_size_session_default(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response(headers={"Custom": "x" * 8190}) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + + async with await client.get("/") as resp: + assert resp.headers["Custom"] == "x" * 8190 + + +async def test_max_field_size_session_default_fail(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response(headers={"Custom": "x" * 8191}) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + with pytest.raises(aiohttp.ClientResponseError): + await client.get("/") + + +async def test_max_field_size_session_explicit(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response(headers={"Custom": "x" * 8191}) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app, max_field_size=8191) + + async with await client.get("/") as resp: + assert resp.headers["Custom"] == "x" * 8191 + + +async def test_max_field_size_request_explicit(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response(headers={"Custom": "x" * 8191}) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + + async with await client.get("/", max_field_size=8191) as resp: + assert resp.headers["Custom"] == "x" * 8191 + + +async def test_max_line_size_session_default(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response(status=200, reason="x" * 8190) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + + async with await client.get("/") as resp: + assert resp.reason == "x" * 8190 + + +async def test_max_line_size_session_default_fail(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response(status=200, reason="x" * 8192) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + with pytest.raises(aiohttp.ClientResponseError): + await client.get("/") + + +async def test_max_line_size_session_explicit(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response(status=200, reason="x" * 8191) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app, max_line_size=8191) + + async with await client.get("/") as resp: + assert resp.reason == "x" * 8191 + + +async def test_max_line_size_request_explicit(aiohttp_client: Any) -> None: + async def handler(request): + return web.Response(status=200, reason="x" * 8191) + + app = web.Application() + app.add_routes([web.get("/", handler)]) + + client = await aiohttp_client(app) + + async with await client.get("/", max_line_size=8191) as resp: + assert resp.reason == "x" * 8191