Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support setting response header size limits #6559

Merged
merged 8 commits into from
Apr 25, 2023
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGES/2304.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Support setting response header parameters max_line_size, max_headers and max_field_size.
Dreamsorcerer marked this conversation as resolved.
Show resolved Hide resolved
30 changes: 30 additions & 0 deletions aiohttp/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,9 @@ class ClientSession:
"_ws_response_class",
"_trace_configs",
"_read_bufsize",
"_max_line_size",
"_max_headers",
Dreamsorcerer marked this conversation as resolved.
Show resolved Hide resolved
"_max_field_size",
)

def __init__(
Expand Down Expand Up @@ -213,6 +216,9 @@ def __init__(
requote_redirect_url: bool = True,
trace_configs: Optional[List[TraceConfig]] = None,
read_bufsize: int = 2**16,
max_line_size: int = 8190,
max_headers: int = 32768,
Dreamsorcerer marked this conversation as resolved.
Show resolved Hide resolved
max_field_size: int = 8190,
) -> None:
if base_url is None or isinstance(base_url, URL):
self._base_url: Optional[URL] = base_url
Expand Down Expand Up @@ -261,6 +267,9 @@ def __init__(
self._trust_env = trust_env
self._requote_redirect_url = requote_redirect_url
self._read_bufsize = read_bufsize
self._max_line_size = max_line_size
self._max_headers = max_headers
Dreamsorcerer marked this conversation as resolved.
Show resolved Hide resolved
self._max_field_size = max_field_size

# Convert to list of tuples
if headers:
Expand Down Expand Up @@ -342,6 +351,9 @@ async def _request(
trace_request_ctx: Optional[SimpleNamespace] = None,
read_bufsize: Optional[int] = None,
auto_decompress: Optional[bool] = None,
max_line_size: Optional[int] = None,
max_headers: Optional[int] = None,
Dreamsorcerer marked this conversation as resolved.
Show resolved Hide resolved
max_field_size: Optional[int] = None,
) -> ClientResponse:
# NOTE: timeout clamps existing connect and read timeouts. We cannot
# set the default to None because we need to detect if the user wants
Expand Down Expand Up @@ -404,6 +416,15 @@ async def _request(
if auto_decompress is None:
auto_decompress = self._auto_decompress

if max_line_size is None:
max_line_size = self._max_line_size

if max_headers is None:
max_headers = self._max_headers

Dreamsorcerer marked this conversation as resolved.
Show resolved Hide resolved
if max_field_size is None:
max_field_size = self._max_field_size

traces = [
Trace(
self,
Expand Down Expand Up @@ -510,6 +531,9 @@ async def _request(
read_timeout=real_timeout.sock_read,
read_bufsize=read_bufsize,
timeout_ceil_threshold=self._connector._timeout_ceil_threshold,
max_line_size=max_line_size,
max_headers=max_headers,
Dreamsorcerer marked this conversation as resolved.
Show resolved Hide resolved
max_field_size=max_field_size,
)

try:
Expand Down Expand Up @@ -1186,6 +1210,9 @@ def request(
version: HttpVersion = http.HttpVersion11,
connector: Optional[BaseConnector] = None,
read_bufsize: Optional[int] = None,
max_line_size: int = 8190,
max_headers: int = 32768,
Dreamsorcerer marked this conversation as resolved.
Show resolved Hide resolved
max_field_size: int = 8190,
) -> _SessionRequestContextManager:
"""Constructs and sends a request.

Expand Down Expand Up @@ -1256,6 +1283,9 @@ def request(
proxy=proxy,
proxy_auth=proxy_auth,
read_bufsize=read_bufsize,
max_line_size=max_line_size,
max_headers=max_headers,
Dreamsorcerer marked this conversation as resolved.
Show resolved Hide resolved
max_field_size=max_field_size,
),
session,
)
6 changes: 6 additions & 0 deletions aiohttp/client_proto.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,9 @@ def set_response_params(
read_timeout: Optional[float] = None,
read_bufsize: int = 2**16,
timeout_ceil_threshold: float = 5,
max_line_size: int = 8190,
max_headers: int = 32768,
Dreamsorcerer marked this conversation as resolved.
Show resolved Hide resolved
max_field_size: int = 8190,
) -> None:
self._skip_payload = skip_payload

Expand All @@ -169,6 +172,9 @@ def set_response_params(
response_with_body=not skip_payload,
read_until_eof=read_until_eof,
auto_decompress=auto_decompress,
max_line_size=max_line_size,
max_headers=max_headers,
Dreamsorcerer marked this conversation as resolved.
Show resolved Hide resolved
max_field_size=max_field_size,
)

if self._tail:
Expand Down
121 changes: 121 additions & 0 deletions tests/test_client_functional.py
Original file line number Diff line number Diff line change
Expand Up @@ -3065,3 +3065,124 @@ async def handler(request):
assert resp.status == 200
assert await resp.text() == "ok"
assert resp.headers["Content-Type"] == "text/plain; charset=utf-8"


async def test_max_field_size_session_default(aiohttp_client: Any) -> None:
async def handler(request):
return web.Response(headers={"Custom": "x" * 8190})

app = web.Application()
app.add_routes([web.get("/", handler)])

client = await aiohttp_client(app)

async with await client.get("/") as resp:
assert resp.headers["Custom"] == "x" * 8190


async def test_max_field_size_session_default_fail(aiohttp_client: Any) -> None:
async def handler(request):
return web.Response(headers={"Custom": "x" * 8191})

app = web.Application()
app.add_routes([web.get("/", handler)])

client = await aiohttp_client(app)
with pytest.raises(aiohttp.ClientResponseError):
await client.get("/")


async def test_max_field_size_session_explicit(aiohttp_client: Any) -> None:
async def handler(request):
return web.Response(headers={"Custom": "x" * 8191})

app = web.Application()
app.add_routes([web.get("/", handler)])

client = await aiohttp_client(app, max_field_size=8191)

async with await client.get("/") as resp:
assert resp.headers["Custom"] == "x" * 8191


async def test_max_field_size_request_explicit(aiohttp_client: Any) -> None:
async def handler(request):
return web.Response(headers={"Custom": "x" * 8191})

app = web.Application()
app.add_routes([web.get("/", handler)])

client = await aiohttp_client(app)

async with await client.get("/", max_field_size=8191) as resp:
assert resp.headers["Custom"] == "x" * 8191


async def test_max_line_size_session_default(aiohttp_client: Any) -> None:
async def handler(request):
return web.Response(status=200, reason="x" * 8190)

app = web.Application()
app.add_routes([web.get("/", handler)])

client = await aiohttp_client(app)

async with await client.get("/") as resp:
assert resp.reason == "x" * 8190


async def test_max_line_size_session_default_fail(aiohttp_client: Any) -> None:
async def handler(request):
return web.Response(status=200, reason="x" * 8192)

app = web.Application()
app.add_routes([web.get("/", handler)])

client = await aiohttp_client(app)
with pytest.raises(aiohttp.ClientResponseError):
await client.get("/")


async def test_max_line_size_session_explicit(aiohttp_client: Any) -> None:
async def handler(request):
return web.Response(status=200, reason="x" * 8191)

app = web.Application()
app.add_routes([web.get("/", handler)])

client = await aiohttp_client(app, max_line_size=8191)

async with await client.get("/") as resp:
assert resp.reason == "x" * 8191


async def test_max_line_size_request_explicit(aiohttp_client: Any) -> None:
async def handler(request):
return web.Response(status=200, reason="x" * 8191)

app = web.Application()
app.add_routes([web.get("/", handler)])

client = await aiohttp_client(app)

async with await client.get("/", max_line_size=8191) as resp:
assert resp.reason == "x" * 8191


async def test_max_headers_session_default(aiohttp_client: Any) -> None:
async def handler(request):
# generate 32764 headers:
# 32768 (max_headers default) minus 4 headers which are set implicitly
# 'Content-Length', 'Content-Type', 'Date' and 'Server'
headers = MultiDict()
for x in range(32764):
headers.add(f"x-header-{x}", str(x))
return web.Response(headers=headers)

app = web.Application()
app.add_routes([web.get("/", handler)])

client = await aiohttp_client(app)

async with await client.get("/") as resp:
assert len(resp.headers) == 32768
Dreamsorcerer marked this conversation as resolved.
Show resolved Hide resolved