Skip to content

Commit b114244

Browse files
yukiiiteruyukiiiteru
authoredJan 25, 2024
feat(http1): support configurable max_headers(num) to client and server (#3523)
·
v1.6.0v1.2.0
1 parent 7177770 commit b114244

File tree

7 files changed

+235
-7
lines changed

7 files changed

+235
-7
lines changed
 

‎Cargo.toml

+3-2
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ httpdate = { version = "1.0", optional = true }
3636
itoa = { version = "1", optional = true }
3737
libc = { version = "0.2", optional = true }
3838
pin-project-lite = { version = "0.2.4", optional = true }
39+
smallvec = { version = "1.12", features = ["const_generics", "const_new"], optional = true }
3940
tracing = { version = "0.1", default-features = false, features = ["std"], optional = true }
4041
want = { version = "0.3", optional = true }
4142

@@ -80,8 +81,8 @@ http1 = ["dep:futures-channel", "dep:futures-util", "dep:httparse", "dep:itoa"]
8081
http2 = ["dep:futures-channel", "dep:futures-util", "dep:h2"]
8182

8283
# Client/Server
83-
client = ["dep:want", "dep:pin-project-lite"]
84-
server = ["dep:httpdate", "dep:pin-project-lite"]
84+
client = ["dep:want", "dep:pin-project-lite", "dep:smallvec"]
85+
server = ["dep:httpdate", "dep:pin-project-lite", "dep:smallvec"]
8586

8687
# C-API support (currently unstable (no semver))
8788
ffi = ["dep:libc", "dep:http-body-util"]

‎src/client/conn/http1.rs

+23
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ pub struct Builder {
115115
h1_writev: Option<bool>,
116116
h1_title_case_headers: bool,
117117
h1_preserve_header_case: bool,
118+
h1_max_headers: Option<usize>,
118119
#[cfg(feature = "ffi")]
119120
h1_preserve_header_order: bool,
120121
h1_read_buf_exact_size: Option<usize>,
@@ -309,6 +310,7 @@ impl Builder {
309310
h1_parser_config: Default::default(),
310311
h1_title_case_headers: false,
311312
h1_preserve_header_case: false,
313+
h1_max_headers: None,
312314
#[cfg(feature = "ffi")]
313315
h1_preserve_header_order: false,
314316
h1_max_buf_size: None,
@@ -439,6 +441,24 @@ impl Builder {
439441
self
440442
}
441443

444+
/// Set the maximum number of headers.
445+
///
446+
/// When a response is received, the parser will reserve a buffer to store headers for optimal
447+
/// performance.
448+
///
449+
/// If client receives more headers than the buffer size, the error "message header too large"
450+
/// is returned.
451+
///
452+
/// Note that headers is allocated on the stack by default, which has higher performance. After
453+
/// setting this value, headers will be allocated in heap memory, that is, heap memory
454+
/// allocation will occur for each response, and there will be a performance drop of about 5%.
455+
///
456+
/// Default is 100.
457+
pub fn max_headers(&mut self, val: usize) -> &mut Self {
458+
self.h1_max_headers = Some(val);
459+
self
460+
}
461+
442462
/// Set whether to support preserving original header order.
443463
///
444464
/// Currently, this will record the order in which headers are received, and store this
@@ -519,6 +539,9 @@ impl Builder {
519539
if opts.h1_preserve_header_case {
520540
conn.set_preserve_header_case();
521541
}
542+
if let Some(max_headers) = opts.h1_max_headers {
543+
conn.set_http1_max_headers(max_headers);
544+
}
522545
#[cfg(feature = "ffi")]
523546
if opts.h1_preserve_header_order {
524547
conn.set_preserve_header_order();

‎src/proto/h1/conn.rs

+7
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ where
5454
keep_alive: KA::Busy,
5555
method: None,
5656
h1_parser_config: ParserConfig::default(),
57+
h1_max_headers: None,
5758
#[cfg(feature = "server")]
5859
h1_header_read_timeout: None,
5960
#[cfg(feature = "server")]
@@ -132,6 +133,10 @@ where
132133
self.state.h09_responses = true;
133134
}
134135

136+
pub(crate) fn set_http1_max_headers(&mut self, val: usize) {
137+
self.state.h1_max_headers = Some(val);
138+
}
139+
135140
#[cfg(feature = "server")]
136141
pub(crate) fn set_http1_header_read_timeout(&mut self, val: Duration) {
137142
self.state.h1_header_read_timeout = Some(val);
@@ -207,6 +212,7 @@ where
207212
cached_headers: &mut self.state.cached_headers,
208213
req_method: &mut self.state.method,
209214
h1_parser_config: self.state.h1_parser_config.clone(),
215+
h1_max_headers: self.state.h1_max_headers,
210216
#[cfg(feature = "server")]
211217
h1_header_read_timeout: self.state.h1_header_read_timeout,
212218
#[cfg(feature = "server")]
@@ -847,6 +853,7 @@ struct State {
847853
/// a body or not.
848854
method: Option<Method>,
849855
h1_parser_config: ParserConfig,
856+
h1_max_headers: Option<usize>,
850857
#[cfg(feature = "server")]
851858
h1_header_read_timeout: Option<Duration>,
852859
#[cfg(feature = "server")]

‎src/proto/h1/io.rs

+2
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ where
184184
cached_headers: parse_ctx.cached_headers,
185185
req_method: parse_ctx.req_method,
186186
h1_parser_config: parse_ctx.h1_parser_config.clone(),
187+
h1_max_headers: parse_ctx.h1_max_headers,
187188
#[cfg(feature = "server")]
188189
h1_header_read_timeout: parse_ctx.h1_header_read_timeout,
189190
#[cfg(feature = "server")]
@@ -725,6 +726,7 @@ mod tests {
725726
cached_headers: &mut None,
726727
req_method: &mut None,
727728
h1_parser_config: Default::default(),
729+
h1_max_headers: None,
728730
h1_header_read_timeout: None,
729731
h1_header_read_timeout_fut: &mut None,
730732
h1_header_read_timeout_running: &mut false,

‎src/proto/h1/mod.rs

+1
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ pub(crate) struct ParseContext<'a> {
7878
cached_headers: &'a mut Option<HeaderMap>,
7979
req_method: &'a mut Option<Method>,
8080
h1_parser_config: ParserConfig,
81+
h1_max_headers: Option<usize>,
8182
#[cfg(feature = "server")]
8283
h1_header_read_timeout: Option<Duration>,
8384
#[cfg(feature = "server")]

‎src/proto/h1/role.rs

+176-5
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ use http::header::Entry;
1313
use http::header::ValueIter;
1414
use http::header::{self, HeaderMap, HeaderName, HeaderValue};
1515
use http::{Method, StatusCode, Version};
16+
use smallvec::{smallvec, smallvec_inline, SmallVec};
1617

1718
use crate::body::DecodedLength;
1819
#[cfg(feature = "server")]
@@ -29,7 +30,7 @@ use crate::proto::h1::{
2930
use crate::proto::RequestHead;
3031
use crate::proto::{BodyLength, MessageHead, RequestLine};
3132

32-
const MAX_HEADERS: usize = 100;
33+
const DEFAULT_MAX_HEADERS: usize = 100;
3334
const AVERAGE_HEADER_SIZE: usize = 30; // totally scientific
3435
#[cfg(feature = "server")]
3536
const MAX_URI_LEN: usize = (u16::MAX - 1) as usize;
@@ -139,9 +140,17 @@ impl Http1Transaction for Server {
139140
// but we *never* read any of it until after httparse has assigned
140141
// values into it. By not zeroing out the stack memory, this saves
141142
// a good ~5% on pipeline benchmarks.
142-
let mut headers_indices = [MaybeUninit::<HeaderIndices>::uninit(); MAX_HEADERS];
143+
let mut headers_indices: SmallVec<[MaybeUninit<HeaderIndices>; DEFAULT_MAX_HEADERS]> =
144+
match ctx.h1_max_headers {
145+
Some(cap) => smallvec![MaybeUninit::uninit(); cap],
146+
None => smallvec_inline![MaybeUninit::uninit(); DEFAULT_MAX_HEADERS],
147+
};
143148
{
144-
let mut headers = [MaybeUninit::<httparse::Header<'_>>::uninit(); MAX_HEADERS];
149+
let mut headers: SmallVec<[MaybeUninit<httparse::Header<'_>>; DEFAULT_MAX_HEADERS]> =
150+
match ctx.h1_max_headers {
151+
Some(cap) => smallvec![MaybeUninit::uninit(); cap],
152+
None => smallvec_inline![MaybeUninit::uninit(); DEFAULT_MAX_HEADERS],
153+
};
145154
trace!(bytes = buf.len(), "Request.parse");
146155
let mut req = httparse::Request::new(&mut []);
147156
let bytes = buf.as_ref();
@@ -966,9 +975,18 @@ impl Http1Transaction for Client {
966975

967976
// Loop to skip information status code headers (100 Continue, etc).
968977
loop {
969-
let mut headers_indices = [MaybeUninit::<HeaderIndices>::uninit(); MAX_HEADERS];
978+
let mut headers_indices: SmallVec<[MaybeUninit<HeaderIndices>; DEFAULT_MAX_HEADERS]> =
979+
match ctx.h1_max_headers {
980+
Some(cap) => smallvec![MaybeUninit::uninit(); cap],
981+
None => smallvec_inline![MaybeUninit::uninit(); DEFAULT_MAX_HEADERS],
982+
};
970983
let (len, status, reason, version, headers_len) = {
971-
let mut headers = [MaybeUninit::<httparse::Header<'_>>::uninit(); MAX_HEADERS];
984+
let mut headers: SmallVec<
985+
[MaybeUninit<httparse::Header<'_>>; DEFAULT_MAX_HEADERS],
986+
> = match ctx.h1_max_headers {
987+
Some(cap) => smallvec![MaybeUninit::uninit(); cap],
988+
None => smallvec_inline![MaybeUninit::uninit(); DEFAULT_MAX_HEADERS],
989+
};
972990
trace!(bytes = buf.len(), "Response.parse");
973991
let mut res = httparse::Response::new(&mut []);
974992
let bytes = buf.as_ref();
@@ -1610,6 +1628,7 @@ mod tests {
16101628
cached_headers: &mut None,
16111629
req_method: &mut method,
16121630
h1_parser_config: Default::default(),
1631+
h1_max_headers: None,
16131632
h1_header_read_timeout: None,
16141633
h1_header_read_timeout_fut: &mut None,
16151634
h1_header_read_timeout_running: &mut false,
@@ -1641,6 +1660,7 @@ mod tests {
16411660
cached_headers: &mut None,
16421661
req_method: &mut Some(crate::Method::GET),
16431662
h1_parser_config: Default::default(),
1663+
h1_max_headers: None,
16441664
h1_header_read_timeout: None,
16451665
h1_header_read_timeout_fut: &mut None,
16461666
h1_header_read_timeout_running: &mut false,
@@ -1667,6 +1687,7 @@ mod tests {
16671687
cached_headers: &mut None,
16681688
req_method: &mut None,
16691689
h1_parser_config: Default::default(),
1690+
h1_max_headers: None,
16701691
h1_header_read_timeout: None,
16711692
h1_header_read_timeout_fut: &mut None,
16721693
h1_header_read_timeout_running: &mut false,
@@ -1691,6 +1712,7 @@ mod tests {
16911712
cached_headers: &mut None,
16921713
req_method: &mut Some(crate::Method::GET),
16931714
h1_parser_config: Default::default(),
1715+
h1_max_headers: None,
16941716
h1_header_read_timeout: None,
16951717
h1_header_read_timeout_fut: &mut None,
16961718
h1_header_read_timeout_running: &mut false,
@@ -1717,6 +1739,7 @@ mod tests {
17171739
cached_headers: &mut None,
17181740
req_method: &mut Some(crate::Method::GET),
17191741
h1_parser_config: Default::default(),
1742+
h1_max_headers: None,
17201743
h1_header_read_timeout: None,
17211744
h1_header_read_timeout_fut: &mut None,
17221745
h1_header_read_timeout_running: &mut false,
@@ -1747,6 +1770,7 @@ mod tests {
17471770
cached_headers: &mut None,
17481771
req_method: &mut Some(crate::Method::GET),
17491772
h1_parser_config,
1773+
h1_max_headers: None,
17501774
h1_header_read_timeout: None,
17511775
h1_header_read_timeout_fut: &mut None,
17521776
h1_header_read_timeout_running: &mut false,
@@ -1774,6 +1798,7 @@ mod tests {
17741798
cached_headers: &mut None,
17751799
req_method: &mut Some(crate::Method::GET),
17761800
h1_parser_config: Default::default(),
1801+
h1_max_headers: None,
17771802
h1_header_read_timeout: None,
17781803
h1_header_read_timeout_fut: &mut None,
17791804
h1_header_read_timeout_running: &mut false,
@@ -1796,6 +1821,7 @@ mod tests {
17961821
cached_headers: &mut None,
17971822
req_method: &mut None,
17981823
h1_parser_config: Default::default(),
1824+
h1_max_headers: None,
17991825
h1_header_read_timeout: None,
18001826
h1_header_read_timeout_fut: &mut None,
18011827
h1_header_read_timeout_running: &mut false,
@@ -1839,6 +1865,7 @@ mod tests {
18391865
cached_headers: &mut None,
18401866
req_method: &mut None,
18411867
h1_parser_config: Default::default(),
1868+
h1_max_headers: None,
18421869
h1_header_read_timeout: None,
18431870
h1_header_read_timeout_fut: &mut None,
18441871
h1_header_read_timeout_running: &mut false,
@@ -1863,6 +1890,7 @@ mod tests {
18631890
cached_headers: &mut None,
18641891
req_method: &mut None,
18651892
h1_parser_config: Default::default(),
1893+
h1_max_headers: None,
18661894
h1_header_read_timeout: None,
18671895
h1_header_read_timeout_fut: &mut None,
18681896
h1_header_read_timeout_running: &mut false,
@@ -2096,6 +2124,7 @@ mod tests {
20962124
cached_headers: &mut None,
20972125
req_method: &mut Some(Method::GET),
20982126
h1_parser_config: Default::default(),
2127+
h1_max_headers: None,
20992128
h1_header_read_timeout: None,
21002129
h1_header_read_timeout_fut: &mut None,
21012130
h1_header_read_timeout_running: &mut false,
@@ -2120,6 +2149,7 @@ mod tests {
21202149
cached_headers: &mut None,
21212150
req_method: &mut Some(m),
21222151
h1_parser_config: Default::default(),
2152+
h1_max_headers: None,
21232153
h1_header_read_timeout: None,
21242154
h1_header_read_timeout_fut: &mut None,
21252155
h1_header_read_timeout_running: &mut false,
@@ -2144,6 +2174,7 @@ mod tests {
21442174
cached_headers: &mut None,
21452175
req_method: &mut Some(Method::GET),
21462176
h1_parser_config: Default::default(),
2177+
h1_max_headers: None,
21472178
h1_header_read_timeout: None,
21482179
h1_header_read_timeout_fut: &mut None,
21492180
h1_header_read_timeout_running: &mut false,
@@ -2663,6 +2694,7 @@ mod tests {
26632694
cached_headers: &mut None,
26642695
req_method: &mut Some(Method::GET),
26652696
h1_parser_config: Default::default(),
2697+
h1_max_headers: None,
26662698
h1_header_read_timeout: None,
26672699
h1_header_read_timeout_fut: &mut None,
26682700
h1_header_read_timeout_running: &mut false,
@@ -2681,6 +2713,143 @@ mod tests {
26812713
assert_eq!(parsed.head.headers["server"], "hello\tworld");
26822714
}
26832715

2716+
#[test]
2717+
fn parse_too_large_headers() {
2718+
fn gen_req_with_headers(num: usize) -> String {
2719+
let mut req = String::from("GET / HTTP/1.1\r\n");
2720+
for i in 0..num {
2721+
req.push_str(&format!("key{i}: val{i}\r\n"));
2722+
}
2723+
req.push_str("\r\n");
2724+
req
2725+
}
2726+
fn gen_resp_with_headers(num: usize) -> String {
2727+
let mut req = String::from("HTTP/1.1 200 OK\r\n");
2728+
for i in 0..num {
2729+
req.push_str(&format!("key{i}: val{i}\r\n"));
2730+
}
2731+
req.push_str("\r\n");
2732+
req
2733+
}
2734+
fn parse(max_headers: Option<usize>, gen_size: usize, should_success: bool) {
2735+
{
2736+
// server side
2737+
let mut bytes = BytesMut::from(gen_req_with_headers(gen_size).as_str());
2738+
let result = Server::parse(
2739+
&mut bytes,
2740+
ParseContext {
2741+
cached_headers: &mut None,
2742+
req_method: &mut None,
2743+
h1_parser_config: Default::default(),
2744+
h1_max_headers: max_headers,
2745+
h1_header_read_timeout: None,
2746+
h1_header_read_timeout_fut: &mut None,
2747+
h1_header_read_timeout_running: &mut false,
2748+
timer: Time::Empty,
2749+
preserve_header_case: false,
2750+
#[cfg(feature = "ffi")]
2751+
preserve_header_order: false,
2752+
h09_responses: false,
2753+
#[cfg(feature = "ffi")]
2754+
on_informational: &mut None,
2755+
},
2756+
);
2757+
if should_success {
2758+
result.expect("parse ok").expect("parse complete");
2759+
} else {
2760+
result.expect_err("parse should err");
2761+
}
2762+
}
2763+
{
2764+
// client side
2765+
let mut bytes = BytesMut::from(gen_resp_with_headers(gen_size).as_str());
2766+
let result = Client::parse(
2767+
&mut bytes,
2768+
ParseContext {
2769+
cached_headers: &mut None,
2770+
req_method: &mut None,
2771+
h1_parser_config: Default::default(),
2772+
h1_max_headers: max_headers,
2773+
h1_header_read_timeout: None,
2774+
h1_header_read_timeout_fut: &mut None,
2775+
h1_header_read_timeout_running: &mut false,
2776+
timer: Time::Empty,
2777+
preserve_header_case: false,
2778+
#[cfg(feature = "ffi")]
2779+
preserve_header_order: false,
2780+
h09_responses: false,
2781+
#[cfg(feature = "ffi")]
2782+
on_informational: &mut None,
2783+
},
2784+
);
2785+
if should_success {
2786+
result.expect("parse ok").expect("parse complete");
2787+
} else {
2788+
result.expect_err("parse should err");
2789+
}
2790+
}
2791+
}
2792+
2793+
// check generator
2794+
assert_eq!(
2795+
gen_req_with_headers(0),
2796+
String::from("GET / HTTP/1.1\r\n\r\n")
2797+
);
2798+
assert_eq!(
2799+
gen_req_with_headers(1),
2800+
String::from("GET / HTTP/1.1\r\nkey0: val0\r\n\r\n")
2801+
);
2802+
assert_eq!(
2803+
gen_req_with_headers(2),
2804+
String::from("GET / HTTP/1.1\r\nkey0: val0\r\nkey1: val1\r\n\r\n")
2805+
);
2806+
assert_eq!(
2807+
gen_req_with_headers(3),
2808+
String::from("GET / HTTP/1.1\r\nkey0: val0\r\nkey1: val1\r\nkey2: val2\r\n\r\n")
2809+
);
2810+
2811+
// default max_headers is 100, so
2812+
//
2813+
// - less than or equal to 100, accepted
2814+
//
2815+
parse(None, 0, true);
2816+
parse(None, 1, true);
2817+
parse(None, 50, true);
2818+
parse(None, 99, true);
2819+
parse(None, 100, true);
2820+
//
2821+
// - more than 100, rejected
2822+
//
2823+
parse(None, 101, false);
2824+
parse(None, 102, false);
2825+
parse(None, 200, false);
2826+
2827+
// max_headers is 0, parser will reject any headers
2828+
//
2829+
// - without header, accepted
2830+
//
2831+
parse(Some(0), 0, true);
2832+
//
2833+
// - with header(s), rejected
2834+
//
2835+
parse(Some(0), 1, false);
2836+
parse(Some(0), 100, false);
2837+
2838+
// max_headers is 200
2839+
//
2840+
// - less than or equal to 200, accepted
2841+
//
2842+
parse(Some(200), 0, true);
2843+
parse(Some(200), 1, true);
2844+
parse(Some(200), 100, true);
2845+
parse(Some(200), 200, true);
2846+
//
2847+
// - more than 200, rejected
2848+
//
2849+
parse(Some(200), 201, false);
2850+
parse(Some(200), 210, false);
2851+
}
2852+
26842853
#[test]
26852854
fn test_write_headers_orig_case_empty_value() {
26862855
let mut headers = HeaderMap::new();
@@ -2751,6 +2920,7 @@ mod tests {
27512920
cached_headers: &mut headers,
27522921
req_method: &mut None,
27532922
h1_parser_config: Default::default(),
2923+
h1_max_headers: None,
27542924
h1_header_read_timeout: None,
27552925
h1_header_read_timeout_fut: &mut None,
27562926
h1_header_read_timeout_running: &mut false,
@@ -2795,6 +2965,7 @@ mod tests {
27952965
cached_headers: &mut headers,
27962966
req_method: &mut None,
27972967
h1_parser_config: Default::default(),
2968+
h1_max_headers: None,
27982969
h1_header_read_timeout: None,
27992970
h1_header_read_timeout_fut: &mut None,
28002971
h1_header_read_timeout_running: &mut false,

‎src/server/conn/http1.rs

+23
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ pub struct Builder {
7575
h1_keep_alive: bool,
7676
h1_title_case_headers: bool,
7777
h1_preserve_header_case: bool,
78+
h1_max_headers: Option<usize>,
7879
h1_header_read_timeout: Dur,
7980
h1_writev: Option<bool>,
8081
max_buf_size: Option<usize>,
@@ -242,6 +243,7 @@ impl Builder {
242243
h1_keep_alive: true,
243244
h1_title_case_headers: false,
244245
h1_preserve_header_case: false,
246+
h1_max_headers: None,
245247
h1_header_read_timeout: Dur::Default(Some(Duration::from_secs(30))),
246248
h1_writev: None,
247249
max_buf_size: None,
@@ -294,6 +296,24 @@ impl Builder {
294296
self
295297
}
296298

299+
/// Set the maximum number of headers.
300+
///
301+
/// When a request is received, the parser will reserve a buffer to store headers for optimal
302+
/// performance.
303+
///
304+
/// If server receives more headers than the buffer size, it responds to the client with
305+
/// "431 Request Header Fields Too Large".
306+
///
307+
/// Note that headers is allocated on the stack by default, which has higher performance. After
308+
/// setting this value, headers will be allocated in heap memory, that is, heap memory
309+
/// allocation will occur for each request, and there will be a performance drop of about 5%.
310+
///
311+
/// Default is 100.
312+
pub fn max_headers(&mut self, val: usize) -> &mut Self {
313+
self.h1_max_headers = Some(val);
314+
self
315+
}
316+
297317
/// Set a timeout for reading client request headers. If a client does not
298318
/// transmit the entire header within this time, the connection is closed.
299319
///
@@ -412,6 +432,9 @@ impl Builder {
412432
if self.h1_preserve_header_case {
413433
conn.set_preserve_header_case();
414434
}
435+
if let Some(max_headers) = self.h1_max_headers {
436+
conn.set_http1_max_headers(max_headers);
437+
}
415438
if let Some(dur) = self
416439
.timer
417440
.check(self.h1_header_read_timeout, "header_read_timeout")

0 commit comments

Comments
 (0)
Please sign in to comment.