From 7b1639203790639c3f3767102168143e1746ca51 Mon Sep 17 00:00:00 2001 From: James Addison Date: Wed, 3 May 2023 17:56:34 +0100 Subject: [PATCH 1/9] tests: linkcheck builder: update test webserver handlers from HTTP/1.0 protocol (default) to HTTP/1.1 --- tests/test_build_linkcheck.py | 43 ++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/tests/test_build_linkcheck.py b/tests/test_build_linkcheck.py index 260cf2c4214..bc143aa94b2 100644 --- a/tests/test_build_linkcheck.py +++ b/tests/test_build_linkcheck.py @@ -27,6 +27,8 @@ class DefaultsHandler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + def do_HEAD(self): if self.path[1:].rstrip() == "": self.send_response(200, "OK") @@ -39,12 +41,22 @@ def do_HEAD(self): self.end_headers() def do_GET(self): - self.do_HEAD() if self.path[1:].rstrip() == "": - self.wfile.write(b"ok\n\n") + content = b"ok\n\n" elif self.path[1:].rstrip() == "anchor.html": doc = '' - self.wfile.write(doc.encode('utf-8')) + content = doc.encode("utf-8") + else: + content = b"" + + if content: + self.send_response(200, "OK") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + else: + self.send_response(404, "Not Found") + self.end_headers() @pytest.mark.sphinx('linkcheck', testroot='linkcheck', freshenv=True) @@ -181,6 +193,8 @@ def test_anchors_ignored(app): @pytest.mark.sphinx('linkcheck', testroot='linkcheck-localserver-anchor', freshenv=True) def test_raises_for_invalid_status(app): class InternalServerErrorHandler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + def do_GET(self): self.send_error(500, "Internal Server Error") @@ -196,6 +210,8 @@ def do_GET(self): def capture_headers_handler(records): class HeadersDumperHandler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + def do_HEAD(self): self.do_GET() @@ -291,6 +307,8 @@ def test_linkcheck_request_headers_default(app): def make_redirect_handler(*, support_head): class RedirectOnceHandler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + def do_HEAD(self): if support_head: self.do_GET() @@ -381,13 +399,18 @@ def test_linkcheck_allowed_redirects(app, warning): class OKHandler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + def do_HEAD(self): self.send_response(200, "OK") self.end_headers() def do_GET(self): - self.do_HEAD() - self.wfile.write(b"ok\n") + content = b"ok\n" + self.send_response(200, "OK") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) @pytest.mark.sphinx('linkcheck', testroot='linkcheck-localserver-https', freshenv=True) @@ -492,15 +515,19 @@ def test_connect_to_selfsigned_nonexistent_cert_file(app): class InfiniteRedirectOnHeadHandler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + def do_HEAD(self): self.send_response(302, "Found") self.send_header("Location", "http://localhost:7777/") self.end_headers() def do_GET(self): + content = b"ok\n" self.send_response(200, "OK") + self.send_header("Content-Length", str(len(content))) self.end_headers() - self.wfile.write(b"ok\n") + self.wfile.write(content) @pytest.mark.sphinx('linkcheck', testroot='linkcheck-localserver', freshenv=True) @@ -526,6 +553,8 @@ def test_TooManyRedirects_on_HEAD(app, monkeypatch): def make_retry_after_handler(responses): class RetryAfterHandler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + def do_HEAD(self): status, retry_after = responses.pop(0) self.send_response(status) @@ -677,6 +706,8 @@ def test_limit_rate_bails_out_after_waiting_max_time(app): class ConnectionResetHandler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + def do_HEAD(self): self.connection.close() From 32f97defc61323a32a064aa89750b826dd21c08d Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 27 Apr 2023 16:38:55 +0100 Subject: [PATCH 2/9] tests: linkcheck: send content-length header for all HTTP/1.1 server responses Ref: https://docs.python.org/3/library/http.server.html#http.server.BaseHTTPRequestHandler.protocol_version --- tests/test_build_linkcheck.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_build_linkcheck.py b/tests/test_build_linkcheck.py index bc143aa94b2..15b4ec37b87 100644 --- a/tests/test_build_linkcheck.py +++ b/tests/test_build_linkcheck.py @@ -32,12 +32,14 @@ class DefaultsHandler(http.server.BaseHTTPRequestHandler): def do_HEAD(self): if self.path[1:].rstrip() == "": self.send_response(200, "OK") + self.send_header("Content-Length", "0") self.end_headers() elif self.path[1:].rstrip() == "anchor.html": self.send_response(200, "OK") self.end_headers() else: self.send_response(404, "Not Found") + self.send_header("Content-Length", "0") self.end_headers() def do_GET(self): @@ -56,6 +58,7 @@ def do_GET(self): self.wfile.write(content) else: self.send_response(404, "Not Found") + self.send_header("Content-Length", "0") self.end_headers() @@ -217,6 +220,7 @@ def do_HEAD(self): def do_GET(self): self.send_response(200, "OK") + self.send_header("Content-Length", "0") self.end_headers() records.append(self.headers.as_string()) return HeadersDumperHandler @@ -314,6 +318,7 @@ def do_HEAD(self): self.do_GET() else: self.send_response(405, "Method Not Allowed") + self.send_header("Content-Length", "0") self.end_headers() def do_GET(self): @@ -322,6 +327,7 @@ def do_GET(self): else: self.send_response(302, "Found") self.send_header("Location", "http://localhost:7777/?redirected=1") + self.send_header("Content-Length", "0") self.end_headers() def log_date_time_string(self): @@ -403,6 +409,7 @@ class OKHandler(http.server.BaseHTTPRequestHandler): def do_HEAD(self): self.send_response(200, "OK") + self.send_header("Content-Length", "0") self.end_headers() def do_GET(self): @@ -520,6 +527,7 @@ class InfiniteRedirectOnHeadHandler(http.server.BaseHTTPRequestHandler): def do_HEAD(self): self.send_response(302, "Found") self.send_header("Location", "http://localhost:7777/") + self.send_header("Content-Length", "0") self.end_headers() def do_GET(self): @@ -560,6 +568,7 @@ def do_HEAD(self): self.send_response(status) if retry_after: self.send_header('Retry-After', retry_after) + self.send_header("Content-Length", "0") self.end_headers() def log_date_time_string(self): @@ -713,6 +722,7 @@ def do_HEAD(self): def do_GET(self): self.send_response(200, "OK") + self.send_header("Content-Length", "0") self.end_headers() From 45053ad54205ac1daac930b0990acb7f04a61e87 Mon Sep 17 00:00:00 2001 From: James Addison Date: Tue, 18 Apr 2023 15:10:43 +0100 Subject: [PATCH 3/9] tests: linkcheck: ConnectionResetHandler: close the connection by setting the 'close_connection' handler attribute instead of calling connection.close Ref: https://docs.python.org/3/library/http.server.html#http.server.BaseHTTPRequestHandler.close_connection --- tests/test_build_linkcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_build_linkcheck.py b/tests/test_build_linkcheck.py index 15b4ec37b87..a57fce05fb7 100644 --- a/tests/test_build_linkcheck.py +++ b/tests/test_build_linkcheck.py @@ -718,7 +718,7 @@ class ConnectionResetHandler(http.server.BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" def do_HEAD(self): - self.connection.close() + self.close_connection = True def do_GET(self): self.send_response(200, "OK") From aec34adec3defcd3362de29dd3e3eba14da23bc7 Mon Sep 17 00:00:00 2001 From: James Addison Date: Fri, 21 Apr 2023 11:29:05 +0100 Subject: [PATCH 4/9] tests: linkcheck: capture_headers_handler: relocate the (client) header capture to before any server-side communication has been initiated by the handler (cherry picked from commit 4d485aee6f304d432121f57c375a479f9674cc5b) --- tests/test_build_linkcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_build_linkcheck.py b/tests/test_build_linkcheck.py index a57fce05fb7..5fab448069b 100644 --- a/tests/test_build_linkcheck.py +++ b/tests/test_build_linkcheck.py @@ -219,10 +219,10 @@ def do_HEAD(self): self.do_GET() def do_GET(self): + records.append(self.headers.as_string()) self.send_response(200, "OK") self.send_header("Content-Length", "0") self.end_headers() - records.append(self.headers.as_string()) return HeadersDumperHandler From 824202e99e69c931c22445d8d3065893a5f285f7 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 27 Apr 2023 19:33:59 +0100 Subject: [PATCH 5/9] tests: linkcheck: InfiniteRedirectOnHeadHandler: close server-side connection after response body is written, since we do not anticipate the client will read it Ref: https://github.com/sphinx-doc/sphinx/pull/11340#issuecomment-1526141880 --- tests/test_build_linkcheck.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_build_linkcheck.py b/tests/test_build_linkcheck.py index 5fab448069b..356e2c8a19d 100644 --- a/tests/test_build_linkcheck.py +++ b/tests/test_build_linkcheck.py @@ -536,6 +536,7 @@ def do_GET(self): self.send_header("Content-Length", str(len(content))) self.end_headers() self.wfile.write(content) + self.close_connection = True # we don't expect the client to read this response body @pytest.mark.sphinx('linkcheck', testroot='linkcheck-localserver', freshenv=True) From b89450cbea91b856d3aea753a15707061c6a6d01 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sun, 14 May 2023 15:22:39 +0100 Subject: [PATCH 6/9] Compatibility issue: pin to 'urllib3<2.0.0' pending support in 'requests' Ref: https://github.com/psf/requests/issues/6432 --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 424af4938f7..d0f636f3142 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ dependencies = [ "alabaster>=0.7,<0.8", "imagesize>=1.3", "requests>=2.25.0", + "urllib3>=1.26.15,<2.0.0", "packaging>=21.0", "importlib-metadata>=4.8; python_version < '3.10'", "colorama>=0.4.5; sys_platform == 'win32'", From c89996367b5182d2ceaff996afef05927e0e0dbd Mon Sep 17 00:00:00 2001 From: James Addison Date: Sun, 14 May 2023 17:06:02 +0100 Subject: [PATCH 7/9] Revert "Compatibility issue: pin to 'urllib3<2.0.0' pending support in 'requests'" This reverts commit b89450cbea91b856d3aea753a15707061c6a6d01. --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d0f636f3142..424af4938f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,6 @@ dependencies = [ "alabaster>=0.7,<0.8", "imagesize>=1.3", "requests>=2.25.0", - "urllib3>=1.26.15,<2.0.0", "packaging>=21.0", "importlib-metadata>=4.8; python_version < '3.10'", "colorama>=0.4.5; sys_platform == 'win32'", From e405f918bad9ce2ea5b5169b3b9975912ceec764 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sun, 14 May 2023 20:02:09 +0100 Subject: [PATCH 8/9] unit tests: enable multi-threaded HTTP webserver --- tests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index 429bbd2b2e2..32636b7936c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -19,7 +19,7 @@ class HttpServerThread(threading.Thread): def __init__(self, handler, *args, **kwargs): super().__init__(*args, **kwargs) - self.server = http.server.HTTPServer(("localhost", 7777), handler) + self.server = http.server.ThreadingHTTPServer(("localhost", 7777), handler) def run(self): self.server.serve_forever(poll_interval=0.001) From f88728d689bf69a58df8938af30724317d1e248d Mon Sep 17 00:00:00 2001 From: James Addison <55152140+jayaddison@users.noreply.github.com> Date: Sat, 22 Jul 2023 10:14:35 +0100 Subject: [PATCH 9/9] linkcheck builder tests: add missing content-length header transmission --- tests/test_build_linkcheck.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_build_linkcheck.py b/tests/test_build_linkcheck.py index 19e90326013..5750c8bca23 100644 --- a/tests/test_build_linkcheck.py +++ b/tests/test_build_linkcheck.py @@ -37,6 +37,7 @@ def do_HEAD(self): self.end_headers() elif self.path[1:].rstrip() == "anchor.html": self.send_response(200, "OK") + self.send_header("Content-Length", "0") self.end_headers() else: self.send_response(404, "Not Found")