linkcheck builder: begin using session-based HTTP requests

sphinx/builders/linkcheck.py

@@ -279,12 +279,16 @@ def __init__(self, config: Config,
         self.tls_verify = config.tls_verify
         self.tls_cacerts = config.tls_cacerts
 
+        self._session = requests._Session()
+
         super().__init__(daemon=True)
 
     def run(self) -> None:
         while True:
             next_check, hyperlink = self.wqueue.get()
             if hyperlink is None:
+                # An empty hyperlink is a signal to shutdown the worker; cleanup resources here
+                self._session.close()
                 break
 
             uri, docname, _docpath, lineno = hyperlink
@@ -346,6 +350,13 @@ def _check(self, docname: str, uri: str, hyperlink: Hyperlink) -> tuple[str, str
 
         return status, info, code
 
+    def _retrieval_methods(self,
+                           check_anchors: bool,
+                           anchor: str) -> Iterator[tuple[Callable, dict]]:
+        if not check_anchors or not anchor:
+            yield self._session.head, {'allow_redirects': True}
+        yield self._session.get, {'stream': True}
+
     def _check_uri(self, uri: str, hyperlink: Hyperlink) -> tuple[str, str, int]:
         req_url, delimiter, anchor = uri.partition('#')
         for rex in self.anchors_ignore if delimiter and anchor else []:
@@ -377,7 +388,7 @@ def _check_uri(self, uri: str, hyperlink: Hyperlink) -> tuple[str, str, int]:
         error_message = None
         status_code = -1
         response_url = retry_after = ''
-        for retrieval_method, kwargs in _retrieval_methods(self.check_anchors, anchor):
+        for retrieval_method, kwargs in self._retrieval_methods(self.check_anchors, anchor):
             try:
                 with retrieval_method(
                     url=req_url, auth=auth_info,
@@ -508,12 +519,6 @@ def _get_request_headers(
     return {}
 
 
-def _retrieval_methods(check_anchors: bool, anchor: str) -> Iterator[tuple[Callable, dict]]:
-    if not check_anchors or not anchor:
-        yield requests.head, {'allow_redirects': True}
-    yield requests.get, {'stream': True}
-
-
 def contains_anchor(response: Response, anchor: str) -> bool:
     """Determine if an anchor is contained within an HTTP response."""
 

sphinx/util/requests.py

@@ -77,3 +77,38 @@ def head(url: str,
 
     with ignore_insecure_warning(verify):
         return requests.head(url, **kwargs)
+
+
+class _Session(requests.Session):
+
+    def get(self, url: str, **kwargs: Any) -> requests.Response:  # type: ignore[override]
+        """Sends a GET request like requests.get().
+
+        This sets up User-Agent header and TLS verification automatically."""
+        headers = kwargs.setdefault('headers', {})
+        headers.setdefault('User-Agent', kwargs.pop('_user_agent', _USER_AGENT))
+        if '_tls_info' in kwargs:
+            tls_verify, tls_cacerts = kwargs.pop('_tls_info')
+            verify = bool(kwargs.get('verify', tls_verify))
+            kwargs.setdefault('verify', verify and _get_tls_cacert(url, tls_cacerts))
+        else:
+            verify = kwargs.get('verify', True)
+
+        with ignore_insecure_warning(verify):
+            return super().get(url, **kwargs)
+
+    def head(self, url: str, **kwargs: Any) -> requests.Response:  # type: ignore[override]
+        """Sends a HEAD request like requests.head().
+
+        This sets up User-Agent header and TLS verification automatically."""
+        headers = kwargs.setdefault('headers', {})
+        headers.setdefault('User-Agent', kwargs.pop('_user_agent', _USER_AGENT))
+        if '_tls_info' in kwargs:
+            tls_verify, tls_cacerts = kwargs.pop('_tls_info')
+            verify = bool(kwargs.get('verify', tls_verify))
+            kwargs.setdefault('verify', verify and _get_tls_cacert(url, tls_cacerts))
+        else:
+            verify = kwargs.get('verify', True)
+
+        with ignore_insecure_warning(verify):
+            return super().head(url, **kwargs)

tests/test_build_linkcheck.py

@@ -104,7 +104,7 @@ def test_defaults(app):
     with http_server(DefaultsHandler):
         with ConnectionMeasurement() as m:
             app.build()
-        assert m.connection_count <= 10
+        assert m.connection_count <= 5
 
     # Text output
     assert (app.outdir / 'output.txt').exists()