Support urllib3 1.26.x and 2.x (#121)

* Support urllib3 1.26.x and 2.x

This changes the assert_fingerprint hack to more directly tell urllib3
that we'll assert the fingerprint ourselves to add support for pinning
root certificates, not only the leaves.

* Fix mypy

---------

Co-authored-by: Seth Michael Larson <seth.larson@elastic.co>

elastic_transport/_node/_http_urllib3.py

@@ -21,6 +21,11 @@
 import warnings
 from typing import Any, Dict, Optional, Union
 
+try:
+    from importlib import metadata
+except ImportError:
+    import importlib_metadata as metadata  # type: ignore[import,no-redef]
+
 import urllib3
 from urllib3.exceptions import ConnectTimeoutError, NewConnectionError, ReadTimeoutError
 from urllib3.util.retry import Retry
@@ -47,7 +52,7 @@
 class Urllib3HttpNode(BaseNode):
     """Default synchronous node class using the ``urllib3`` library via HTTP"""
 
-    _CLIENT_META_HTTP_CLIENT = ("ur", client_meta_version(urllib3.__version__))
+    _CLIENT_META_HTTP_CLIENT = ("ur", client_meta_version(metadata.version("urllib3")))
 
     def __init__(self, config: NodeConfig):
         super().__init__(config)
@@ -159,13 +164,13 @@ def perform_request(
             else:
                 body_to_send = None
 
-            response = self.pool.urlopen(  # type: ignore[no-untyped-call]
+            response = self.pool.urlopen(
                 method,
                 target,
                 body=body_to_send,
                 retries=Retry(False),
                 headers=request_headers,
-                **kw,
+                **kw,  # type: ignore[arg-type]
             )
             response_headers = HttpHeaders(response.headers)
             data = response.data

elastic_transport/_node/_urllib3_chain_certs.py

@@ -36,7 +36,21 @@
 __all__ = ["HTTPSConnectionPool"]
 
 
+class HTTPSConnection(urllib3.connection.HTTPSConnection):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        self._elastic_assert_fingerprint: Optional[str] = None
+        super().__init__(*args, **kwargs)
+
+    def connect(self) -> None:
+        super().connect()
+        # Hack to prevent a warning within HTTPSConnectionPool._validate_conn()
+        if self._elastic_assert_fingerprint:
+            self.is_verified = True
+
+
 class HTTPSConnectionPool(urllib3.HTTPSConnectionPool):
+    ConnectionCls = HTTPSConnection
+
     """HTTPSConnectionPool implementation which supports ``assert_fingerprint``
     on certificates within the chain instead of only the leaf cert using private
     APIs in CPython 3.10+
@@ -60,18 +74,26 @@ def __init__(
                 f", should be one of '{valid_lengths}'"
             )
 
-        if assert_fingerprint:
-            # Falsey but not None. This is a hack to skip fingerprinting by urllib3
-            # but still set 'is_verified=True' within HTTPSConnectionPool._validate_conn()
-            kwargs["assert_fingerprint"] = ""
+        if self._elastic_assert_fingerprint:
+            # Skip fingerprinting by urllib3 as we'll do it ourselves
+            kwargs["assert_fingerprint"] = None
 
         super().__init__(*args, **kwargs)
 
-    def _validate_conn(self, conn: urllib3.connection.HTTPSConnection) -> None:
+    def _new_conn(self) -> HTTPSConnection:
+        """
+        Return a fresh :class:`urllib3.connection.HTTPSConnection`.
+        """
+        conn: HTTPSConnection = super()._new_conn()  # type: ignore[assignment]
+        # Tell our custom connection if we'll assert fingerprint ourselves
+        conn._elastic_assert_fingerprint = self._elastic_assert_fingerprint
+        return conn
+
+    def _validate_conn(self, conn: HTTPSConnection) -> None:  # type: ignore[override]
         """
         Called right before a request is made, after the socket is created.
         """
-        super(HTTPSConnectionPool, self)._validate_conn(conn)  # type: ignore[misc]
+        super(HTTPSConnectionPool, self)._validate_conn(conn)
 
         if self._elastic_assert_fingerprint:
             hash_func = _HASHES_BY_LENGTH[len(self._elastic_assert_fingerprint)]
@@ -89,7 +111,7 @@ def _validate_conn(self, conn: urllib3.connection.HTTPSConnection) -> None:
                 # See: https://github.com/python/cpython/pull/25467
                 fingerprints = [
                     hash_func(cert.public_bytes(_ENCODING_DER)).digest()
-                    for cert in conn.sock._sslobj.get_verified_chain()
+                    for cert in conn.sock._sslobj.get_verified_chain()  # type: ignore[union-attr]
                 ]
             except RERAISE_EXCEPTIONS:  # pragma: nocover
                 raise
@@ -100,7 +122,7 @@ def _validate_conn(self, conn: urllib3.connection.HTTPSConnection) -> None:
 
             # Only add the peercert in front of the chain if it's not there for some reason.
             # This is to make sure old behavior of 'ssl_assert_fingerprint' still works.
-            peercert_fingerprint = hash_func(conn.sock.getpeercert(True)).digest()
+            peercert_fingerprint = hash_func(conn.sock.getpeercert(True)).digest()  # type: ignore[union-attr]
             if peercert_fingerprint not in fingerprints:  # pragma: nocover
                 fingerprints.insert(0, peercert_fingerprint)
 

noxfile.py

@@ -43,11 +43,14 @@ def lint(session):
         "flake8",
         "black~=23.0",
         "isort",
-        "mypy==1.0.1",
-        "types-urllib3",
+        "mypy==1.5.1",
         "types-requests",
         "types-certifi",
     )
+    # https://github.com/python/typeshed/issues/10786
+    session.run(
+        "python", "-m", "pip", "uninstall", "--yes", "types-urllib3", silent=True
+    )
     session.install(".[develop]")
     session.run("black", "--check", "--target-version=py36", *SOURCE_FILES)
     session.run("isort", "--check", *SOURCE_FILES)

setup.py

@@ -48,9 +48,10 @@
     package_data={"elastic_transport": ["py.typed"]},
     packages=packages,
     install_requires=[
-        "urllib3>=1.26.2, <2",
+        "urllib3>=1.26.2, <3",
         "certifi",
         "dataclasses; python_version<'3.7'",
+        "importlib-metadata; python_version<'3.8'",
     ],
     python_requires=">=3.6",
     extras_require={