Support --hostnames (#1325)

* Support --hostnames

* Support python binaries that are not called "python"

* Update log statement to use self.hostname now

---------

Co-authored-by: Abhinav Singh <126065+abhinavsingh@users.noreply.github.com>

Makefile

@@ -1,4 +1,5 @@
 SHELL := /bin/bash
+PYTHON ?= python
 
 NS ?= abhinavsingh
 IMAGE_NAME ?= proxy.py
@@ -40,23 +41,23 @@ all: lib-test
 
 https-certificates:
 	# Generate server key
-	python -m proxy.common.pki gen_private_key \
+	$(PYTHON) -m proxy.common.pki gen_private_key \
 		--private-key-path $(HTTPS_KEY_FILE_PATH)
-	python -m proxy.common.pki remove_passphrase \
+	$(PYTHON) -m proxy.common.pki remove_passphrase \
 		--private-key-path $(HTTPS_KEY_FILE_PATH)
 	# Generate server certificate
-	python -m proxy.common.pki gen_public_key \
+	$(PYTHON) -m proxy.common.pki gen_public_key \
 		--private-key-path $(HTTPS_KEY_FILE_PATH) \
 		--public-key-path $(HTTPS_CERT_FILE_PATH)
 
 sign-https-certificates:
 	# Generate CSR request
-	python -m proxy.common.pki gen_csr \
+	$(PYTHON) -m proxy.common.pki gen_csr \
 		--csr-path $(HTTPS_CSR_FILE_PATH) \
 		--private-key-path $(HTTPS_KEY_FILE_PATH) \
 		--public-key-path $(HTTPS_CERT_FILE_PATH)
 	# Sign CSR with CA
-	python -m proxy.common.pki sign_csr \
+	$(PYTHON) -m proxy.common.pki sign_csr \
 		--csr-path $(HTTPS_CSR_FILE_PATH) \
 		--crt-path $(HTTPS_SIGNED_CERT_FILE_PATH) \
 		--hostname localhost \
@@ -65,23 +66,23 @@ sign-https-certificates:
 
 ca-certificates:
 	# Generate CA key
-	python -m proxy.common.pki gen_private_key \
+	$(PYTHON) -m proxy.common.pki gen_private_key \
 		--private-key-path $(CA_KEY_FILE_PATH)
-	python -m proxy.common.pki remove_passphrase \
+	$(PYTHON) -m proxy.common.pki remove_passphrase \
 		--private-key-path $(CA_KEY_FILE_PATH)
 	# Generate CA certificate
-	python -m proxy.common.pki gen_public_key \
+	$(PYTHON) -m proxy.common.pki gen_public_key \
 		--private-key-path $(CA_KEY_FILE_PATH) \
 		--public-key-path $(CA_CERT_FILE_PATH)
 	# Generate key that will be used to generate domain certificates on the fly
 	# Generated certificates are then signed with CA certificate / key generated above
-	python -m proxy.common.pki gen_private_key \
+	$(PYTHON) -m proxy.common.pki gen_private_key \
 		--private-key-path $(CA_SIGNING_KEY_FILE_PATH)
-	python -m proxy.common.pki remove_passphrase \
+	$(PYTHON) -m proxy.common.pki remove_passphrase \
 		--private-key-path $(CA_SIGNING_KEY_FILE_PATH)
 
 lib-check:
-	python check.py
+	$(PYTHON) check.py
 
 lib-clean:
 	find . -name '*.pyc' -exec rm -f {} +
@@ -107,10 +108,10 @@ lib-dep:
 	pip install "setuptools>=42"
 
 lib-pre-commit:
-	python -m pre_commit run --hook-stage manual --all-files -v
+	$(PYTHON) -m pre_commit run --hook-stage manual --all-files -v
 
 lib-lint:
-	python -m tox -e lint
+	$(PYTHON) -m tox -e lint
 
 lib-flake8:
 	tox -e lint -- flake8 --all-files
@@ -119,12 +120,12 @@ lib-mypy:
 	tox -e lint -- mypy --all-files
 
 lib-pytest:
-	python -m tox -e python -- -v
+	$(PYTHON) -m tox -e python -- -v
 
 lib-test: lib-clean lib-check lib-lint lib-pytest
 
 lib-package: lib-clean lib-check
-	python -m tox -e cleanup-dists,build-dists,metadata-validation
+	$(PYTHON) -m tox -e cleanup-dists,build-dists,metadata-validation
 
 lib-release-test: lib-package
 	twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/*
@@ -133,7 +134,7 @@ lib-release: lib-package
 	twine upload dist/*
 
 lib-doc:
-	python -m tox -e build-docs && \
+	$(PYTHON) -m tox -e build-docs && \
 	$(OPEN) .tox/build-docs/docs_out/index.html || true
 
 lib-coverage: lib-clean
@@ -145,7 +146,7 @@ lib-profile:
 	sudo py-spy record \
 		-o profile.svg \
 		-t -F -s -- \
-		python -m proxy \
+		$(PYTHON) -m proxy \
 			--hostname 127.0.0.1 \
 			--num-acceptors 1 \
 			--num-workers 1 \
@@ -161,7 +162,7 @@ lib-speedscope:
 		-o profile.speedscope.json \
 		-f speedscope \
 		-t -F -s -- \
-		python -m proxy \
+		$(PYTHON) -m proxy \
 			--hostname 127.0.0.1 \
 			--num-acceptors 1 \
 			--num-workers 1 \

README.md

@@ -213,7 +213,8 @@
     - `--enable-reverse-proxy --plugins proxy.plugin.ReverseProxyPlugin`
   - Plugin API is currently in *development phase*. Expect breaking changes. See [Deploying proxy.py in production](#deploying-proxypy-in-production) on how to ensure reliability across code changes.
 
-- Can listen on multiple ports
+- Can listen on multiple addresses and ports
+  - Use `--hostnames` flag to provide additional addresses
   - Use `--ports` flag to provide additional ports
   - Optionally, use `--port` flag to override default port `8899`
   - Capable of serving multiple protocols over the same port
@@ -2335,8 +2336,9 @@ usage: -m [-h] [--tunnel-hostname TUNNEL_HOSTNAME] [--tunnel-port TUNNEL_PORT]
           [--tunnel-remote-port TUNNEL_REMOTE_PORT] [--threadless]
           [--threaded] [--num-workers NUM_WORKERS] [--enable-events]
           [--local-executor LOCAL_EXECUTOR] [--backlog BACKLOG]
-          [--hostname HOSTNAME] [--port PORT] [--ports PORTS [PORTS ...]]
-          [--port-file PORT_FILE] [--unix-socket-path UNIX_SOCKET_PATH]
+          [--hostname HOSTNAME] [--hostnames HOSTNAMES [HOSTNAMES ...]]
+          [--port PORT] [--ports PORTS [PORTS ...]] [--port-file PORT_FILE]
+          [--unix-socket-path UNIX_SOCKET_PATH]
           [--num-acceptors NUM_ACCEPTORS] [--version] [--log-level LOG_LEVEL]
           [--log-file LOG_FILE] [--log-format LOG_FORMAT]
           [--open-file-limit OPEN_FILE_LIMIT]
@@ -2405,6 +2407,8 @@ options:
   --backlog BACKLOG     Default: 100. Maximum number of pending connections to
                         proxy server.
   --hostname HOSTNAME   Default: 127.0.0.1. Server IP address.
+  --hostnames HOSTNAMES [HOSTNAMES ...]
+                        Default: None. Additional IP addresses to listen on.
   --port PORT           Default: 8899. Server port. To listen on more ports,
                         pass them using --ports flag.
   --ports PORTS [PORTS ...]

docs/changelog-fragments.d/1325.feature.md

@@ -0,0 +1 @@
+Support `--hostnames` to specify multiple IP addresses to listen on.

proxy/common/flag.py

@@ -11,7 +11,6 @@
 import os
 import sys
 import base64
-import socket
 import argparse
 import ipaddress
 import itertools
@@ -25,7 +24,7 @@
 from .plugins import Plugins
 from .version import __version__
 from .constants import (
-    COMMA, IS_WINDOWS, PLUGIN_PAC_FILE, PLUGIN_DASHBOARD, PLUGIN_HTTP_PROXY,
+    COMMA, PLUGIN_PAC_FILE, PLUGIN_DASHBOARD, PLUGIN_HTTP_PROXY,
     PLUGIN_PROXY_AUTH, PLUGIN_WEB_SERVER, DEFAULT_NUM_WORKERS,
     PLUGIN_REVERSE_PROXY, DEFAULT_NUM_ACCEPTORS, PLUGIN_INSPECT_TRAFFIC,
     DEFAULT_DISABLE_HEADERS, PY2_DEPRECATION_MESSAGE, DEFAULT_DEVTOOLS_WS_PATH,
@@ -291,24 +290,15 @@ def initialize(
             IpAddress,
             opts.get('hostname', ipaddress.ip_address(args.hostname)),
         )
+        hostnames: List[List[str]] = opts.get('hostnames', args.hostnames)
+        args.hostnames = [
+            ipaddress.ip_address(hostname) for hostname in list(
+                itertools.chain.from_iterable([] if hostnames is None else hostnames),
+            )
+        ]
         args.unix_socket_path = opts.get(
             'unix_socket_path', args.unix_socket_path,
         )
-        # AF_UNIX is not available on Windows
-        # See https://bugs.python.org/issue33408
-        if not IS_WINDOWS:
-            args.family = socket.AF_UNIX if args.unix_socket_path else (
-                socket.AF_INET6 if args.hostname.version == 6 else socket.AF_INET
-            )
-        else:
-            # FIXME: Not true for tests, as this value will be a mock.
-            #
-            # It's a problem only on Windows.  Instead of a proper
-            # fix in the tests, simply commenting this line of assertion
-            # for now.
-            #
-            # assert args.unix_socket_path is None
-            args.family = socket.AF_INET6 if args.hostname.version == 6 else socket.AF_INET
         args.port = cast(int, opts.get('port', args.port))
         ports: List[List[int]] = opts.get('ports', args.ports)
         args.ports = [

proxy/core/acceptor/acceptor.py

@@ -186,12 +186,8 @@ def _recv_and_setup_socks(self) -> None:
         # dynamically accept from new fds.
         for _ in range(self.fd_queue.recv()):
             fileno = recv_handle(self.fd_queue)
-            # TODO: Convert to socks i.e. list of fds
-            self.socks[fileno] = socket.fromfd(
-                fileno,
-                family=self.flags.family,
-                type=socket.SOCK_STREAM,
-            )
+            sock = socket.socket(fileno=socket.dup(fileno))  # type: ignore[attr-defined]
+            self.socks[fileno] = sock
         self.fd_queue.close()
 
     def _start_local(self) -> None:

proxy/core/listener/pool.py

@@ -9,6 +9,7 @@
     :license: BSD, see LICENSE for more details.
 """
 import argparse
+import itertools
 from typing import TYPE_CHECKING, Any, List, Type
 
 from .tcp import TcpSocketListener
@@ -37,10 +38,12 @@ def __exit__(self, *args: Any) -> None:
     def setup(self) -> None:
         if self.flags.unix_socket_path:
             self.add(UnixSocketListener)
-        else:
-            self.add(TcpSocketListener)
-        for port in self.flags.ports:
-            self.add(TcpSocketListener, port=port)
+        hostnames = {self.flags.hostname, *self.flags.hostnames}
+        ports = set(self.flags.ports)
+        if not self.flags.unix_socket_path:
+            ports.add(self.flags.port)
+        for hostname, port in itertools.product(hostnames, ports):
+            self.add(TcpSocketListener, hostname=hostname, port=port)
 
     def shutdown(self) -> None:
         for listener in self.pool:

proxy/core/listener/tcp.py

@@ -10,7 +10,8 @@
 """
 import socket
 import logging
-from typing import Any, Optional
+import ipaddress
+from typing import Any, Union, Optional
 
 from .base import BaseListener
 from ...common.flag import flags
@@ -26,6 +27,15 @@
     help='Default: 127.0.0.1. Server IP address.',
 )
 
+flags.add_argument(
+    '--hostnames',
+    action='append',
+    nargs='+',
+    type=str,
+    default=None,
+    help='Default: None.  Additional IP addresses to listen on.',
+)
+
 flags.add_argument(
     '--port',
     type=int,
@@ -37,6 +47,7 @@
     '--ports',
     action='append',
     nargs='+',
+    type=int,
     default=None,
     help='Default: None.  Additional ports to listen on.',
 )
@@ -54,9 +65,14 @@
 class TcpSocketListener(BaseListener):
     """Tcp listener."""
 
-    def __init__(self, *args: Any, port: Optional[int] = None, **kwargs: Any) -> None:
-        # Port if passed will be used, otherwise
-        # flag port value will be used.
+    def __init__(
+        self,
+        hostname: Union[ipaddress.IPv4Address, ipaddress.IPv6Address],
+        port: int,
+        *args: Any,
+        **kwargs: Any,
+    ) -> None:
+        self.hostname = hostname
         self.port = port
         # Set after binding to a port.
         #
@@ -66,19 +82,18 @@ def __init__(self, *args: Any, port: Optional[int] = None, **kwargs: Any) -> Non
 
     def listen(self) -> socket.socket:
         sock = socket.socket(
-            socket.AF_INET6 if self.flags.hostname.version == 6 else socket.AF_INET,
+            socket.AF_INET6 if self.hostname.version == 6 else socket.AF_INET,
             socket.SOCK_STREAM,
         )
         sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
         sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
         # s.setsockopt(socket.SOL_TCP, socket.TCP_FASTOPEN, 5)
-        port = self.port if self.port is not None else self.flags.port
-        sock.bind((str(self.flags.hostname), port))
+        sock.bind((str(self.hostname), self.port))
         sock.listen(self.flags.backlog)
         sock.setblocking(False)
         self._port = sock.getsockname()[1]
         logger.info(
             'Listening on %s:%s' %
-            (self.flags.hostname, self._port),
+            (self.hostname, self._port),
         )
         return sock

proxy/core/work/fd/fd.py

@@ -30,10 +30,7 @@ def work(self, *args: Any) -> None:
         fileno: int = args[0]
         addr: Optional[HostPort] = args[1]
         conn: Optional[TcpOrTlsSocket] = args[2]
-        conn = conn or socket.fromfd(
-            fileno, family=socket.AF_INET if self.flags.hostname.version == 4 else socket.AF_INET6,
-            type=socket.SOCK_STREAM,
-        )
+        conn = conn or socket.socket(fileno=socket.dup(fileno))  # type: ignore[attr-defined]
         uid = '%s-%s-%s' % (self.iid, self._total, fileno)
         self.works[fileno] = self.create(uid, conn, addr)
         self.works[fileno].publish_event(

proxy/proxy.py

@@ -211,16 +211,18 @@ def setup(self) -> None:
             )._port
         # --ports flag can also use 0 as value for ephemeral port selection.
         # Here, we override flags.ports to reflect actual listening ports.
-        ports = []
-        offset = 1 if self.flags.unix_socket_path or self.flags.port else 0
+        ports = set()
+        offset = 1 if self.flags.unix_socket_path else 0
         for index in range(offset, offset + len(self.flags.ports)):
-            ports.append(
+            ports.add(
                 cast(
                     'TcpSocketListener',
                     self.listeners.pool[index],
                 )._port,
             )
-        self.flags.ports = ports
+        if self.flags.port in ports:
+            ports.remove(self.flags.port)
+        self.flags.ports = list(ports)
         # Write ports to port file
         self._write_port_file()
         # Setup EventManager