Add a DOWNLOAD_BIND_ADDRESS setting for download handlers (#7283)

docs/topics/request-response.rst

@@ -644,7 +644,36 @@ Those are:
 bindaddress
 -----------
 
-The IP of the outgoing IP address to use for the performing the request.
+The default local outgoing address for download-handler connections.
+
+This setting can be either:
+
+- a host address as a string (e.g. ``"127.0.0.2"``), in which case the local
+  port is chosen automatically, or
+
+- a ``(host, port)`` tuple (e.g. ``("127.0.0.2", 50000)``) to bind to both a
+  specific local interface and a specific local port.
+
+For example:
+
+.. code-block:: python
+
+    Request(
+        "https://example.org",
+        meta={"bindaddress": "127.0.0.2"},
+    )
+
+.. code-block:: python
+
+    Request(
+        "https://example.org",
+        meta={"bindaddress": ("127.0.0.2", 50000)},
+    )
+
+If not set, built-in HTTP download handlers use the value of
+:setting:`DOWNLOAD_BIND_ADDRESS` as the default bind address.
+Set the :reqmeta:`bindaddress` request meta key to override it for a
+specific request.
 
 .. reqmeta:: download_timeout
 

docs/topics/settings.rst

@@ -897,6 +897,39 @@ It is also possible to change this setting per domain, although it requires
 non-trivial code. See the implementation of the :ref:`AutoThrottle
 <topics-autothrottle>` extension for an example.
 
+.. setting:: DOWNLOAD_BIND_ADDRESS
+
+DOWNLOAD_BIND_ADDRESS
+---------------------
+
+Default: ``None``
+
+The default local outgoing address for download-handler connections.
+
+This setting can be either:
+
+- a host address as a string (e.g. ``"127.0.0.2"``), in which case the local
+  port is chosen automatically, or
+
+- a ``(host, port)`` tuple (e.g. ``("127.0.0.2", 50000)``) to bind to both a
+  specific local interface and a specific local port.
+
+For example:
+
+.. code-block:: python
+
+    # Bind to this local address
+    DOWNLOAD_BIND_ADDRESS = "127.0.0.2"
+
+.. code-block:: python
+
+    # Bind to this local address and local port
+    DOWNLOAD_BIND_ADDRESS = ("127.0.0.2", 5000)
+
+If set, built-in HTTP download handlers use this value by default.
+Set the :reqmeta:`bindaddress` request meta key to override it for a specific
+request.
+
 .. setting:: DOWNLOAD_HANDLERS
 
 DOWNLOAD_HANDLERS

scrapy/core/downloader/handlers/_httpx.py

@@ -30,6 +30,7 @@ from scrapy.utils._download_handlers import (
     get_maxsize_msg,
     get_warnsize_msg,
     make_response,
+    normalize_bind_address,
 )
 from scrapy.utils.asyncio import is_asyncio_available
 from scrapy.utils.ssl import _log_sslobj_debug_info, _make_ssl_context
@@ -87,8 +88,30 @@ class HttpxDownloadHandler(BaseHttpDownloadHandler):
         self._tls_verbose_logging: bool = self.crawler.settings.getbool(
             "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
         )
+        bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS")
+        bind_address = normalize_bind_address(bind_address)
+
+        self._bind_address: str | None = None
+
+        if bind_address is not None:
+            host, port = bind_address
+            if port != 0:
+                logger.warning(
+                    "DOWNLOAD_BIND_ADDRESS specifies a port (%s), but %s does not "
+                    "support binding to a specific local port. Ignoring the port "
+                    "and binding only to %r.",
+                    port,
+                    type(self).__name__,
+                    host,
+                )
+            self._bind_address = host
+
         self._client = httpx.AsyncClient(
-            verify=_make_ssl_context(crawler.settings), cookies=_NullCookieJar()
+            cookies=_NullCookieJar(),
+            transport=httpx.AsyncHTTPTransport(
+                verify=_make_ssl_context(crawler.settings),
+                local_address=self._bind_address,
+            ),
         )
 
     async def download_request(self, request: Request) -> Response:
@@ -108,7 +131,11 @@ class HttpxDownloadHandler(BaseHttpDownloadHandler):
         except httpx.UnsupportedProtocol as e:
             raise UnsupportedURLSchemeError(str(e)) from e
         except httpx.ConnectError as e:
-            if "Name or service not known" in str(e) or "getaddrinfo failed" in str(e):
+            if (
+                "Name or service not known" in str(e)
+                or "getaddrinfo failed" in str(e)
+                or "nodename nor servname" in str(e)
+            ):
                 raise CannotResolveHostError(str(e)) from e
             raise DownloadConnectionRefusedError(str(e)) from e
         except httpx.NetworkError as e:

scrapy/core/downloader/handlers/http11.py

@@ -45,6 +45,7 @@ from scrapy.utils._download_handlers import (
     get_maxsize_msg,
     get_warnsize_msg,
     make_response,
+    normalize_bind_address,
     wrap_twisted_exceptions,
 )
 from scrapy.utils.defer import maybe_deferred_to_future
@@ -93,6 +94,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
         self._contextFactory: IPolicyForHTTPS = load_context_factory_from_settings(
             crawler.settings, crawler
         )
+        self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS")
         self._disconnect_timeout: int = 1
 
     async def download_request(self, request: Request) -> Response:
@@ -106,6 +108,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
 
         agent = ScrapyAgent(
             contextFactory=self._contextFactory,
+            bindAddress=self._bind_address,
             pool=self._pool,
             maxsize=getattr(
                 self._crawler.spider, "download_maxsize", self._default_maxsize
@@ -286,7 +289,7 @@ class TunnelingAgent(Agent):
         proxyConf: tuple[str, int, bytes | None],
         contextFactory: IPolicyForHTTPS,
         connectTimeout: float | None = None,
-        bindAddress: bytes | None = None,
+        bindAddress: tuple[str, int] | None = None,
         pool: HTTPConnectionPool | None = None,
     ):
         super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
@@ -335,7 +338,7 @@ class ScrapyProxyAgent(Agent):
         reactor: ReactorBase,
         proxyURI: bytes,
         connectTimeout: float | None = None,
-        bindAddress: bytes | None = None,
+        bindAddress: tuple[str, int] | None = None,
         pool: HTTPConnectionPool | None = None,
     ):
         super().__init__(
@@ -379,7 +382,7 @@ class ScrapyAgent:
         *,
         contextFactory: IPolicyForHTTPS,
         connectTimeout: float = 10,
-        bindAddress: bytes | None = None,
+        bindAddress: str | tuple[str, int] | None = None,
         pool: HTTPConnectionPool | None = None,
         maxsize: int = 0,
         warnsize: int = 0,
@@ -388,7 +391,7 @@ class ScrapyAgent:
     ):
         self._contextFactory: IPolicyForHTTPS = contextFactory
         self._connectTimeout: float = connectTimeout
-        self._bindAddress: bytes | None = bindAddress
+        self._bindAddress: str | tuple[str, int] | None = bindAddress
         self._pool: HTTPConnectionPool | None = pool
         self._maxsize: int = maxsize
         self._warnsize: int = warnsize
@@ -400,6 +403,7 @@ class ScrapyAgent:
         from twisted.internet import reactor
 
         bindaddress = request.meta.get("bindaddress") or self._bindAddress
+        bindaddress = normalize_bind_address(bindaddress)
         proxy = request.meta.get("proxy")
         if proxy:
             proxy = add_http_if_no_scheme(proxy)

scrapy/core/downloader/handlers/http2.py

@@ -10,7 +10,10 @@ from scrapy.core.downloader.contextfactory import load_context_factory_from_sett
 from scrapy.core.downloader.handlers.base import BaseDownloadHandler
 from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent
 from scrapy.exceptions import DownloadTimeoutError
-from scrapy.utils._download_handlers import wrap_twisted_exceptions
+from scrapy.utils._download_handlers import (
+    normalize_bind_address,
+    wrap_twisted_exceptions,
+)
 from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_bytes
@@ -38,11 +41,13 @@ class H2DownloadHandler(BaseDownloadHandler):
         self._context_factory = load_context_factory_from_settings(
             crawler.settings, crawler
         )
+        self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS")
 
     async def download_request(self, request: Request) -> Response:
         agent = ScrapyH2Agent(
             context_factory=self._context_factory,
             pool=self._pool,
+            bind_address=self._bind_address,
             crawler=self._crawler,
         )
         assert self._crawler.spider
@@ -64,7 +69,7 @@ class ScrapyH2Agent:
         context_factory: IPolicyForHTTPS,
         pool: H2ConnectionPool,
         connect_timeout: int = 10,
-        bind_address: bytes | None = None,
+        bind_address: str | tuple[str, int] | None = None,
         crawler: Crawler | None = None,
     ) -> None:
         self._context_factory = context_factory
@@ -77,6 +82,7 @@ class ScrapyH2Agent:
         from twisted.internet import reactor
 
         bind_address = request.meta.get("bindaddress") or self._bind_address
+        bind_address = normalize_bind_address(bind_address)
         proxy = request.meta.get("proxy")
         if proxy:
             if urlparse_cached(request).scheme == "https":

scrapy/core/http2/agent.py

@@ -122,7 +122,7 @@ class H2Agent:
         pool: H2ConnectionPool,
         context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(),  # noqa: B008
         connect_timeout: float | None = None,
-        bind_address: bytes | None = None,
+        bind_address: tuple[str, int] | None = None,
     ) -> None:
         self._reactor = reactor
         self._pool = pool
@@ -166,7 +166,7 @@ class ScrapyProxyH2Agent(H2Agent):
         pool: H2ConnectionPool,
         context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(),  # noqa: B008
         connect_timeout: float | None = None,
-        bind_address: bytes | None = None,
+        bind_address: tuple[str, int] | None = None,
     ) -> None:
         super().__init__(
             reactor=reactor,

scrapy/settings/default_settings.py

@@ -59,6 +59,7 @@ __all__ = [
     "DOWNLOADER_MIDDLEWARES",
     "DOWNLOADER_MIDDLEWARES_BASE",
     "DOWNLOADER_STATS",
+    "DOWNLOAD_BIND_ADDRESS",
     "DOWNLOAD_DELAY",
     "DOWNLOAD_FAIL_ON_DATALOSS",
     "DOWNLOAD_HANDLERS",
@@ -245,6 +246,8 @@ DNS_TIMEOUT = 60
 
 DOWNLOAD_DELAY = 0
 
+DOWNLOAD_BIND_ADDRESS = None
+
 DOWNLOAD_FAIL_ON_DATALOSS = True
 
 DOWNLOAD_HANDLERS = {}

scrapy/utils/_download_handlers.py

@@ -146,3 +146,11 @@ def get_dataloss_msg(url: str) -> str:
         f"responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
         f" -- This message won't be shown in further requests"
     )
+
+
+def normalize_bind_address(
+    value: str | tuple[str, int] | None,
+) -> tuple[str, int] | None:
+    if isinstance(value, str):
+        return (value, 0)
+    return value

tests/mockserver/http.py

@@ -14,6 +14,7 @@ from .http_resources import (
     BrokenChunkedResource,
     BrokenDownloadResource,
     ChunkedResource,
+    ClientIPResource,
     Compress,
     ContentLengthHeaderResource,
     Delay,
@@ -72,6 +73,7 @@ class Root(resource.Resource):
         self.putChild(b"wait", ForeverTakingResource())
         self.putChild(b"hang-after-headers", ForeverTakingResource(write=True))
         self.putChild(b"host", HostHeaderResource())
+        self.putChild(b"client-ip", ClientIPResource())
         self.putChild(b"broken", BrokenDownloadResource())
         self.putChild(b"chunked", ChunkedResource())
         self.putChild(b"broken-chunked", BrokenChunkedResource())

tests/mockserver/http_resources.py

@@ -56,6 +56,18 @@ class HostHeaderResource(resource.Resource):
         return request.requestHeaders.getRawHeaders(b"host")[0]
 
 
+class ClientIPResource(resource.Resource):
+    """
+    A testing resource which renders itself as the request client IP address.
+    """
+
+    def render(self, request):
+        client_address = request.getClientAddress()
+        if client_address is None or client_address.host is None:
+            return b""
+        return to_bytes(client_address.host)
+
+
 class PayloadResource(resource.Resource):
     """
     A testing resource which renders itself as the contents of the request body

tests/test_downloader_handler_httpx.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import sys
 from typing import TYPE_CHECKING, Any
 
 import pytest
@@ -44,7 +45,7 @@ class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base):
     async def test_unsupported_bindaddress(
         self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
     ) -> None:
-        meta = {"bindaddress": "127.0.0.2"}
+        meta = {"bindaddress": ("127.0.0.2", 0)}
         request = Request(mockserver.url("/text"), meta=meta)
         async with self.get_dh() as download_handler:
             response = await download_handler.download_request(request)
@@ -54,6 +55,24 @@ class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base):
             in caplog.text
         )
 
+    # skip macOS tests
+    @pytest.mark.skipif(
+        sys.platform == "darwin",
+        reason="127.0.0.2 is not available on macOS by default",
+    )
+    @coroutine_test
+    async def test_bind_address_port_warning(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
+        request = Request(mockserver.url("/client-ip"))
+        async with self.get_dh(
+            {"DOWNLOAD_BIND_ADDRESS": ("127.0.0.2", 12345)}
+        ) as download_handler:
+            response = await download_handler.download_request(request)
+        assert response.body == b"127.0.0.2"
+        assert "DOWNLOAD_BIND_ADDRESS specifies a port (12345)" in caplog.text
+        assert "Ignoring the port" in caplog.text
+
     @coroutine_test
     async def test_unsupported_proxy(
         self, caplog: pytest.LogCaptureFixture, mockserver: MockServer

tests/test_downloader_handlers_http_base.py

@@ -671,6 +671,36 @@ class TestHttp11Base(TestHttpBase):
             response = await download_handler.download_request(request)
         assert response.protocol == "HTTP/1.1"
 
+    # skip macOS tests
+    @pytest.mark.skipif(
+        sys.platform == "darwin",
+        reason="127.0.0.2 is not available on macOS by default",
+    )
+    @coroutine_test
+    async def test_download_bind_address_setting(self, mockserver: MockServer) -> None:
+        request = Request(mockserver.url("/client-ip", is_secure=self.is_secure))
+        async with self.get_dh(
+            {"DOWNLOAD_BIND_ADDRESS": ("127.0.0.2", 0)}
+        ) as download_handler:
+            response = await download_handler.download_request(request)
+        assert response.body == b"127.0.0.2"
+
+    # skip macOS tests
+    @pytest.mark.skipif(
+        sys.platform == "darwin",
+        reason="127.0.0.2 is not available on macOS by default",
+    )
+    @coroutine_test
+    async def test_download_bind_address_setting_string(
+        self, mockserver: MockServer
+    ) -> None:
+        request = Request(mockserver.url("/client-ip", is_secure=self.is_secure))
+        async with self.get_dh(
+            {"DOWNLOAD_BIND_ADDRESS": "127.0.0.2"}
+        ) as download_handler:
+            response = await download_handler.download_request(request)
+        assert response.body == b"127.0.0.2"
+
 
 class TestHttps11Base(TestHttp11Base):
     is_secure = True