diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index cb2410207..5acabb0de 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -644,7 +644,36 @@ Those are: bindaddress ----------- -The IP of the outgoing IP address to use for the performing the request. +The default local outgoing address for download-handler connections. + +This setting can be either: + +- a host address as a string (e.g. ``"127.0.0.2"``), in which case the local + port is chosen automatically, or + +- a ``(host, port)`` tuple (e.g. ``("127.0.0.2", 50000)``) to bind to both a + specific local interface and a specific local port. + +For example: + +.. code-block:: python + + Request( + "https://example.org", + meta={"bindaddress": "127.0.0.2"}, + ) + +.. code-block:: python + + Request( + "https://example.org", + meta={"bindaddress": ("127.0.0.2", 50000)}, + ) + +If not set, built-in HTTP download handlers use the value of +:setting:`DOWNLOAD_BIND_ADDRESS` as the default bind address. +Set the :reqmeta:`bindaddress` request meta key to override it for a +specific request. .. reqmeta:: download_timeout diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5b4a9b5f9..2be319ed2 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -897,6 +897,39 @@ It is also possible to change this setting per domain, although it requires non-trivial code. See the implementation of the :ref:`AutoThrottle ` extension for an example. +.. setting:: DOWNLOAD_BIND_ADDRESS + +DOWNLOAD_BIND_ADDRESS +--------------------- + +Default: ``None`` + +The default local outgoing address for download-handler connections. + +This setting can be either: + +- a host address as a string (e.g. ``"127.0.0.2"``), in which case the local + port is chosen automatically, or + +- a ``(host, port)`` tuple (e.g. ``("127.0.0.2", 50000)``) to bind to both a + specific local interface and a specific local port. + +For example: + +.. code-block:: python + + # Bind to this local address + DOWNLOAD_BIND_ADDRESS = "127.0.0.2" + +.. code-block:: python + + # Bind to this local address and local port + DOWNLOAD_BIND_ADDRESS = ("127.0.0.2", 5000) + +If set, built-in HTTP download handlers use this value by default. +Set the :reqmeta:`bindaddress` request meta key to override it for a specific +request. + .. setting:: DOWNLOAD_HANDLERS DOWNLOAD_HANDLERS diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index 4a9617506..596e44f4c 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -30,6 +30,7 @@ from scrapy.utils._download_handlers import ( get_maxsize_msg, get_warnsize_msg, make_response, + normalize_bind_address, ) from scrapy.utils.asyncio import is_asyncio_available from scrapy.utils.ssl import _log_sslobj_debug_info, _make_ssl_context @@ -87,8 +88,30 @@ class HttpxDownloadHandler(BaseHttpDownloadHandler): self._tls_verbose_logging: bool = self.crawler.settings.getbool( "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" ) + bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS") + bind_address = normalize_bind_address(bind_address) + + self._bind_address: str | None = None + + if bind_address is not None: + host, port = bind_address + if port != 0: + logger.warning( + "DOWNLOAD_BIND_ADDRESS specifies a port (%s), but %s does not " + "support binding to a specific local port. Ignoring the port " + "and binding only to %r.", + port, + type(self).__name__, + host, + ) + self._bind_address = host + self._client = httpx.AsyncClient( - verify=_make_ssl_context(crawler.settings), cookies=_NullCookieJar() + cookies=_NullCookieJar(), + transport=httpx.AsyncHTTPTransport( + verify=_make_ssl_context(crawler.settings), + local_address=self._bind_address, + ), ) async def download_request(self, request: Request) -> Response: @@ -108,7 +131,11 @@ class HttpxDownloadHandler(BaseHttpDownloadHandler): except httpx.UnsupportedProtocol as e: raise UnsupportedURLSchemeError(str(e)) from e except httpx.ConnectError as e: - if "Name or service not known" in str(e) or "getaddrinfo failed" in str(e): + if ( + "Name or service not known" in str(e) + or "getaddrinfo failed" in str(e) + or "nodename nor servname" in str(e) + ): raise CannotResolveHostError(str(e)) from e raise DownloadConnectionRefusedError(str(e)) from e except httpx.NetworkError as e: diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 2fd3bd988..089400ebd 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -45,6 +45,7 @@ from scrapy.utils._download_handlers import ( get_maxsize_msg, get_warnsize_msg, make_response, + normalize_bind_address, wrap_twisted_exceptions, ) from scrapy.utils.defer import maybe_deferred_to_future @@ -93,6 +94,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): self._contextFactory: IPolicyForHTTPS = load_context_factory_from_settings( crawler.settings, crawler ) + self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS") self._disconnect_timeout: int = 1 async def download_request(self, request: Request) -> Response: @@ -106,6 +108,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): agent = ScrapyAgent( contextFactory=self._contextFactory, + bindAddress=self._bind_address, pool=self._pool, maxsize=getattr( self._crawler.spider, "download_maxsize", self._default_maxsize @@ -286,7 +289,7 @@ class TunnelingAgent(Agent): proxyConf: tuple[str, int, bytes | None], contextFactory: IPolicyForHTTPS, connectTimeout: float | None = None, - bindAddress: bytes | None = None, + bindAddress: tuple[str, int] | None = None, pool: HTTPConnectionPool | None = None, ): super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) @@ -335,7 +338,7 @@ class ScrapyProxyAgent(Agent): reactor: ReactorBase, proxyURI: bytes, connectTimeout: float | None = None, - bindAddress: bytes | None = None, + bindAddress: tuple[str, int] | None = None, pool: HTTPConnectionPool | None = None, ): super().__init__( @@ -379,7 +382,7 @@ class ScrapyAgent: *, contextFactory: IPolicyForHTTPS, connectTimeout: float = 10, - bindAddress: bytes | None = None, + bindAddress: str | tuple[str, int] | None = None, pool: HTTPConnectionPool | None = None, maxsize: int = 0, warnsize: int = 0, @@ -388,7 +391,7 @@ class ScrapyAgent: ): self._contextFactory: IPolicyForHTTPS = contextFactory self._connectTimeout: float = connectTimeout - self._bindAddress: bytes | None = bindAddress + self._bindAddress: str | tuple[str, int] | None = bindAddress self._pool: HTTPConnectionPool | None = pool self._maxsize: int = maxsize self._warnsize: int = warnsize @@ -400,6 +403,7 @@ class ScrapyAgent: from twisted.internet import reactor bindaddress = request.meta.get("bindaddress") or self._bindAddress + bindaddress = normalize_bind_address(bindaddress) proxy = request.meta.get("proxy") if proxy: proxy = add_http_if_no_scheme(proxy) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 75fa0772d..c749a4a73 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -10,7 +10,10 @@ from scrapy.core.downloader.contextfactory import load_context_factory_from_sett from scrapy.core.downloader.handlers.base import BaseDownloadHandler from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent from scrapy.exceptions import DownloadTimeoutError -from scrapy.utils._download_handlers import wrap_twisted_exceptions +from scrapy.utils._download_handlers import ( + normalize_bind_address, + wrap_twisted_exceptions, +) from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes @@ -38,11 +41,13 @@ class H2DownloadHandler(BaseDownloadHandler): self._context_factory = load_context_factory_from_settings( crawler.settings, crawler ) + self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS") async def download_request(self, request: Request) -> Response: agent = ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, + bind_address=self._bind_address, crawler=self._crawler, ) assert self._crawler.spider @@ -64,7 +69,7 @@ class ScrapyH2Agent: context_factory: IPolicyForHTTPS, pool: H2ConnectionPool, connect_timeout: int = 10, - bind_address: bytes | None = None, + bind_address: str | tuple[str, int] | None = None, crawler: Crawler | None = None, ) -> None: self._context_factory = context_factory @@ -77,6 +82,7 @@ class ScrapyH2Agent: from twisted.internet import reactor bind_address = request.meta.get("bindaddress") or self._bind_address + bind_address = normalize_bind_address(bind_address) proxy = request.meta.get("proxy") if proxy: if urlparse_cached(request).scheme == "https": diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 8760b13ec..22019d661 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -122,7 +122,7 @@ class H2Agent: pool: H2ConnectionPool, context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), # noqa: B008 connect_timeout: float | None = None, - bind_address: bytes | None = None, + bind_address: tuple[str, int] | None = None, ) -> None: self._reactor = reactor self._pool = pool @@ -166,7 +166,7 @@ class ScrapyProxyH2Agent(H2Agent): pool: H2ConnectionPool, context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), # noqa: B008 connect_timeout: float | None = None, - bind_address: bytes | None = None, + bind_address: tuple[str, int] | None = None, ) -> None: super().__init__( reactor=reactor, diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 385d5f933..bc04ebcfb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -59,6 +59,7 @@ __all__ = [ "DOWNLOADER_MIDDLEWARES", "DOWNLOADER_MIDDLEWARES_BASE", "DOWNLOADER_STATS", + "DOWNLOAD_BIND_ADDRESS", "DOWNLOAD_DELAY", "DOWNLOAD_FAIL_ON_DATALOSS", "DOWNLOAD_HANDLERS", @@ -245,6 +246,8 @@ DNS_TIMEOUT = 60 DOWNLOAD_DELAY = 0 +DOWNLOAD_BIND_ADDRESS = None + DOWNLOAD_FAIL_ON_DATALOSS = True DOWNLOAD_HANDLERS = {} diff --git a/scrapy/utils/_download_handlers.py b/scrapy/utils/_download_handlers.py index 8b0f0fa53..94063a831 100644 --- a/scrapy/utils/_download_handlers.py +++ b/scrapy/utils/_download_handlers.py @@ -146,3 +146,11 @@ def get_dataloss_msg(url: str) -> str: f"responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" f" -- This message won't be shown in further requests" ) + + +def normalize_bind_address( + value: str | tuple[str, int] | None, +) -> tuple[str, int] | None: + if isinstance(value, str): + return (value, 0) + return value diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py index d7c892828..6cf0046c0 100644 --- a/tests/mockserver/http.py +++ b/tests/mockserver/http.py @@ -14,6 +14,7 @@ from .http_resources import ( BrokenChunkedResource, BrokenDownloadResource, ChunkedResource, + ClientIPResource, Compress, ContentLengthHeaderResource, Delay, @@ -72,6 +73,7 @@ class Root(resource.Resource): self.putChild(b"wait", ForeverTakingResource()) self.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) self.putChild(b"host", HostHeaderResource()) + self.putChild(b"client-ip", ClientIPResource()) self.putChild(b"broken", BrokenDownloadResource()) self.putChild(b"chunked", ChunkedResource()) self.putChild(b"broken-chunked", BrokenChunkedResource()) diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.py index 91577aa6b..30f7861e7 100644 --- a/tests/mockserver/http_resources.py +++ b/tests/mockserver/http_resources.py @@ -56,6 +56,18 @@ class HostHeaderResource(resource.Resource): return request.requestHeaders.getRawHeaders(b"host")[0] +class ClientIPResource(resource.Resource): + """ + A testing resource which renders itself as the request client IP address. + """ + + def render(self, request): + client_address = request.getClientAddress() + if client_address is None or client_address.host is None: + return b"" + return to_bytes(client_address.host) + + class PayloadResource(resource.Resource): """ A testing resource which renders itself as the contents of the request body diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index ad6c6d3c7..233bc62fe 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -2,6 +2,7 @@ from __future__ import annotations +import sys from typing import TYPE_CHECKING, Any import pytest @@ -44,7 +45,7 @@ class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base): async def test_unsupported_bindaddress( self, caplog: pytest.LogCaptureFixture, mockserver: MockServer ) -> None: - meta = {"bindaddress": "127.0.0.2"} + meta = {"bindaddress": ("127.0.0.2", 0)} request = Request(mockserver.url("/text"), meta=meta) async with self.get_dh() as download_handler: response = await download_handler.download_request(request) @@ -54,6 +55,24 @@ class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base): in caplog.text ) + # skip macOS tests + @pytest.mark.skipif( + sys.platform == "darwin", + reason="127.0.0.2 is not available on macOS by default", + ) + @coroutine_test + async def test_bind_address_port_warning( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + request = Request(mockserver.url("/client-ip")) + async with self.get_dh( + {"DOWNLOAD_BIND_ADDRESS": ("127.0.0.2", 12345)} + ) as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"127.0.0.2" + assert "DOWNLOAD_BIND_ADDRESS specifies a port (12345)" in caplog.text + assert "Ignoring the port" in caplog.text + @coroutine_test async def test_unsupported_proxy( self, caplog: pytest.LogCaptureFixture, mockserver: MockServer diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 8e9fa0ae1..88813f0b6 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -671,6 +671,36 @@ class TestHttp11Base(TestHttpBase): response = await download_handler.download_request(request) assert response.protocol == "HTTP/1.1" + # skip macOS tests + @pytest.mark.skipif( + sys.platform == "darwin", + reason="127.0.0.2 is not available on macOS by default", + ) + @coroutine_test + async def test_download_bind_address_setting(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/client-ip", is_secure=self.is_secure)) + async with self.get_dh( + {"DOWNLOAD_BIND_ADDRESS": ("127.0.0.2", 0)} + ) as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"127.0.0.2" + + # skip macOS tests + @pytest.mark.skipif( + sys.platform == "darwin", + reason="127.0.0.2 is not available on macOS by default", + ) + @coroutine_test + async def test_download_bind_address_setting_string( + self, mockserver: MockServer + ) -> None: + request = Request(mockserver.url("/client-ip", is_secure=self.is_secure)) + async with self.get_dh( + {"DOWNLOAD_BIND_ADDRESS": "127.0.0.2"} + ) as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"127.0.0.2" + class TestHttps11Base(TestHttp11Base): is_secure = True