diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 75158440b..2176bc5ba 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -898,6 +898,7 @@ Those are: * :reqmeta:`is_start_request` * :reqmeta:`max_retry_times` * :reqmeta:`proxy` +* :reqmeta:`proxy_headers` * :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` * :reqmeta:`referrer_policy` @@ -1015,6 +1016,40 @@ The meta key is used set retry times per request. When set, the :reqmeta:`max_retry_times` meta key takes higher precedence over the :setting:`RETRY_TIMES` setting. +.. reqmeta:: proxy_headers + +proxy_headers +------------- + +.. versionadded:: VERSION + +Headers for the proxy set in the :reqmeta:`proxy` metadata key, as a mapping, +e.g. ``{"X-Proxy-Country": "us"}``. + +.. code-block:: python + + Request( + "https://example.com", + meta={ + "proxy": "http://proxy.example:8080", + "proxy_headers": {"X-Proxy-Country": "us"}, + }, + ) + +For HTTPS URLs these headers are sent in the ``CONNECT`` request that opens the +tunnel through the proxy, which the target server cannot see. For other URLs +there is no tunnel, so they are sent among the request headers, and it is up to +the proxy to consume them instead of passing them on to the target server. + +.. note:: + + Handling of this metadata key needs to be implemented inside the + :ref:`download handler `, so it's not guaranteed + to be supported by all 3rd-party handlers. Among the built-in handlers, + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` and + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` + support it for HTTP and HTTPS proxies. + .. reqmeta:: verbatim_url verbatim_url diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index 8bbffb233..661026b37 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -3,10 +3,11 @@ from __future__ import annotations import ipaddress +import logging import ssl from contextlib import asynccontextmanager from socket import gaierror -from typing import TYPE_CHECKING, ClassVar +from typing import TYPE_CHECKING, ClassVar, TypeAlias from scrapy.exceptions import ( CannotResolveHostError, @@ -17,7 +18,7 @@ from scrapy.exceptions import ( UnsupportedURLSchemeError, ) from scrapy.http import Headers -from scrapy.utils._download_handlers import NullCookieJar +from scrapy.utils._download_handlers import NullCookieJar, get_proxy_headers from scrapy.utils.python import _iter_exc_causes from scrapy.utils.ssl import ( _log_sslobj_debug_info, @@ -36,6 +37,11 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler +logger = logging.getLogger(__name__) + +# a proxy URL and the headers to send to that proxy +_ProxyKey: TypeAlias = tuple[str, tuple[tuple[str, str], ...]] + HAS_SOCKS = HAS_HTTP2 = False try: @@ -101,8 +107,9 @@ class HttpxDownloadHandler(_Base): self._default_client: httpx.AsyncClient = self._make_client() # httpx2 doesn't support per-request proxies: https://github.com/pydantic/httpx2/issues/818, - # so we keep a pool of clients per proxy URL. LRU eviction can be added here if needed. - self._proxy_clients: dict[str, httpx.AsyncClient] = {} + # so we keep a pool of clients per proxy URL and set of proxy headers. + # LRU eviction can be added here if needed. + self._proxy_clients: dict[_ProxyKey, httpx.AsyncClient] = {} @staticmethod def _check_deps_installed() -> None: @@ -111,13 +118,27 @@ class HttpxDownloadHandler(_Base): "HttpxDownloadHandler requires the httpx2 library to be installed." ) - def _make_client(self, proxy_url: str | None = None) -> httpx.AsyncClient: + def _make_client( + self, + proxy_url: str | None = None, + proxy_headers: tuple[tuple[str, str], ...] = (), + ) -> httpx.AsyncClient: if proxy_url: if proxy_url.startswith("https:") and not self._verify_certificates: proxy_ssl_context = _make_insecure_ssl_ctx() else: proxy_ssl_context = None - proxy = httpx.Proxy(proxy_url, ssl_context=proxy_ssl_context) + if proxy_headers and proxy_url.startswith("socks"): + logger.warning( + f"Ignoring the proxy_headers request metadata key for " + f"{proxy_url}, as the SOCKS protocol has no headers." + ) + proxy_headers = () + proxy = httpx.Proxy( + proxy_url, + ssl_context=proxy_ssl_context, + headers=list(proxy_headers), + ) else: proxy = None @@ -137,13 +158,18 @@ class HttpxDownloadHandler(_Base): client.headers.pop(header_name, None) return client - def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient: + def _get_client( + self, + proxy_url: str | None, + proxy_headers: tuple[tuple[str, str], ...] = (), + ) -> httpx.AsyncClient: if proxy_url is None: return self._default_client - if cached := self._proxy_clients.get(proxy_url): + key = (proxy_url, proxy_headers) + if cached := self._proxy_clients.get(key): return cached - client = self._make_client(proxy_url) - self._proxy_clients[proxy_url] = client + client = self._make_client(proxy_url, proxy_headers) + self._proxy_clients[key] = client return client @asynccontextmanager @@ -155,7 +181,7 @@ class HttpxDownloadHandler(_Base): raise ValueError( f"SOCKS proxy support in {type(self).__name__} requires the 'httpx2[socks]' extra to be installed." ) - client = self._get_client(proxy) + client = self._get_client(proxy, get_proxy_headers(request)) headers = self._request_headers(request).to_tuple_list() try: diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 17dca5acb..02fe23737 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -44,6 +44,7 @@ from scrapy.utils._download_handlers import ( check_stop_download, get_dataloss_msg, get_maxsize_msg, + get_proxy_headers, get_warnsize_msg, make_response, normalize_bind_address, @@ -181,12 +182,12 @@ class _TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): reactor: ReactorBase, host: str, port: int, - proxyConf: tuple[str, int, bytes | None], + proxyConf: tuple[str, int, tuple[tuple[str, str], ...]], contextFactory: IPolicyForHTTPS, timeout: float = 30, bindAddress: tuple[str, int] | None = None, ): - proxyHost, proxyPort, self._proxyAuthHeader = proxyConf + proxyHost, proxyPort, self._proxyHeaders = proxyConf super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) self._tunnelReadyDeferred: Deferred[Protocol] = Deferred() self._tunneledHost: str = host @@ -198,7 +199,7 @@ class _TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): """Asks the proxy to open a tunnel.""" assert protocol.transport tunnelReq = _tunnel_request_data( - self._tunneledHost, self._tunneledPort, self._proxyAuthHeader + self._tunneledHost, self._tunneledPort, self._proxyHeaders ) protocol.transport.write(tunnelReq) self._protocolDataReceived = protocol.dataReceived @@ -259,7 +260,7 @@ class _TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def _tunnel_request_data( - host: str, port: int, proxy_auth_header: bytes | None = None + host: str, port: int, proxy_headers: tuple[tuple[str, str], ...] = () ) -> bytes: r""" Return binary content of a CONNECT request. @@ -267,7 +268,7 @@ def _tunnel_request_data( >>> from scrapy.utils.python import to_unicode as s >>> s(_tunnel_request_data("example.com", 8080)) 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n' - >>> s(_tunnel_request_data("example.com", 8080, b"123")) + >>> s(_tunnel_request_data("example.com", 8080, (("Proxy-Authorization", "123"),))) 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n' >>> s(_tunnel_request_data(b"example.com", "8090")) 'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n' @@ -275,8 +276,8 @@ def _tunnel_request_data( host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port)) tunnel_req = b"CONNECT " + host_value + b" HTTP/1.1\r\n" tunnel_req += b"Host: " + host_value + b"\r\n" - if proxy_auth_header: - tunnel_req += b"Proxy-Authorization: " + proxy_auth_header + b"\r\n" + for name, value in proxy_headers: + tunnel_req += to_bytes(name) + b": " + to_bytes(value) + b"\r\n" tunnel_req += b"\r\n" return tunnel_req @@ -293,14 +294,14 @@ class _TunnelingAgent(Agent): self, *, reactor: ReactorBase, - proxyConf: tuple[str, int, bytes | None], + proxyConf: tuple[str, int, tuple[tuple[str, str], ...]], contextFactory: IPolicyForHTTPS, connectTimeout: float | None = None, bindAddress: tuple[str, int] | None = None, pool: HTTPConnectionPool | None = None, ): super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) # type: ignore[no-untyped-call] - self._proxyConf: tuple[str, int, bytes | None] = proxyConf + self._proxyConf: tuple[str, int, tuple[tuple[str, str], ...]] = proxyConf self._contextFactory: IPolicyForHTTPS = contextFactory def _getEndpoint(self, uri: URI) -> _TunnelingTCP4ClientEndpoint: @@ -324,9 +325,10 @@ class _TunnelingAgent(Agent): bodyProducer: IBodyProducer | None, requestPath: bytes, ) -> Deferred[IResponse]: - # proxy host and port are required for HTTP pool `key` - # otherwise, same remote host connection request could reuse - # a cached tunneled connection to a different proxy + # the proxy host and port and the headers sent to the proxy are part of + # the HTTP pool `key`, otherwise a request for the same remote host + # could reuse a tunnel that was opened through a different proxy or + # with different proxy headers key += self._proxyConf return super()._requestWithEndpoint( key=key, @@ -425,8 +427,16 @@ class _ScrapyAgent: "HTTPS proxies for HTTPS destinations are not supported" ) assert proxy_host is not None + proxy_headers = get_proxy_headers(request) proxyAuth = request.headers.get(b"Proxy-Authorization", None) - proxyConf = (proxy_host, proxy_port, proxyAuth) + if proxyAuth and not any( + name == "Proxy-Authorization" for name, _ in proxy_headers + ): + proxy_headers = ( + ("Proxy-Authorization", to_unicode(proxyAuth)), + *proxy_headers, + ) + proxyConf = (proxy_host, proxy_port, proxy_headers) return _TunnelingAgent( reactor=reactor, proxyConf=proxyConf, @@ -464,6 +474,12 @@ class _ScrapyAgent: headers = TxHeaders(request.headers) if isinstance(agent, _TunnelingAgent): headers.removeHeader(b"Proxy-Authorization") + elif isinstance(agent, _ScrapyProxyAgent): + # without a tunnel the proxy reads the request headers, so the + # headers meant for it travel among them, and it is up to the proxy + # not to pass them on to the target server + for name, value in get_proxy_headers(request): + headers.addRawHeader(name, value) bodyproducer = _RequestBodyProducer(request.body) if request.body else None start_time = monotonic() d: Deferred[IResponse] = agent.request( diff --git a/scrapy/utils/_download_handlers.py b/scrapy/utils/_download_handlers.py index 23b715ac6..cd25b76ff 100644 --- a/scrapy/utils/_download_handlers.py +++ b/scrapy/utils/_download_handlers.py @@ -24,6 +24,7 @@ from scrapy.exceptions import ( StopDownload, UnsupportedURLSchemeError, ) +from scrapy.http import Headers from scrapy.utils.log import logger if TYPE_CHECKING: @@ -35,7 +36,7 @@ if TYPE_CHECKING: from scrapy import Request from scrapy.crawler import Crawler - from scrapy.http import Headers, Response + from scrapy.http import Response class NullCookieJar(CookieJar): # pragma: no cover @@ -151,3 +152,16 @@ def normalize_bind_address( if isinstance(value, str): return (value, 0) return value + + +def get_proxy_headers(request: Request) -> tuple[tuple[str, str], ...]: + """Return the headers to send to the proxy, from the ``proxy_headers`` + request metadata key, as name-value pairs. + + Pairs are sorted so that handlers that key their connections on them treat + header sets that only differ in order as the same set. + """ + proxy_headers = request.meta.get("proxy_headers") + if not proxy_headers: + return () + return tuple(sorted(Headers(proxy_headers).to_tuple_list())) diff --git a/tests/mockserver/mitm_proxy_addon.py b/tests/mockserver/mitm_proxy_addon.py index 04ba2e4a1..fbeccdd48 100644 --- a/tests/mockserver/mitm_proxy_addon.py +++ b/tests/mockserver/mitm_proxy_addon.py @@ -1,5 +1,20 @@ +from typing import Any + +# headers of the CONNECT request of each client connection, so that headers +# sent to the proxy can also be checked for requests that go through a tunnel, +# which the proxy cannot read +connect_headers: dict[Any, Any] = {} + + +def http_connect(flow) -> None: + connect_headers[flow.client_conn.peername] = flow.request.headers.copy() + + def response(flow) -> None: # add custom headers to be able to check that the request went through the proxy flow.response.headers["X-Via-Mitmproxy"] = "1" if flow.client_conn.tls_established: flow.response.headers["X-Via-Mitmproxy-TLS"] = "1" + headers = connect_headers.get(flow.client_conn.peername, flow.request.headers) + if echo := headers.get("X-Proxy-Echo"): + flow.response.headers["X-Proxy-Echo"] = echo diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index 976daacaf..d1caf954f 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -165,6 +165,35 @@ class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase): pass +@coroutine_test +async def test_proxy_clients() -> None: + proxy = "http://proxy.example:8080" + handler = build_from_crawler(HttpxDownloadHandler, get_crawler()) + try: + client = handler._get_client(proxy, (("X-A", "1"),)) + assert handler._get_client(proxy, (("X-A", "1"),)) is client + assert handler._get_client(proxy, (("X-A", "2"),)) is not client + assert handler._get_client(proxy) is not client + assert len(handler._proxy_clients) == 3 + finally: + await handler.close() + + +@coroutine_test +async def test_proxy_headers_ignored_for_socks( + caplog: pytest.LogCaptureFixture, +) -> None: + if not HAS_SOCKS: # pragma: no cover + pytest.skip("SOCKS support is not installed") + handler = build_from_crawler(HttpxDownloadHandler, get_crawler()) + try: + with caplog.at_level("WARNING"): + handler._get_client("socks5://proxy.example:1080", (("X-A", "1"),)) + finally: + await handler.close() + assert "the SOCKS protocol has no headers" in caplog.text + + @pytest.mark.parametrize(("concurrency", "expected"), [(16, 16), (0, None)]) @coroutine_test async def test_pool_limits(concurrency: int, expected: int | None) -> None: diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index 32dfd7540..68c28c305 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -7,10 +7,16 @@ from typing import TYPE_CHECKING, Any import pytest -from scrapy import Spider -from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler +from scrapy import Request, Spider +from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings +from scrapy.core.downloader.handlers.http11 import ( + HTTP11DownloadHandler, + _ScrapyAgent, + _TunnelingAgent, +) from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.utils.test import get_crawler from tests.utils.bases.download_handlers_http import ( TestHttpBase, TestHttpProxyBase, @@ -54,6 +60,53 @@ def test_not_configured_without_reactor() -> None: HTTP11DownloadHandler.from_crawler(crawler) +@pytest.mark.parametrize( + ("proxy_headers", "proxy_auth", "expected"), + [ + (None, None, ()), + ({"X-B": "2", "X-A": "1"}, None, (("X-A", "1"), ("X-B", "2"))), + (None, "Basic Zm9v", (("Proxy-Authorization", "Basic Zm9v"),)), + ( + {"X-A": "1"}, + "Basic Zm9v", + (("Proxy-Authorization", "Basic Zm9v"), ("X-A", "1")), + ), + ( + {"Proxy-Authorization": "Custom foo"}, + "Basic Zm9v", + (("Proxy-Authorization", "Custom foo"),), + ), + ], + ids=[ + "no headers", + "sorted headers", + "auth header", + "auth header and headers", + "auth header overridden", + ], +) +def test_tunnel_proxy_headers( + proxy_headers: dict[str, str] | None, + proxy_auth: str | None, + expected: tuple[tuple[str, str], ...], +) -> None: + meta: dict[str, Any] = {"proxy": "http://proxy.example:8080"} + if proxy_headers is not None: + meta["proxy_headers"] = proxy_headers + request = Request( + "https://example.com", + meta=meta, + headers={"Proxy-Authorization": proxy_auth} if proxy_auth else None, + ) + crawler = get_crawler(Spider) + agent = _ScrapyAgent( + contextFactory=_load_context_factory_from_settings(crawler), crawler=crawler + ) + tunneling_agent = agent._get_agent(request, 10) + assert isinstance(tunneling_agent, _TunnelingAgent) + assert tunneling_agent._proxyConf == ("proxy.example", 8080, expected) + + class TestHttp(HTTP11DownloadHandlerMixin, TestHttpBase): pass diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py index e44f9bcb8..1153e7f0a 100644 --- a/tests/utils/bases/download_handlers_http.py +++ b/tests/utils/bases/download_handlers_http.py @@ -1460,6 +1460,38 @@ class TestMitmProxyBase(ABC): self._assert_got_response_code(200, caplog.text) self._assert_headers(responses[0].headers, https_dest) + @pytest.mark.parametrize("proxy_server", ["http", "https"], indirect=True) + @pytest.mark.parametrize( + "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] + ) + @coroutine_test + async def test_proxy_headers( + self, + proxy_server: str, + mockserver: MockServer, + https_dest: bool, + ) -> None: + """HTTP/HTTPS proxy, HTTP or HTTPS destination, headers for the proxy. + + The proxy echoes the X-Proxy-Echo header that it receives into the + response, and /echo reports the headers that the target server got. + """ + self._maybe_skip(proxy_server, https_dest) + request = Request( + mockserver.url("/echo", is_secure=https_dest), + meta={"proxy_headers": {"X-Proxy-Echo": "foo"}}, + ) + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + await crawler.crawl_async(seed=request) + assert isinstance(crawler.spider, SingleRequestSpider) + response = crawler.spider.meta["responses"][0] + assert response.headers.get(b"X-Proxy-Echo") == b"foo" + if https_dest: + # the tunnel keeps the headers for the proxy away from the target + # server; without one that is up to the proxy, and mitmproxy, which + # knows nothing about this header, does pass it on + assert "X-Proxy-Echo" not in json.loads(response.text)["headers"] + @staticmethod def _assert_headers(headers: Headers, https_dest: bool) -> None: assert b"X-Via-Mitmproxy" in headers