RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support sending proxy headers

This commit is contained in:
Adrian Chaves 2026-08-05 08:42:20 +02:00
parent 91b70e4db4
commit c69be51a89
8 changed files with 247 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
docs/topics/request-response.rst
View File

@ -898,6 +898,7 @@ Those are:
* :reqmeta:`is_start_request`
* :reqmeta:`max_retry_times`
* :reqmeta:`proxy`
* :reqmeta:`proxy_headers`
* :reqmeta:`redirect_reasons`
* :reqmeta:`redirect_urls`
* :reqmeta:`referrer_policy`
@ -1015,6 +1016,40 @@ The meta key is used set retry times per request. When set, the
:reqmeta:`max_retry_times` meta key takes higher precedence over the
:setting:`RETRY_TIMES` setting.
.. reqmeta:: proxy_headers
proxy_headers
-------------
.. versionadded:: VERSION
Headers for the proxy set in the :reqmeta:`proxy` metadata key, as a mapping,
e.g. ``{"X-Proxy-Country": "us"}``.
.. code-block:: python
Request(
"https://example.com",
meta={
"proxy": "http://proxy.example:8080",
"proxy_headers": {"X-Proxy-Country": "us"},
},
)
For HTTPS URLs these headers are sent in the ``CONNECT`` request that opens the
tunnel through the proxy, which the target server cannot see. For other URLs
there is no tunnel, so they are sent among the request headers, and it is up to
the proxy to consume them instead of passing them on to the target server.
.. note::
Handling of this metadata key needs to be implemented inside the
:ref:`download handler <topics-download-handlers>`, so it's not guaranteed
to be supported by all 3rd-party handlers. Among the built-in handlers,
:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` and
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`
support it for HTTP and HTTPS proxies.
.. reqmeta:: verbatim_url
verbatim_url

48
scrapy/core/downloader/handlers/_httpx.py
View File

@ -3,10 +3,11 @@
from __future__ import annotations
import ipaddress
import logging
import ssl
from contextlib import asynccontextmanager
from socket import gaierror
from typing import TYPE_CHECKING, ClassVar
from typing import TYPE_CHECKING, ClassVar, TypeAlias
from scrapy.exceptions import (
CannotResolveHostError,
@ -17,7 +18,7 @@ from scrapy.exceptions import (
UnsupportedURLSchemeError,
)
from scrapy.http import Headers
from scrapy.utils._download_handlers import NullCookieJar
from scrapy.utils._download_handlers import NullCookieJar, get_proxy_headers
from scrapy.utils.python import _iter_exc_causes
from scrapy.utils.ssl import (
_log_sslobj_debug_info,
@ -36,6 +37,11 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
logger = logging.getLogger(__name__)
# a proxy URL and the headers to send to that proxy
_ProxyKey: TypeAlias = tuple[str, tuple[tuple[str, str], ...]]
HAS_SOCKS = HAS_HTTP2 = False
try:
@ -101,8 +107,9 @@ class HttpxDownloadHandler(_Base):
self._default_client: httpx.AsyncClient = self._make_client()
# httpx2 doesn't support per-request proxies: https://github.com/pydantic/httpx2/issues/818,
# so we keep a pool of clients per proxy URL. LRU eviction can be added here if needed.
self._proxy_clients: dict[str, httpx.AsyncClient] = {}
# so we keep a pool of clients per proxy URL and set of proxy headers.
# LRU eviction can be added here if needed.
self._proxy_clients: dict[_ProxyKey, httpx.AsyncClient] = {}
@staticmethod
def _check_deps_installed() -> None:
@ -111,13 +118,27 @@ class HttpxDownloadHandler(_Base):
"HttpxDownloadHandler requires the httpx2 library to be installed."
)
def _make_client(self, proxy_url: str | None = None) -> httpx.AsyncClient:
def _make_client(
self,
proxy_url: str | None = None,
proxy_headers: tuple[tuple[str, str], ...] = (),
) -> httpx.AsyncClient:
if proxy_url:
if proxy_url.startswith("https:") and not self._verify_certificates:
proxy_ssl_context = _make_insecure_ssl_ctx()
else:
proxy_ssl_context = None
proxy = httpx.Proxy(proxy_url, ssl_context=proxy_ssl_context)
if proxy_headers and proxy_url.startswith("socks"):
logger.warning(
f"Ignoring the proxy_headers request metadata key for "
f"{proxy_url}, as the SOCKS protocol has no headers."
)
proxy_headers = ()
proxy = httpx.Proxy(
proxy_url,
ssl_context=proxy_ssl_context,
headers=list(proxy_headers),
)
else:
proxy = None
@ -137,13 +158,18 @@ class HttpxDownloadHandler(_Base):
client.headers.pop(header_name, None)
return client
def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient:
def _get_client(
self,
proxy_url: str | None,
proxy_headers: tuple[tuple[str, str], ...] = (),
) -> httpx.AsyncClient:
if proxy_url is None:
return self._default_client
if cached := self._proxy_clients.get(proxy_url):
key = (proxy_url, proxy_headers)
if cached := self._proxy_clients.get(key):
return cached
client = self._make_client(proxy_url)
self._proxy_clients[proxy_url] = client
client = self._make_client(proxy_url, proxy_headers)
self._proxy_clients[key] = client
return client
@asynccontextmanager
@ -155,7 +181,7 @@ class HttpxDownloadHandler(_Base):
raise ValueError(
f"SOCKS proxy support in {type(self).__name__} requires the 'httpx2[socks]' extra to be installed."
)
client = self._get_client(proxy)
client = self._get_client(proxy, get_proxy_headers(request))
headers = self._request_headers(request).to_tuple_list()
try:

42
scrapy/core/downloader/handlers/http11.py
View File

@ -44,6 +44,7 @@ from scrapy.utils._download_handlers import (
check_stop_download,
get_dataloss_msg,
get_maxsize_msg,
get_proxy_headers,
get_warnsize_msg,
make_response,
normalize_bind_address,
@ -181,12 +182,12 @@ class _TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
reactor: ReactorBase,
host: str,
port: int,
proxyConf: tuple[str, int, bytes | None],
proxyConf: tuple[str, int, tuple[tuple[str, str], ...]],
contextFactory: IPolicyForHTTPS,
timeout: float = 30,
bindAddress: tuple[str, int] | None = None,
):
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
proxyHost, proxyPort, self._proxyHeaders = proxyConf
super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
self._tunnelReadyDeferred: Deferred[Protocol] = Deferred()
self._tunneledHost: str = host
@ -198,7 +199,7 @@ class _TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
"""Asks the proxy to open a tunnel."""
assert protocol.transport
tunnelReq = _tunnel_request_data(
self._tunneledHost, self._tunneledPort, self._proxyAuthHeader
self._tunneledHost, self._tunneledPort, self._proxyHeaders
)
protocol.transport.write(tunnelReq)
self._protocolDataReceived = protocol.dataReceived
@ -259,7 +260,7 @@ class _TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def _tunnel_request_data(
host: str, port: int, proxy_auth_header: bytes | None = None
host: str, port: int, proxy_headers: tuple[tuple[str, str], ...] = ()
) -> bytes:
r"""
Return binary content of a CONNECT request.
@ -267,7 +268,7 @@ def _tunnel_request_data(
>>> from scrapy.utils.python import to_unicode as s
>>> s(_tunnel_request_data("example.com", 8080))
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
>>> s(_tunnel_request_data("example.com", 8080, b"123"))
>>> s(_tunnel_request_data("example.com", 8080, (("Proxy-Authorization", "123"),)))
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n'
>>> s(_tunnel_request_data(b"example.com", "8090"))
'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n'
@ -275,8 +276,8 @@ def _tunnel_request_data(
host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port))
tunnel_req = b"CONNECT " + host_value + b" HTTP/1.1\r\n"
tunnel_req += b"Host: " + host_value + b"\r\n"
if proxy_auth_header:
tunnel_req += b"Proxy-Authorization: " + proxy_auth_header + b"\r\n"
for name, value in proxy_headers:
tunnel_req += to_bytes(name) + b": " + to_bytes(value) + b"\r\n"
tunnel_req += b"\r\n"
return tunnel_req
@ -293,14 +294,14 @@ class _TunnelingAgent(Agent):
self,
*,
reactor: ReactorBase,
proxyConf: tuple[str, int, bytes | None],
proxyConf: tuple[str, int, tuple[tuple[str, str], ...]],
contextFactory: IPolicyForHTTPS,
connectTimeout: float | None = None,
bindAddress: tuple[str, int] | None = None,
pool: HTTPConnectionPool | None = None,
):
super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) # type: ignore[no-untyped-call]
self._proxyConf: tuple[str, int, bytes | None] = proxyConf
self._proxyConf: tuple[str, int, tuple[tuple[str, str], ...]] = proxyConf
self._contextFactory: IPolicyForHTTPS = contextFactory
def _getEndpoint(self, uri: URI) -> _TunnelingTCP4ClientEndpoint:
@ -324,9 +325,10 @@ class _TunnelingAgent(Agent):
bodyProducer: IBodyProducer | None,
requestPath: bytes,
) -> Deferred[IResponse]:
# proxy host and port are required for HTTP pool `key`
# otherwise, same remote host connection request could reuse
# a cached tunneled connection to a different proxy
# the proxy host and port and the headers sent to the proxy are part of
# the HTTP pool `key`, otherwise a request for the same remote host
# could reuse a tunnel that was opened through a different proxy or
# with different proxy headers
key += self._proxyConf
return super()._requestWithEndpoint(
key=key,
@ -425,8 +427,16 @@ class _ScrapyAgent:
"HTTPS proxies for HTTPS destinations are not supported"
)
assert proxy_host is not None
proxy_headers = get_proxy_headers(request)
proxyAuth = request.headers.get(b"Proxy-Authorization", None)
proxyConf = (proxy_host, proxy_port, proxyAuth)
if proxyAuth and not any(
name == "Proxy-Authorization" for name, _ in proxy_headers
):
proxy_headers = (
("Proxy-Authorization", to_unicode(proxyAuth)),
*proxy_headers,
)
proxyConf = (proxy_host, proxy_port, proxy_headers)
return _TunnelingAgent(
reactor=reactor,
proxyConf=proxyConf,
@ -464,6 +474,12 @@ class _ScrapyAgent:
headers = TxHeaders(request.headers)
if isinstance(agent, _TunnelingAgent):
headers.removeHeader(b"Proxy-Authorization")
elif isinstance(agent, _ScrapyProxyAgent):
# without a tunnel the proxy reads the request headers, so the
# headers meant for it travel among them, and it is up to the proxy
# not to pass them on to the target server
for name, value in get_proxy_headers(request):
headers.addRawHeader(name, value)
bodyproducer = _RequestBodyProducer(request.body) if request.body else None
start_time = monotonic()
d: Deferred[IResponse] = agent.request(

16
scrapy/utils/_download_handlers.py
View File

@ -24,6 +24,7 @@ from scrapy.exceptions import (
StopDownload,
UnsupportedURLSchemeError,
)
from scrapy.http import Headers
from scrapy.utils.log import logger
if TYPE_CHECKING:
@ -35,7 +36,7 @@ if TYPE_CHECKING:
from scrapy import Request
from scrapy.crawler import Crawler
from scrapy.http import Headers, Response
from scrapy.http import Response
class NullCookieJar(CookieJar): # pragma: no cover
@ -151,3 +152,16 @@ def normalize_bind_address(
if isinstance(value, str):
return (value, 0)
return value
def get_proxy_headers(request: Request) -> tuple[tuple[str, str], ...]:
"""Return the headers to send to the proxy, from the ``proxy_headers``
request metadata key, as name-value pairs.
Pairs are sorted so that handlers that key their connections on them treat
header sets that only differ in order as the same set.
"""
proxy_headers = request.meta.get("proxy_headers")
if not proxy_headers:
return ()
return tuple(sorted(Headers(proxy_headers).to_tuple_list()))

15
tests/mockserver/mitm_proxy_addon.py
View File

@ -1,5 +1,20 @@
from typing import Any
# headers of the CONNECT request of each client connection, so that headers
# sent to the proxy can also be checked for requests that go through a tunnel,
# which the proxy cannot read
connect_headers: dict[Any, Any] = {}
def http_connect(flow) -> None:
connect_headers[flow.client_conn.peername] = flow.request.headers.copy()
def response(flow) -> None:
# add custom headers to be able to check that the request went through the proxy
flow.response.headers["X-Via-Mitmproxy"] = "1"
if flow.client_conn.tls_established:
flow.response.headers["X-Via-Mitmproxy-TLS"] = "1"
headers = connect_headers.get(flow.client_conn.peername, flow.request.headers)
if echo := headers.get("X-Proxy-Echo"):
flow.response.headers["X-Proxy-Echo"] = echo

29
tests/test_downloader_handler_httpx.py
View File

@ -165,6 +165,35 @@ class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase):
pass
@coroutine_test
async def test_proxy_clients() -> None:
proxy = "http://proxy.example:8080"
handler = build_from_crawler(HttpxDownloadHandler, get_crawler())
try:
client = handler._get_client(proxy, (("X-A", "1"),))
assert handler._get_client(proxy, (("X-A", "1"),)) is client
assert handler._get_client(proxy, (("X-A", "2"),)) is not client
assert handler._get_client(proxy) is not client
assert len(handler._proxy_clients) == 3
finally:
await handler.close()
@coroutine_test
async def test_proxy_headers_ignored_for_socks(
caplog: pytest.LogCaptureFixture,
) -> None:
if not HAS_SOCKS: # pragma: no cover
pytest.skip("SOCKS support is not installed")
handler = build_from_crawler(HttpxDownloadHandler, get_crawler())
try:
with caplog.at_level("WARNING"):
handler._get_client("socks5://proxy.example:1080", (("X-A", "1"),))
finally:
await handler.close()
assert "the SOCKS protocol has no headers" in caplog.text
@pytest.mark.parametrize(("concurrency", "expected"), [(16, 16), (0, None)])
@coroutine_test
async def test_pool_limits(concurrency: int, expected: int | None) -> None:

57
tests/test_downloader_handler_twisted_http11.py
View File

@ -7,10 +7,16 @@ from typing import TYPE_CHECKING, Any
import pytest
from scrapy import Spider
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
from scrapy import Request, Spider
from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings
from scrapy.core.downloader.handlers.http11 import (
HTTP11DownloadHandler,
_ScrapyAgent,
_TunnelingAgent,
)
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.utils.test import get_crawler
from tests.utils.bases.download_handlers_http import (
TestHttpBase,
TestHttpProxyBase,
@ -54,6 +60,53 @@ def test_not_configured_without_reactor() -> None:
HTTP11DownloadHandler.from_crawler(crawler)
@pytest.mark.parametrize(
("proxy_headers", "proxy_auth", "expected"),
[
(None, None, ()),
({"X-B": "2", "X-A": "1"}, None, (("X-A", "1"), ("X-B", "2"))),
(None, "Basic Zm9v", (("Proxy-Authorization", "Basic Zm9v"),)),
(
{"X-A": "1"},
"Basic Zm9v",
(("Proxy-Authorization", "Basic Zm9v"), ("X-A", "1")),
),
(
{"Proxy-Authorization": "Custom foo"},
"Basic Zm9v",
(("Proxy-Authorization", "Custom foo"),),
),
],
ids=[
"no headers",
"sorted headers",
"auth header",
"auth header and headers",
"auth header overridden",
],
)
def test_tunnel_proxy_headers(
proxy_headers: dict[str, str] | None,
proxy_auth: str | None,
expected: tuple[tuple[str, str], ...],
) -> None:
meta: dict[str, Any] = {"proxy": "http://proxy.example:8080"}
if proxy_headers is not None:
meta["proxy_headers"] = proxy_headers
request = Request(
"https://example.com",
meta=meta,
headers={"Proxy-Authorization": proxy_auth} if proxy_auth else None,
)
crawler = get_crawler(Spider)
agent = _ScrapyAgent(
contextFactory=_load_context_factory_from_settings(crawler), crawler=crawler
)
tunneling_agent = agent._get_agent(request, 10)
assert isinstance(tunneling_agent, _TunnelingAgent)
assert tunneling_agent._proxyConf == ("proxy.example", 8080, expected)
class TestHttp(HTTP11DownloadHandlerMixin, TestHttpBase):
pass

32
tests/utils/bases/download_handlers_http.py
View File

@ -1460,6 +1460,38 @@ class TestMitmProxyBase(ABC):
self._assert_got_response_code(200, caplog.text)
self._assert_headers(responses[0].headers, https_dest)
@pytest.mark.parametrize("proxy_server", ["http", "https"], indirect=True)
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@coroutine_test
async def test_proxy_headers(
self,
proxy_server: str,
mockserver: MockServer,
https_dest: bool,
) -> None:
"""HTTP/HTTPS proxy, HTTP or HTTPS destination, headers for the proxy.
The proxy echoes the X-Proxy-Echo header that it receives into the
response, and /echo reports the headers that the target server got.
"""
self._maybe_skip(proxy_server, https_dest)
request = Request(
mockserver.url("/echo", is_secure=https_dest),
meta={"proxy_headers": {"X-Proxy-Echo": "foo"}},
)
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
await crawler.crawl_async(seed=request)
assert isinstance(crawler.spider, SingleRequestSpider)
response = crawler.spider.meta["responses"][0]
assert response.headers.get(b"X-Proxy-Echo") == b"foo"
if https_dest:
# the tunnel keeps the headers for the proxy away from the target
# server; without one that is up to the proxy, and mitmproxy, which
# knows nothing about this header, does pass it on
assert "X-Proxy-Echo" not in json.loads(response.text)["headers"]
@staticmethod
def _assert_headers(headers: Headers, https_dest: bool) -> None:
assert b"X-Via-Mitmproxy" in headers