From 2d007bc4508423bbd57ee9017da22c16bbd576e1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 13 May 2026 01:03:14 +0500 Subject: [PATCH] Make more of the internal handler helpers private. (#7510) --- docs/news.rst | 22 ++++++++++++--- scrapy/core/downloader/handlers/http11.py | 33 +++++++++++------------ scrapy/core/downloader/handlers/http2.py | 8 +++--- 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 7f3952eaf..5eef10d84 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -9,10 +9,24 @@ Scrapy VERSION (unreleased) Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent`` has been made - private as it's an implementation detail of - :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. - (:issue:`7496`) +- The following classes and functions, intended for internal use by + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + and :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`, have + been made private: + + - ``scrapy.core.downloader.handlers.http11.ScrapyAgent`` + + - ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent`` + + - ``scrapy.core.downloader.handlers.http11.TunnelingAgent`` + + - ``scrapy.core.downloader.handlers.http11.TunnelingTCP4ClientEndpoint`` + + - ``scrapy.core.downloader.handlers.http11.tunnel_request_data()`` + + - ``scrapy.core.downloader.handlers.http2.ScrapyH2Agent`` + + (:issue:`7496`, #TBD) .. _release-2.15.2: diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index e6451cdd7..fbe24e52e 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -111,7 +111,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): "download_warnsize", "DOWNLOAD_WARNSIZE" ) - agent = ScrapyAgent( + agent = _ScrapyAgent( contextFactory=self._contextFactory, bindAddress=self._bind_address, pool=self._pool, @@ -161,7 +161,7 @@ class TunnelError(Exception): """An HTTP CONNECT tunnel could not be established by the proxy.""" -class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): +class _TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): """An endpoint that tunnels through proxies to allow HTTPS downloads. To accomplish that, this endpoint sends an HTTP CONNECT to the proxy. The HTTP CONNECT is always sent when using this endpoint, I think this could @@ -197,7 +197,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def requestTunnel(self, protocol: Protocol) -> Protocol: """Asks the proxy to open a tunnel.""" assert protocol.transport - tunnelReq = tunnel_request_data( + tunnelReq = _tunnel_request_data( self._tunneledHost, self._tunneledPort, self._proxyAuthHeader ) protocol.transport.write(tunnelReq) @@ -221,7 +221,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): if b"\r\n\r\n" not in self._connectBuffer: return self._protocol.dataReceived = self._protocolDataReceived # type: ignore[method-assign] - respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer) + respm = _TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer) if respm and int(respm.group("status")) == 200: # set proper Server Name Indication extension sslOptions = self._contextFactory.creatorForNetloc( # type: ignore[call-arg,misc] @@ -258,18 +258,18 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): return self._tunnelReadyDeferred -def tunnel_request_data( +def _tunnel_request_data( host: str, port: int, proxy_auth_header: bytes | None = None ) -> bytes: r""" Return binary content of a CONNECT request. >>> from scrapy.utils.python import to_unicode as s - >>> s(tunnel_request_data("example.com", 8080)) + >>> s(_tunnel_request_data("example.com", 8080)) 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n' - >>> s(tunnel_request_data("example.com", 8080, b"123")) + >>> s(_tunnel_request_data("example.com", 8080, b"123")) 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n' - >>> s(tunnel_request_data(b"example.com", "8090")) + >>> s(_tunnel_request_data(b"example.com", "8090")) 'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n' """ host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port)) @@ -281,7 +281,7 @@ def tunnel_request_data( return tunnel_req -class TunnelingAgent(Agent): +class _TunnelingAgent(Agent): """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS downloads. It may look strange that we have chosen to subclass Agent and not ProxyAgent but consider that after the tunnel is opened the proxy is @@ -303,8 +303,8 @@ class TunnelingAgent(Agent): self._proxyConf: tuple[str, int, bytes | None] = proxyConf self._contextFactory: IPolicyForHTTPS = contextFactory - def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint: - return TunnelingTCP4ClientEndpoint( + def _getEndpoint(self, uri: URI) -> _TunnelingTCP4ClientEndpoint: + return _TunnelingTCP4ClientEndpoint( reactor=self._reactor, host=uri.host, port=uri.port, @@ -381,10 +381,7 @@ class _ScrapyProxyAgent(Agent): ) -class ScrapyAgent: - _Agent = Agent - _TunnelingAgent = TunnelingAgent - +class _ScrapyAgent: def __init__( self, *, @@ -430,7 +427,7 @@ class ScrapyAgent: assert proxy_host is not None proxyAuth = request.headers.get(b"Proxy-Authorization", None) proxyConf = (proxy_host, proxy_port, proxyAuth) - return self._TunnelingAgent( + return _TunnelingAgent( reactor=reactor, proxyConf=proxyConf, contextFactory=self._contextFactory, @@ -447,7 +444,7 @@ class ScrapyAgent: pool=self._pool, ) - return self._Agent( # type: ignore[no-untyped-call] + return Agent( reactor=reactor, contextFactory=self._contextFactory, connectTimeout=timeout, @@ -465,7 +462,7 @@ class ScrapyAgent: url = urldefrag(request.url)[0] method = to_bytes(request.method) headers = TxHeaders(request.headers) - if isinstance(agent, self._TunnelingAgent): + if isinstance(agent, _TunnelingAgent): headers.removeHeader(b"Proxy-Authorization") bodyproducer = _RequestBodyProducer(request.body) if request.body else None start_time = monotonic() diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index b28bcfd89..f60c58d1b 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -49,7 +49,7 @@ class H2DownloadHandler(BaseDownloadHandler): raise UnsupportedURLSchemeError( f"{type(self).__name__} doesn't support plain HTTP." ) - agent = ScrapyH2Agent( + agent = _ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, bind_address=self._bind_address, @@ -65,9 +65,7 @@ class H2DownloadHandler(BaseDownloadHandler): self._pool.close_connections() -class ScrapyH2Agent: - _Agent = H2Agent - +class _ScrapyH2Agent: def __init__( self, context_factory: IPolicyForHTTPS, @@ -89,7 +87,7 @@ class ScrapyH2Agent: raise NotImplementedError(f"{type(self).__name__} doesn't support proxies.") bind_address = request.meta.get("bindaddress") or self._bind_address bind_address = normalize_bind_address(bind_address) - return self._Agent( + return H2Agent( reactor=reactor, context_factory=self._context_factory, connect_timeout=timeout,