From 510f09a961473da15b3db4f67bf9200fd1a0927c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 2 Apr 2026 12:29:47 +0500 Subject: [PATCH] Deprecate ScrapyClientContextFactory. (#7391) --- scrapy/core/downloader/contextfactory.py | 42 +++++++---------------- scrapy/core/downloader/handlers/http10.py | 6 ++-- scrapy/core/downloader/tls.py | 2 +- tests/test_core_downloader.py | 26 ++++---------- tests/test_webclient.py | 10 ++++-- 5 files changed, 30 insertions(+), 56 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index a01f021a6..a0e0dd933 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -21,7 +21,7 @@ from scrapy.core.downloader.tls import ( openssl_methods, ) from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.deprecate import create_deprecated_class, method_is_overridden +from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: @@ -35,7 +35,7 @@ if TYPE_CHECKING: @implementer(IPolicyForHTTPS) -class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): +class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): """Non-peer-certificate verifying HTTPS context factory. Default OpenSSL method is ``TLS_METHOD`` (also called ``SSLv23_METHOD``) @@ -70,22 +70,6 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): ) self._ctx = self._get_context() self._verify_certificates = verify_certificates - if method_is_overridden(type(self), ScrapyClientContextFactory, "getContext"): - warnings.warn( - "Overriding ScrapyClientContextFactory.getContext() is deprecated and that method" - " will be removed in a future Scrapy version. Override creatorForNetloc() instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if method_is_overridden( - type(self), ScrapyClientContextFactory, "getCertificateOptions" - ): # pragma: no cover - warnings.warn( - "Overriding ScrapyClientContextFactory.getCertificateOptions() is deprecated and that method" - " will be removed in a future Scrapy version. Override creatorForNetloc() instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) @classmethod def from_crawler( @@ -110,21 +94,11 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): ) def getCertificateOptions(self) -> CertificateOptions: # pragma: no cover - warnings.warn( - "ScrapyClientContextFactory.getCertificateOptions() is deprecated.", - ScrapyDeprecationWarning, - stacklevel=2, - ) return self._certificate_options # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context: - warnings.warn( - "ScrapyClientContextFactory.getContext() is deprecated.", - ScrapyDeprecationWarning, - stacklevel=2, - ) return self._ctx def _get_context(self) -> SSL.Context: @@ -145,8 +119,16 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): ) +ScrapyClientContextFactory = create_deprecated_class( + "ScrapyClientContextFactory", + _ScrapyClientContextFactory, + subclass_warn_message="{old} is deprecated.", + instance_warn_message="{cls} is deprecated.", +) + + @implementer(IPolicyForHTTPS) -class BrowserLikeContextFactory(ScrapyClientContextFactory): +class BrowserLikeContextFactory(_ScrapyClientContextFactory): """ Twisted-recommended context factory for web clients. @@ -223,7 +205,7 @@ def _load_context_factory_from_settings(crawler: Crawler) -> IPolicyForHTTPS: Also passes values of other relevant settings to the factory class. """ if crawler.settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] == "SENTINEL": - context_factory_cls = ScrapyClientContextFactory + context_factory_cls = _ScrapyClientContextFactory else: # pragma: no cover warnings.warn( "The 'DOWNLOADER_CLIENTCONTEXTFACTORY' setting is deprecated.", diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 0ce7eff3b..e720132f0 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -5,7 +5,7 @@ from __future__ import annotations import warnings from typing import TYPE_CHECKING -from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory +from scrapy.core.downloader.contextfactory import _ScrapyClientContextFactory from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.misc import build_from_crawler, load_object @@ -39,8 +39,8 @@ class HTTP10DownloadHandler: settings["DOWNLOADER_HTTPCLIENTFACTORY"] ) if settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] == "SENTINEL": - self.ClientContextFactory: type[ScrapyClientContextFactory] = ( - ScrapyClientContextFactory + self.ClientContextFactory: type[_ScrapyClientContextFactory] = ( + _ScrapyClientContextFactory ) else: # pragma: no cover warnings.warn( diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 390309900..9ec5b8c48 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -38,7 +38,7 @@ class _ScrapyClientTLSOptions(ClientTLSOptions): logging warnings. Instances of this class are returned from - :class:`.ScrapyClientContextFactory`. + :class:`._ScrapyClientContextFactory`. """ def _identityVerifyingInfoCallback( diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index b24d6c843..14b1771f5 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, cast import OpenSSL.SSL import pytest @@ -12,8 +12,8 @@ from twisted.web.client import Response as TxResponse from scrapy.core.downloader import Downloader, Slot from scrapy.core.downloader.contextfactory import ( - ScrapyClientContextFactory, _load_context_factory_from_settings, + _ScrapyClientContextFactory, ) from scrapy.core.downloader.handlers.http11 import _RequestBodyProducer from scrapy.exceptions import ScrapyDeprecationWarning @@ -107,24 +107,10 @@ class TestContextFactory(TestContextFactoryBase): ) assert body == to_bytes(s) - def test_override_getContext(self): - class MyFactory(ScrapyClientContextFactory): - def getContext( - self, hostname: Any = None, port: Any = None - ) -> OpenSSL.SSL.Context: - ctx: OpenSSL.SSL.Context = super().getContext(hostname, port) - return ctx - - with pytest.warns( - ScrapyDeprecationWarning, - match=r"ScrapyClientContextFactory\.getContext\(\) is deprecated", - ): - MyFactory() - class TestContextFactoryTLSMethod(TestContextFactoryBase): async def _assert_factory_works( - self, server_url: str, client_context_factory: ScrapyClientContextFactory + self, server_url: str, client_context_factory: _ScrapyClientContextFactory ) -> None: s = "0123456789" * 10 body = await self.get_page( @@ -160,13 +146,15 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase): async def test_direct_from_crawler(self, server_url: str) -> None: # the setting is ignored crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) - client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler) + client_context_factory = build_from_crawler( + _ScrapyClientContextFactory, crawler + ) assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD await self._assert_factory_works(server_url, client_context_factory) @coroutine_test async def test_direct_init(self, server_url: str) -> None: - client_context_factory = ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD) + client_context_factory = _ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD) assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD await self._assert_factory_works(server_url, client_context_factory) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index b14d15be4..ac05d457d 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -16,7 +16,7 @@ from twisted.web import resource, server, static, util from twisted.web.client import _makeGetterFactory from scrapy.core.downloader import webclient as client -from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory +from scrapy.core.downloader.contextfactory import _ScrapyClientContextFactory from scrapy.exceptions import DownloadTimeoutError from scrapy.http import Headers, Request from scrapy.utils.misc import build_from_crawler @@ -379,7 +379,9 @@ class TestWebClientCustomCiphersSSL(TestWebClientSSL): crawler = get_crawler( settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers} ) - client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler) + client_context_factory = build_from_crawler( + _ScrapyClientContextFactory, crawler + ) body = yield getPage( server_url + "payload", body=s, contextFactory=client_context_factory ) @@ -393,7 +395,9 @@ class TestWebClientCustomCiphersSSL(TestWebClientSSL): "DOWNLOADER_CLIENT_TLS_CIPHERS": "ECDHE-RSA-AES256-GCM-SHA384" } ) - client_context_factory = build_from_crawler(ScrapyClientContextFactory, crawler) + client_context_factory = build_from_crawler( + _ScrapyClientContextFactory, crawler + ) with pytest.raises(OpenSSL.SSL.Error): yield getPage( server_url + "payload", body=s, contextFactory=client_context_factory