From 74e6b61071c9fee372da4342604cd35510d03718 Mon Sep 17 00:00:00 2001 From: Adrian Date: Thu, 25 Jun 2026 13:44:51 +0200 Subject: [PATCH 1/3] Make DOWNLOADER_CLIENT_TLS_CIPHERS=None enable Twisted defaults (#7665) * Make DOWNLOADER_CLIENT_TLS_CIPHERS=None enable Twisted defaults * Remove pointless comment * Remove unnecessary mypy comments --- docs/topics/settings.rst | 3 ++ scrapy/core/downloader/contextfactory.py | 11 +++-- scrapy/core/downloader/tls.py | 12 +++--- tests/test_core_downloader.py | 52 +++++++++++++++++++++++- 4 files changed, 66 insertions(+), 12 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index c452f88d1..b92733ede 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -763,6 +763,9 @@ necessary to access certain HTTPS websites: for example, you may need to use ``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a specific cipher that is not included in ``DEFAULT`` if a website requires it. +Set this setting to ``None`` to use the default ciphers of the underlying TLS +implementation. + .. _OpenSSL cipher list format: https://docs.openssl.org/master/man1/openssl-ciphers/#cipher-list-format .. note:: diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 40f09ffb2..c1796c723 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -17,7 +17,6 @@ from zope.interface.verify import verifyObject from scrapy.core.downloader.tls import ( _TWISTED_VERSION_MAP, - DEFAULT_CIPHERS, _openssl_methods, _ScrapyClientTLSOptions, _ScrapyClientTLSOptions26, @@ -68,11 +67,11 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): self.tls_min_version: TLSVersion | None = tls_min_version self.tls_max_version: TLSVersion | None = tls_max_version self.tls_verbose_logging: bool = tls_verbose_logging # unused - self.tls_ciphers: AcceptableCiphers - if tls_ciphers: - self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers) - else: - self.tls_ciphers = DEFAULT_CIPHERS + self.tls_ciphers: AcceptableCiphers | None = ( + AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers) + if tls_ciphers + else None + ) self._verify_certificates = verify_certificates @classmethod diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 3b903b39d..c5cf1156d 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -43,6 +43,13 @@ _openssl_methods: dict[str, int] = { def __getattr__(name: str) -> Any: + if name == "DEFAULT_CIPHERS": + warnings.warn( + "scrapy.core.downloader.tls.DEFAULT_CIPHERS is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return AcceptableCiphers.fromOpenSSLCipherString("DEFAULT") deprecated = { "METHOD_TLS": "TLS", "METHOD_TLSv10": "TLSv1.0", @@ -177,8 +184,3 @@ class _ScrapyClientTLSOptions26(ClientTLSOptions): return True return verifyCallback - - -DEFAULT_CIPHERS: AcceptableCiphers = AcceptableCiphers.fromOpenSSLCipherString( - "DEFAULT" -) diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index e348bfb7f..fdd5edc27 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -8,7 +8,7 @@ import pytest from pytest_twisted import async_yield_fixture from twisted.internet.protocol import Factory from twisted.internet.protocol import Protocol as TxProtocol -from twisted.internet.ssl import optionsForClientTLS +from twisted.internet.ssl import AcceptableCiphers, optionsForClientTLS from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody @@ -180,6 +180,49 @@ class TestContextFactory(TestContextFactoryBase): assert options & 0x4 # OP_LEGACY_SERVER_CONNECT +class TestContextFactoryCiphers(TestContextFactoryBase): + async def _assert_factory_works( + self, server_url: str, client_context_factory: _ScrapyClientContextFactory + ) -> None: + s = "0123456789" * 10 + body = await self.get_page( + server_url + "payload", client_context_factory, body=s + ) + assert body == to_bytes(s) + + def test_default(self) -> None: + """The default 'DEFAULT' value is passed to Twisted as is.""" + crawler = get_crawler() + factory = build_from_crawler(_ScrapyClientContextFactory, crawler) + assert factory.tls_ciphers is not None + # OpenSSLAcceptableCiphers has no __eq__, so compare the parsed ciphers. + assert ( + factory.tls_ciphers._ciphers + == AcceptableCiphers.fromOpenSSLCipherString("DEFAULT")._ciphers + ) + assert factory._get_cert_options_kwargs()["acceptableCiphers"] is not None + + def test_custom(self) -> None: + crawler = get_crawler( + settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": "CAMELLIA256-SHA"} + ) + factory = build_from_crawler(_ScrapyClientContextFactory, crawler) + assert factory.tls_ciphers is not None + assert ( + factory.tls_ciphers._ciphers + == AcceptableCiphers.fromOpenSSLCipherString("CAMELLIA256-SHA")._ciphers + ) + + @coroutine_test + async def test_none(self, server_url: str) -> None: + """A None value enables the Twisted default ciphers.""" + crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": None}) + factory = build_from_crawler(_ScrapyClientContextFactory, crawler) + assert factory.tls_ciphers is None + assert factory._get_cert_options_kwargs()["acceptableCiphers"] is None + await self._assert_factory_works(server_url, factory) + + class TestContextFactoryTLSMethod(TestContextFactoryBase): async def _assert_factory_works( self, server_url: str, client_context_factory: _ScrapyClientContextFactory @@ -278,3 +321,10 @@ def test_deprecated_tls_module_names() -> None: match="scrapy.core.downloader.tls.openssl_methods is deprecated", ): assert isinstance(tls.openssl_methods, dict) + with pytest.warns( + ScrapyDeprecationWarning, + match="scrapy.core.downloader.tls.DEFAULT_CIPHERS is deprecated", + ): + assert tls.DEFAULT_CIPHERS._ciphers == ( + AcceptableCiphers.fromOpenSSLCipherString("DEFAULT")._ciphers + ) From d8d7de2339b3aeabdc144fa7d0d11cae3ecfd1bb Mon Sep 17 00:00:00 2001 From: Sriniketh24 Date: Fri, 26 Jun 2026 07:56:59 +0530 Subject: [PATCH 2/3] Fix FTPDownloadHandler not closing FTP connection after download (#7667) --- scrapy/core/downloader/handlers/ftp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 6258067c1..9064aa53a 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -119,7 +119,10 @@ class FTPDownloadHandler(BaseDownloadHandler): httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"]) return Response(url=request.url, status=httpcode, body=message.encode()) raise - protocol.close() + finally: + protocol.close() + assert client.transport + client.transport.loseConnection() headers = {"local filename": protocol.filename or b"", "size": protocol.size} body = protocol.filename or protocol.body.read() respcls = responsetypes.from_args(url=request.url, body=body) From 0007676e8d5210f8d0d05ffb12610ca780a3c5ea Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 26 Jun 2026 09:21:12 +0200 Subject: [PATCH 3/3] Improve test coverage for http/ (#7672) * Improve test coverage for http/ * Use isinstance() instead of type() --- tests/test_http_cookies.py | 53 +++++++++++++++++++++++++++++++- tests/test_http_headers.py | 1 + tests/test_http_request.py | 10 ++++++ tests/test_http_request_form.py | 3 ++ tests/test_http_response.py | 5 +++ tests/test_http_response_text.py | 7 +++++ 6 files changed, 78 insertions(+), 1 deletion(-) diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 660b76d08..ce9296764 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -1,8 +1,59 @@ +from http.cookiejar import DefaultCookiePolicy + from scrapy.http import Request, Response -from scrapy.http.cookies import WrappedRequest, WrappedResponse +from scrapy.http.cookies import CookieJar, WrappedRequest, WrappedResponse from scrapy.utils.httpobj import urlparse_cached +class TestCookieJar: + def setup_method(self): + self.jar = CookieJar() + self.request = Request("http://example.com/") + self.response = Response( + "http://example.com/", + headers={"Set-Cookie": "name=value; Domain=example.com; Path=/"}, + ) + + def test_extract_cookies(self): + assert len(self.jar) == 0 + self.jar.extract_cookies(self.response, self.request) + assert len(self.jar) == 1 + cookie = next(iter(self.jar)) + assert cookie.name == "name" + assert cookie.value == "value" + assert ".example.com" in self.jar._cookies + + def test_make_cookies_and_set_cookie(self): + cookies = self.jar.make_cookies(self.response, self.request) + assert len(cookies) == 1 + jar = CookieJar() + for cookie in cookies: + jar.set_cookie(cookie) + assert len(jar) == 1 + + def test_clear(self): + self.jar.extract_cookies(self.response, self.request) + assert len(self.jar) == 1 + self.jar.clear() + assert len(self.jar) == 0 + + def test_clear_session_cookies(self): + self.jar.extract_cookies(self.response, self.request) + assert len(self.jar) == 1 + self.jar.clear_session_cookies() + assert len(self.jar) == 0 + + def test_set_policy(self): + policy = DefaultCookiePolicy() + self.jar.set_policy(policy) + assert self.jar.jar._policy is policy + + def test_check_expired_frequency(self): + jar = CookieJar(check_expired_frequency=1) + jar.add_cookie_header(self.request) + assert jar.processed == 1 + + class TestWrappedRequest: def setup_method(self): self.request = Request( diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 243aa6afe..aff3562e3 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -140,6 +140,7 @@ class TestHeaders: h1["foo"] = "bar" h1["foo"] = None h1.setdefault("foo", "bar") + assert h1["foo"] is None assert h1.get("foo") is None assert h1.getlist("foo") == [] diff --git a/tests/test_http_request.py b/tests/test_http_request.py index e22689d49..1941b826f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -24,6 +24,10 @@ class TestRequest: with pytest.raises(TypeError): self.request_class(123) + # priority argument must be an integer + with pytest.raises(TypeError, match="Request priority not an integer"): + self.request_class("http://www.example.com", priority="1") + r = self.request_class("http://www.example.com") assert isinstance(r.url, str) assert r.url == "http://www.example.com" @@ -274,6 +278,12 @@ class TestRequest: assert r4.meta == {} assert r4.dont_filter is False + # the cls argument allows changing the resulting class + custom_request_cls = type("CustomRequest", (self.request_class,), {}) + r5 = r1.replace(cls=custom_request_cls) + assert isinstance(r5, custom_request_cls) + assert r5.url == r1.url + def test_method_always_str(self): r = self.request_class("http://www.example.com", method="POST") assert isinstance(r.method, str) diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py index a4f87d50d..c667e48a1 100644 --- a/tests/test_http_request_form.py +++ b/tests/test_http_request_form.py @@ -294,6 +294,9 @@ class TestFormRequest(TestRequest): assert request.method == "GET" request = FormRequest.from_response(response, method="POST") assert request.method == "POST" + # an explicit method=None skips form-method normalization + request = FormRequest.from_response(response, method=None) + assert request.method == "NONE" def test_from_response_override_url(self): response = _buildresponse( diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 079c547c7..a8ea4920e 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -254,6 +254,11 @@ class TestResponse: with pytest.raises(ValueError, match="url can't be None"): r.follow(None) + def test_follow_None_encoding(self): + r = self.response_class("http://example.com") + with pytest.raises(ValueError, match="encoding can't be None"): + r.follow("foo", encoding=None) + @pytest.mark.xfail( not W3LIB_STRIPS_URLS, reason="https://github.com/scrapy/w3lib/pull/207", diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py index 4b3fa2302..507fd1864 100644 --- a/tests/test_http_response_text.py +++ b/tests/test_http_response_text.py @@ -14,6 +14,13 @@ from tests.test_http_response import TestResponse class TestTextResponse(TestResponse): response_class = TextResponse + def test_follow_None_encoding(self): + # unlike the base Response, TextResponse.follow() falls back to the + # response encoding when encoding is None instead of raising + r = self.response_class("http://example.com", body=b"hello", encoding="cp1252") + req = r.follow("foo", encoding=None) + assert req.encoding == "cp1252" + def test_replace(self): super().test_replace() r1 = self.response_class(