From 03d105ac924e0f21ab1fa5daa46dda9544ac111d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 25 Mar 2026 13:21:08 +0500 Subject: [PATCH] Add warnings to components not working without a reactor (#7355) --- scrapy/commands/shell.py | 4 ++++ scrapy/core/downloader/handlers/ftp.py | 3 +++ scrapy/core/downloader/handlers/http10.py | 4 +++- scrapy/core/downloader/handlers/http11.py | 3 +++ scrapy/core/downloader/handlers/http2.py | 4 +++- scrapy/mail.py | 3 +++ scrapy/shell.py | 5 +++++ tests/test_command_shell.py | 9 +++++++++ tests/test_downloader_handler_twisted_ftp.py | 9 +++++++++ tests/test_downloader_handler_twisted_http11.py | 9 +++++++++ tests/test_downloader_handler_twisted_http2.py | 12 +++++++++++- 11 files changed, 62 insertions(+), 3 deletions(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 080f62382..9f4faba3a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -83,6 +83,10 @@ class Command(ScrapyCommand): # crawling engine, so the set up in the crawl method won't work crawler = self.crawler_process._create_crawler(spidercls) crawler._apply_settings() + if not crawler.settings.getbool("TWISTED_ENABLED"): + raise RuntimeError( + "scrapy shell currently doesn't support TWISTED_ENABLED=False" + ) # The Shell class needs a persistent engine in the crawler crawler.engine = crawler._create_engine() _schedule_coro(crawler.engine.start_async(_start_request_processing=False)) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index f261e8e7f..65660d389 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -39,6 +39,7 @@ from urllib.parse import unquote from twisted.internet.protocol import ClientCreator, Protocol from scrapy.core.downloader.handlers.base import BaseDownloadHandler +from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.responsetypes import responsetypes from scrapy.utils.defer import maybe_deferred_to_future @@ -84,6 +85,8 @@ class FTPDownloadHandler(BaseDownloadHandler): } def __init__(self, crawler: Crawler): + if not crawler.settings.getbool("TWISTED_ENABLED"): + raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.") super().__init__(crawler) self.default_user = crawler.settings["FTP_USER"] self.default_password = crawler.settings["FTP_PASSWORD"] diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index af3588019..3820e79bd 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -5,7 +5,7 @@ from __future__ import annotations import warnings from typing import TYPE_CHECKING -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import to_unicode @@ -33,6 +33,8 @@ class HTTP10DownloadHandler: category=ScrapyDeprecationWarning, stacklevel=2, ) + if not crawler.settings.getbool("TWISTED_ENABLED"): # pragma: no cover + raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.") self.HTTPClientFactory: type[ScrapyHTTPClientFactory] = load_object( settings["DOWNLOADER_HTTPCLIENTFACTORY"] ) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 16879cf76..acf2be55e 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -34,6 +34,7 @@ from scrapy.core.downloader.contextfactory import load_context_factory_from_sett from scrapy.exceptions import ( DownloadCancelledError, DownloadTimeoutError, + NotConfigured, ResponseDataLossError, StopDownload, ) @@ -80,6 +81,8 @@ class _ResultT(TypedDict): class HTTP11DownloadHandler(BaseHttpDownloadHandler): def __init__(self, crawler: Crawler): + if not crawler.settings.getbool("TWISTED_ENABLED"): + raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.") super().__init__(crawler) self._crawler = crawler diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index c749a4a73..9b241d61d 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -9,7 +9,7 @@ from twisted.web.client import URI from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.handlers.base import BaseDownloadHandler from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent -from scrapy.exceptions import DownloadTimeoutError +from scrapy.exceptions import DownloadTimeoutError, NotConfigured from scrapy.utils._download_handlers import ( normalize_bind_address, wrap_twisted_exceptions, @@ -32,6 +32,8 @@ class H2DownloadHandler(BaseDownloadHandler): lazy = True def __init__(self, crawler: Crawler): + if not crawler.settings.getbool("TWISTED_ENABLED"): + raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.") super().__init__(crawler) self._crawler = crawler diff --git a/scrapy/mail.py b/scrapy/mail.py index 6ebbe195f..718548b03 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -23,6 +23,7 @@ from twisted.internet.defer import Deferred from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes +from scrapy.utils.reactorless import is_reactorless if TYPE_CHECKING: from collections.abc import Callable, Sequence @@ -70,6 +71,8 @@ class MailSender: smtpssl: bool = False, debug: bool = False, ): + if is_reactorless(): # pragma: no cover + raise RuntimeError(f"{type(self).__name__} requires a Twisted reactor.") self.smtphost: str = smtphost self.smtpport: int = smtpport self.smtpuser: bytes | None = _to_bytes_or_none(smtpuser) diff --git a/scrapy/shell.py b/scrapy/shell.py index 6f33f3f14..4ce6e20c4 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -27,6 +27,7 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import _schedule_coro, deferred_f_from_coro_f from scrapy.utils.misc import load_object +from scrapy.utils.python import global_object_name from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop from scrapy.utils.response import open_in_browser @@ -44,6 +45,10 @@ class Shell: code: str | None = None, ): self.crawler: Crawler = crawler + if not crawler.settings.getbool("TWISTED_ENABLED"): # pragma: no cover + raise RuntimeError( + f"{global_object_name(self.__class__)} currently doesn't support TWISTED_ENABLED=False." + ) self.update_vars: Callable[[dict[str, Any]], None] = update_vars or ( lambda x: None ) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index ee515ede4..5699a64a1 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -132,6 +132,15 @@ class TestShellCommand: ret, _, err = proc("shell", "-c", code, "--set", "TWISTED_ENABLED=False") assert ret == 0, err + def test_no_reactor_unsupported(self) -> None: + # to be removed when it's supported + ret, out, err = proc("shell", "-c", "item", "--set", "TWISTED_ENABLED=False") + assert ret == 1, out or err + assert ( + "RuntimeError: scrapy shell currently doesn't support TWISTED_ENABLED=False" + in err + ) + class TestInteractiveShell: def test_fetch(self, mockserver: MockServer) -> None: diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py index f39e5ecaf..8f43d8071 100644 --- a/tests/test_downloader_handler_twisted_ftp.py +++ b/tests/test_downloader_handler_twisted_ftp.py @@ -11,7 +11,10 @@ import pytest from pytest_twisted import async_yield_fixture from twisted.cred import checkers, credentials, portal +from scrapy import Spider from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler +from scrapy.crawler import Crawler +from scrapy.exceptions import NotConfigured from scrapy.http import HtmlResponse, Request, Response from scrapy.http.response.text import TextResponse from scrapy.utils.defer import deferred_f_from_coro_f @@ -200,3 +203,9 @@ class TestAnonymousFTP(TestFTPBase): p = portal.Portal(realm) p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous) return FTPFactory(portal=p, userAnonymous=self.username) + + +def test_not_configured_without_reactor() -> None: + crawler = Crawler(Spider, {"TWISTED_ENABLED": False}) + with pytest.raises(NotConfigured): + FTPDownloadHandler.from_crawler(crawler) diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index 954dcd246..5d093cc17 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -6,7 +6,10 @@ from typing import TYPE_CHECKING, Any import pytest +from scrapy import Spider from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler +from scrapy.crawler import Crawler +from scrapy.exceptions import NotConfigured from tests.test_downloader_handlers_http_base import ( TestHttp11Base, TestHttpProxyBase, @@ -32,6 +35,12 @@ class HTTP11DownloadHandlerMixin: return HTTP11DownloadHandler +def test_not_configured_without_reactor() -> None: + crawler = Crawler(Spider, {"TWISTED_ENABLED": False}) + with pytest.raises(NotConfigured): + HTTP11DownloadHandler.from_crawler(crawler) + + class TestHttp11(HTTP11DownloadHandlerMixin, TestHttp11Base): pass diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index b14356453..7e2c06d6b 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -9,7 +9,9 @@ import pytest from testfixtures import LogCapture from twisted.web.http import H2_ENABLED -from scrapy.exceptions import UnsupportedURLSchemeError +from scrapy import Spider +from scrapy.crawler import Crawler +from scrapy.exceptions import NotConfigured, UnsupportedURLSchemeError from scrapy.http import Request from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from tests.test_downloader_handlers_http_base import ( @@ -47,6 +49,14 @@ class H2DownloadHandlerMixin: return H2DownloadHandler +def test_not_configured_without_reactor() -> None: + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler # noqa: PLC0415 + + crawler = Crawler(Spider, {"TWISTED_ENABLED": False}) + with pytest.raises(NotConfigured): + H2DownloadHandler.from_crawler(crawler) + + class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base): HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"