Add warnings to components not working without a reactor (#7355)

scrapy/commands/shell.py

@@ -83,6 +83,10 @@ class Command(ScrapyCommand):
         # crawling engine, so the set up in the crawl method won't work
         crawler = self.crawler_process._create_crawler(spidercls)
         crawler._apply_settings()
+        if not crawler.settings.getbool("TWISTED_ENABLED"):
+            raise RuntimeError(
+                "scrapy shell currently doesn't support TWISTED_ENABLED=False"
+            )
         # The Shell class needs a persistent engine in the crawler
         crawler.engine = crawler._create_engine()
         _schedule_coro(crawler.engine.start_async(_start_request_processing=False))

scrapy/core/downloader/handlers/ftp.py

@@ -39,6 +39,7 @@ from urllib.parse import unquote
 from twisted.internet.protocol import ClientCreator, Protocol
 
 from scrapy.core.downloader.handlers.base import BaseDownloadHandler
+from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.defer import maybe_deferred_to_future
@@ -84,6 +85,8 @@ class FTPDownloadHandler(BaseDownloadHandler):
     }
 
     def __init__(self, crawler: Crawler):
+        if not crawler.settings.getbool("TWISTED_ENABLED"):
+            raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.")
         super().__init__(crawler)
         self.default_user = crawler.settings["FTP_USER"]
         self.default_password = crawler.settings["FTP_PASSWORD"]

scrapy/core/downloader/handlers/http10.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import warnings
 from typing import TYPE_CHECKING
 
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.python import to_unicode
@@ -33,6 +33,8 @@ class HTTP10DownloadHandler:
             category=ScrapyDeprecationWarning,
             stacklevel=2,
         )
+        if not crawler.settings.getbool("TWISTED_ENABLED"):  # pragma: no cover
+            raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.")
         self.HTTPClientFactory: type[ScrapyHTTPClientFactory] = load_object(
             settings["DOWNLOADER_HTTPCLIENTFACTORY"]
         )

scrapy/core/downloader/handlers/http11.py

@@ -34,6 +34,7 @@ from scrapy.core.downloader.contextfactory import load_context_factory_from_sett
 from scrapy.exceptions import (
     DownloadCancelledError,
     DownloadTimeoutError,
+    NotConfigured,
     ResponseDataLossError,
     StopDownload,
 )
@@ -80,6 +81,8 @@ class _ResultT(TypedDict):
 
 class HTTP11DownloadHandler(BaseHttpDownloadHandler):
     def __init__(self, crawler: Crawler):
+        if not crawler.settings.getbool("TWISTED_ENABLED"):
+            raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.")
         super().__init__(crawler)
         self._crawler = crawler
 

scrapy/core/downloader/handlers/http2.py

@@ -9,7 +9,7 @@ from twisted.web.client import URI
 from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
 from scrapy.core.downloader.handlers.base import BaseDownloadHandler
 from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent
-from scrapy.exceptions import DownloadTimeoutError
+from scrapy.exceptions import DownloadTimeoutError, NotConfigured
 from scrapy.utils._download_handlers import (
     normalize_bind_address,
     wrap_twisted_exceptions,
@@ -32,6 +32,8 @@ class H2DownloadHandler(BaseDownloadHandler):
     lazy = True
 
     def __init__(self, crawler: Crawler):
+        if not crawler.settings.getbool("TWISTED_ENABLED"):
+            raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.")
         super().__init__(crawler)
         self._crawler = crawler
 

scrapy/mail.py

@@ -23,6 +23,7 @@ from twisted.internet.defer import Deferred
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import to_bytes
+from scrapy.utils.reactorless import is_reactorless
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Sequence
@@ -70,6 +71,8 @@ class MailSender:
         smtpssl: bool = False,
         debug: bool = False,
     ):
+        if is_reactorless():  # pragma: no cover
+            raise RuntimeError(f"{type(self).__name__} requires a Twisted reactor.")
         self.smtphost: str = smtphost
         self.smtpport: int = smtpport
         self.smtpuser: bytes | None = _to_bytes_or_none(smtpuser)

scrapy/shell.py

@@ -27,6 +27,7 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.defer import _schedule_coro, deferred_f_from_coro_f
 from scrapy.utils.misc import load_object
+from scrapy.utils.python import global_object_name
 from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop
 from scrapy.utils.response import open_in_browser
 
@@ -44,6 +45,10 @@ class Shell:
         code: str | None = None,
     ):
         self.crawler: Crawler = crawler
+        if not crawler.settings.getbool("TWISTED_ENABLED"):  # pragma: no cover
+            raise RuntimeError(
+                f"{global_object_name(self.__class__)} currently doesn't support TWISTED_ENABLED=False."
+            )
         self.update_vars: Callable[[dict[str, Any]], None] = update_vars or (
             lambda x: None
         )

tests/test_command_shell.py

@@ -132,6 +132,15 @@ class TestShellCommand:
         ret, _, err = proc("shell", "-c", code, "--set", "TWISTED_ENABLED=False")
         assert ret == 0, err
 
+    def test_no_reactor_unsupported(self) -> None:
+        # to be removed when it's supported
+        ret, out, err = proc("shell", "-c", "item", "--set", "TWISTED_ENABLED=False")
+        assert ret == 1, out or err
+        assert (
+            "RuntimeError: scrapy shell currently doesn't support TWISTED_ENABLED=False"
+            in err
+        )
+
 
 class TestInteractiveShell:
     def test_fetch(self, mockserver: MockServer) -> None:

tests/test_downloader_handler_twisted_ftp.py

@@ -11,7 +11,10 @@ import pytest
 from pytest_twisted import async_yield_fixture
 from twisted.cred import checkers, credentials, portal
 
+from scrapy import Spider
 from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
+from scrapy.crawler import Crawler
+from scrapy.exceptions import NotConfigured
 from scrapy.http import HtmlResponse, Request, Response
 from scrapy.http.response.text import TextResponse
 from scrapy.utils.defer import deferred_f_from_coro_f
@@ -200,3 +203,9 @@ class TestAnonymousFTP(TestFTPBase):
         p = portal.Portal(realm)
         p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous)
         return FTPFactory(portal=p, userAnonymous=self.username)
+
+
+def test_not_configured_without_reactor() -> None:
+    crawler = Crawler(Spider, {"TWISTED_ENABLED": False})
+    with pytest.raises(NotConfigured):
+        FTPDownloadHandler.from_crawler(crawler)

tests/test_downloader_handler_twisted_http11.py

@@ -6,7 +6,10 @@ from typing import TYPE_CHECKING, Any
 
 import pytest
 
+from scrapy import Spider
 from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
+from scrapy.crawler import Crawler
+from scrapy.exceptions import NotConfigured
 from tests.test_downloader_handlers_http_base import (
     TestHttp11Base,
     TestHttpProxyBase,
@@ -32,6 +35,12 @@ class HTTP11DownloadHandlerMixin:
         return HTTP11DownloadHandler
 
 
+def test_not_configured_without_reactor() -> None:
+    crawler = Crawler(Spider, {"TWISTED_ENABLED": False})
+    with pytest.raises(NotConfigured):
+        HTTP11DownloadHandler.from_crawler(crawler)
+
+
 class TestHttp11(HTTP11DownloadHandlerMixin, TestHttp11Base):
     pass
 

tests/test_downloader_handler_twisted_http2.py

@@ -9,7 +9,9 @@ import pytest
 from testfixtures import LogCapture
 from twisted.web.http import H2_ENABLED
 
-from scrapy.exceptions import UnsupportedURLSchemeError
+from scrapy import Spider
+from scrapy.crawler import Crawler
+from scrapy.exceptions import NotConfigured, UnsupportedURLSchemeError
 from scrapy.http import Request
 from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
 from tests.test_downloader_handlers_http_base import (
@@ -47,6 +49,14 @@ class H2DownloadHandlerMixin:
         return H2DownloadHandler
 
 
+def test_not_configured_without_reactor() -> None:
+    from scrapy.core.downloader.handlers.http2 import H2DownloadHandler  # noqa: PLC0415
+
+    crawler = Crawler(Spider, {"TWISTED_ENABLED": False})
+    with pytest.raises(NotConfigured):
+        H2DownloadHandler.from_crawler(crawler)
+
+
 class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
     HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"