diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 66148904b..33c2bb44c 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -34,9 +34,11 @@ from scrapy.utils.reactor import ( install_reactor, is_asyncio_reactor_installed, is_reactor_installed, + set_asyncio_event_loop, verify_installed_asyncio_event_loop, verify_installed_reactor, ) +from scrapy.utils.reactorless import install_reactor_import_hook if TYPE_CHECKING: from collections.abc import Awaitable, Generator, Iterable @@ -103,22 +105,27 @@ class Crawler: self, ) - reactor_class: str = self.settings["TWISTED_REACTOR"] - event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] - if self._init_reactor: - # this needs to be done after the spider settings are merged, - # but before something imports twisted.internet.reactor + use_reactor = self.settings.getbool("TWISTED_ENABLED") + if use_reactor: + reactor_class: str = self.settings["TWISTED_REACTOR"] + event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] + if self._init_reactor: + # this needs to be done after the spider settings are merged, + # but before something imports twisted.internet.reactor + if reactor_class: + install_reactor(reactor_class, event_loop) + else: + from twisted.internet import reactor # noqa: F401 if reactor_class: - install_reactor(reactor_class, event_loop) - else: - from twisted.internet import reactor # noqa: F401 - if reactor_class: - verify_installed_reactor(reactor_class) - if is_asyncio_reactor_installed() and event_loop: - verify_installed_asyncio_event_loop(event_loop) + verify_installed_reactor(reactor_class) + if is_asyncio_reactor_installed() and event_loop: + verify_installed_asyncio_event_loop(event_loop) - if self._init_reactor or reactor_class: - log_reactor_info() + if self._init_reactor or reactor_class: + log_reactor_info() + else: + logger.debug("Not using a Twisted reactor") + self._apply_reactorless_default_settings() self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() @@ -128,6 +135,15 @@ class Crawler: "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} ) + def _apply_reactorless_default_settings(self): + """Change some setting defaults when not using a Twisted reactor. + + Some settings need different defaults when using and not using a + reactor, but as we can't put this logic into default_settings.py we + change them here when the reactor is not used. + """ + self.settings.set("TELNETCONSOLE_ENABLED", False, priority="default") + # Cannot use @deferred_f_from_coro_f because that relies on the reactor # being installed already, which is done within _apply_settings(), inside # this method. @@ -375,6 +391,10 @@ class CrawlerRunner(CrawlerRunnerBase): def __init__(self, settings: dict[str, Any] | Settings | None = None): super().__init__(settings) + if not self.settings.getbool("TWISTED_ENABLED"): + raise RuntimeError( + f"{type(self).__name__} doesn't support TWISTED_ENABLED=False." + ) self._active: set[Deferred[None]] = set() def crawl( @@ -499,9 +519,16 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): "The crawler_or_spidercls argument cannot be a spider object, " "it must be a spider class (or a Crawler object)" ) - if not is_asyncio_reactor_installed(): + if self.settings.getbool("TWISTED_ENABLED"): + if not is_asyncio_reactor_installed(): + raise RuntimeError( + f"When TWISTED_ENABLED is True, {type(self).__name__} " + f"requires that the installed Twisted reactor is " + f'"twisted.internet.asyncioreactor.AsyncioSelectorReactor".' + ) + elif is_reactor_installed(): raise RuntimeError( - f"{type(self).__name__} requires AsyncioSelectorReactor." + "TWISTED_ENABLED is False but a Twisted reactor is installed." ) crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) @@ -730,7 +757,14 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): # The ASYNCIO_EVENT_LOOP setting cannot be overridden by add-ons and # spiders when using AsyncCrawlerProcess. loop_path = self.settings["ASYNCIO_EVENT_LOOP"] - if is_reactor_installed(): + if not self.settings.getbool("TWISTED_ENABLED"): + if is_reactor_installed(): + raise RuntimeError( + "TWISTED_ENABLED is False but a Twisted reactor is installed." + ) + set_asyncio_event_loop(loop_path) + install_reactor_import_hook() + elif is_reactor_installed(): # The user could install a reactor before this class is instantiated. # We need to make sure the reactor is the correct one and the loop # type matches the setting. @@ -761,6 +795,33 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): :param bool install_signal_handlers: whether to install the OS signal handlers from Twisted and Scrapy (default: True) """ + + if not self.settings.getbool("TWISTED_ENABLED"): + self._start_asyncio(stop_after_crawl, install_signal_handlers) + else: + self._start_twisted(stop_after_crawl, install_signal_handlers) + + def _start_asyncio( + self, stop_after_crawl: bool, install_signal_handlers: bool + ) -> None: + # Very basic and will need multiple improvements. + # TODO https://docs.python.org/3/library/asyncio-runner.html#handling-keyboard-interruption + # TODO various exception handling + # TODO consider asyncio.run() + + loop = asyncio.get_event_loop() + if stop_after_crawl: + join_task = loop.create_task(self.join()) + join_task.add_done_callback(lambda _: loop.stop()) + try: + loop.run_forever() # blocking call + finally: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() + + def _start_twisted( + self, stop_after_crawl: bool, install_signal_handlers: bool + ) -> None: from twisted.internet import reactor if stop_after_crawl: diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index d24c1b6c4..f95cdb040 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -44,6 +44,12 @@ class TelnetConsole(protocol.ServerFactory): if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"): raise NotConfigured + if not crawler.settings.getbool("TWISTED_ENABLED"): + raise NotConfigured( + "The TelnetConsole extension requires a Twisted reactor." + " You can set the TELNETCONSOLE_ENABLED setting to False to remove this warning." + ) + self.crawler: Crawler = crawler self.noisy: bool = False self.portrange: list[int] = [ diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7ef5feb9f..385d5f933 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -184,6 +184,7 @@ __all__ = [ "TELNETCONSOLE_PORT", "TELNETCONSOLE_USERNAME", "TEMPLATES_DIR", + "TWISTED_ENABLED", "TWISTED_REACTOR", "URLLENGTH_LIMIT", "USER_AGENT", @@ -522,6 +523,7 @@ TELNETCONSOLE_PASSWORD = None TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) +TWISTED_ENABLED = True TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" URLLENGTH_LIMIT = 2083 diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index a8216bc2b..9ec090995 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -35,15 +35,16 @@ def is_asyncio_available() -> bool: .. versionadded:: 2.14 - Currently this function is identical to - :func:`scrapy.utils.reactor.is_asyncio_reactor_installed`: it returns - ``True`` if the Twisted reactor that is installed is + This function returns ``True`` if there is a running asyncio event loop. If + there is no such loop, it returns ``True`` if the Twisted reactor that is + installed is :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, returns ``False`` if a different reactor is installed, and raises a - :exc:`RuntimeError` if no reactor is installed. In a future Scrapy version, - when Scrapy supports running without a Twisted reactor, this function will - also return ``True`` when running in that mode, so code that doesn't - directly require a Twisted reactor should use this function instead of + :exc:`RuntimeError` if no reactor is installed. + + Code that doesn't directly require a Twisted reactor should use this + function while code that requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` should use :func:`~scrapy.utils.reactor.is_asyncio_reactor_installed`. When this returns ``True``, an asyncio loop is installed and used by @@ -56,10 +57,35 @@ def is_asyncio_available() -> bool: loop or await on :class:`asyncio.Future` objects in Scrapy-related code, but it's possible to await on :class:`~twisted.internet.defer.Deferred` objects. + + .. note:: As this function uses :func:`asyncio.get_running_loop()`, it will + only detect the event loop if called in the same thread and from the + code that runs inside that loop (this shouldn't be a problem when + calling it from code such as spiders and Scrapy components, if Scrapy + is run using one of the supported ways). + + .. versionchanged:: VERSION + This function now also returns ``True`` if there is a running asyncio + loop, even if no Twisted reactor is installed. """ + + # Check if there is a running asyncio loop. + # Can't easily check for an installed but not running one, and if we + # checked that there could be false positives due to some 3rd-party code + # installing it as a side effect (e.g. by calling get_event_loop()). + try: + asyncio.get_running_loop() + except RuntimeError: + pass + else: + return True + + # Check if there is an installed asyncio reactor (it doesn't need to be + # running). if not is_reactor_installed(): raise RuntimeError( - "is_asyncio_available() called without an installed reactor." + "is_asyncio_available() called without an installed reactor" + " or running asyncio loop." ) return is_asyncio_reactor_installed() diff --git a/scrapy/utils/reactorless.py b/scrapy/utils/reactorless.py new file mode 100644 index 000000000..cd60e6c7d --- /dev/null +++ b/scrapy/utils/reactorless.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import sys +from importlib.abc import MetaPathFinder +from typing import TYPE_CHECKING + +from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.reactor import is_reactor_installed + +if TYPE_CHECKING: + from collections.abc import Sequence + from importlib.machinery import ModuleSpec + from types import ModuleType + + +def is_reactorless() -> bool: + """Check if we are running in the reactorless mode, i.e. with + :setting:`TWISTED_ENABLED` set to ``False``. + + As this checks the runtime state and not the setting itself, it can be + wrong when executed very early, before the reactor and/or the asyncio event + loop are initialized. + + .. note:: As this function uses + :func:`scrapy.utils.asyncio.is_asyncio_available()`, it has the same + limitations for detecting a running asyncio event loop as that one. + + .. versionadded:: VERSION + """ + return is_asyncio_available() and not is_reactor_installed() + + +class ReactorImportHook(MetaPathFinder): + """Hook that prevents importing :mod:`twisted.internet.reactor`.""" + + def find_spec( + self, + fullname: str, + path: Sequence[str] | None, + target: ModuleType | None = None, + ) -> ModuleSpec | None: + if fullname == "twisted.internet.reactor": + raise ImportError( + f"Import of {fullname} is forbidden when running without a Twisted reactor," + f" as importing it installs the reactor, which can lead to unexpected behavior." + ) + return None + + +def install_reactor_import_hook() -> None: + """Prevent importing :mod:`twisted.internet.reactor`.""" + + sys.meta_path.insert(0, ReactorImportHook()) diff --git a/tests/AsyncCrawlerProcess/reactorless_datauri.py b/tests/AsyncCrawlerProcess/reactorless_datauri.py new file mode 100644 index 000000000..7906cc9de --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_datauri.py @@ -0,0 +1,27 @@ +from scrapy import Request, Spider +from scrapy.crawler import AsyncCrawlerProcess + + +class DataSpider(Spider): + name = "data" + + async def start(self): + yield Request("data:,foo") + + def parse(self, response): + return {"data": response.text} + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_ENABLED": False, + "DOWNLOAD_HANDLERS": { + "http": None, + "https": None, + "ftp": None, + }, + } +) + +process.crawl(DataSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_import_hook.py b/tests/AsyncCrawlerProcess/reactorless_import_hook.py new file mode 100644 index 000000000..2f949cfc1 --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_import_hook.py @@ -0,0 +1,27 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + import twisted.internet.reactor # noqa: F401 + + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_ENABLED": False, + "DOWNLOAD_HANDLERS": { + "http": None, + "https": None, + "ftp": None, + }, + } +) + +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_reactor.py b/tests/AsyncCrawlerProcess/reactorless_reactor.py new file mode 100644 index 000000000..a32beee22 --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_reactor.py @@ -0,0 +1,15 @@ +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.reactor import install_reactor + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + +AsyncCrawlerProcess( + settings={ + "TWISTED_ENABLED": False, + "DOWNLOAD_HANDLERS": { + "http": None, + "https": None, + "ftp": None, + }, + } +) diff --git a/tests/AsyncCrawlerProcess/reactorless_simple.py b/tests/AsyncCrawlerProcess/reactorless_simple.py new file mode 100644 index 000000000..dbe9c73b4 --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_simple.py @@ -0,0 +1,27 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.reactorless import is_reactorless + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + self.logger.info(f"is_reactorless(): {is_reactorless()}") + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_ENABLED": False, + "DOWNLOAD_HANDLERS": { + "http": None, + "https": None, + "ftp": None, + }, + } +) + +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_telnetconsole_default.py b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_default.py new file mode 100644 index 000000000..1a4bc5148 --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_default.py @@ -0,0 +1,25 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_ENABLED": False, + "DOWNLOAD_HANDLERS": { + "http": None, + "https": None, + "ftp": None, + }, + } +) + +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_telnetconsole_disabled.py b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_disabled.py new file mode 100644 index 000000000..1814071ee --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_disabled.py @@ -0,0 +1,26 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_ENABLED": False, + "DOWNLOAD_HANDLERS": { + "http": None, + "https": None, + "ftp": None, + }, + "TELNETCONSOLE_ENABLED": False, + } +) + +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_telnetconsole_enabled.py b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_enabled.py new file mode 100644 index 000000000..0026a3f45 --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_enabled.py @@ -0,0 +1,26 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_ENABLED": False, + "DOWNLOAD_HANDLERS": { + "http": None, + "https": None, + "ftp": None, + }, + "TELNETCONSOLE_ENABLED": True, + } +) + +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/simple.py b/tests/AsyncCrawlerProcess/simple.py index d24b4f193..368e05608 100644 --- a/tests/AsyncCrawlerProcess/simple.py +++ b/tests/AsyncCrawlerProcess/simple.py @@ -1,11 +1,13 @@ import scrapy from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.reactorless import is_reactorless class NoRequestsSpider(scrapy.Spider): name = "no_request" async def start(self): + self.logger.info(f"is_reactorless(): {is_reactorless()}") return yield diff --git a/tests/AsyncCrawlerRunner/reactorless_datauri.py b/tests/AsyncCrawlerRunner/reactorless_datauri.py new file mode 100644 index 000000000..cd764b8db --- /dev/null +++ b/tests/AsyncCrawlerRunner/reactorless_datauri.py @@ -0,0 +1,33 @@ +import asyncio + +from scrapy import Request, Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.log import configure_logging + + +class DataSpider(Spider): + name = "data" + + async def start(self): + yield Request("data:,foo") + + def parse(self, response): + return {"data": response.text} + + +async def main(): + configure_logging() + runner = AsyncCrawlerRunner( + settings={ + "TWISTED_ENABLED": False, + "DOWNLOAD_HANDLERS": { + "http": None, + "https": None, + "ftp": None, + }, + } + ) + await runner.crawl(DataSpider) + + +asyncio.run(main()) diff --git a/tests/AsyncCrawlerRunner/reactorless_reactor.py b/tests/AsyncCrawlerRunner/reactorless_reactor.py new file mode 100644 index 000000000..8266ffbc0 --- /dev/null +++ b/tests/AsyncCrawlerRunner/reactorless_reactor.py @@ -0,0 +1,33 @@ +import asyncio + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +async def main(): + configure_logging() + runner = AsyncCrawlerRunner( + settings={ + "TWISTED_ENABLED": False, + "DOWNLOAD_HANDLERS": { + "http": None, + "https": None, + "ftp": None, + }, + } + ) + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +asyncio.run(main()) diff --git a/tests/AsyncCrawlerRunner/reactorless_simple.py b/tests/AsyncCrawlerRunner/reactorless_simple.py new file mode 100644 index 000000000..7a9e061e9 --- /dev/null +++ b/tests/AsyncCrawlerRunner/reactorless_simple.py @@ -0,0 +1,33 @@ +import asyncio + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactorless import is_reactorless + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + self.logger.info(f"is_reactorless(): {is_reactorless()}") + return + yield + + +async def main(): + configure_logging() + runner = AsyncCrawlerRunner( + settings={ + "TWISTED_ENABLED": False, + "DOWNLOAD_HANDLERS": { + "http": None, + "https": None, + "ftp": None, + }, + } + ) + await runner.crawl(NoRequestsSpider) + + +asyncio.run(main()) diff --git a/tests/AsyncCrawlerRunner/simple.py b/tests/AsyncCrawlerRunner/simple.py index 140777b4f..29b132d75 100644 --- a/tests/AsyncCrawlerRunner/simple.py +++ b/tests/AsyncCrawlerRunner/simple.py @@ -5,12 +5,14 @@ from scrapy.crawler import AsyncCrawlerRunner from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.log import configure_logging from scrapy.utils.reactor import install_reactor +from scrapy.utils.reactorless import is_reactorless class NoRequestsSpider(Spider): name = "no_request" async def start(self): + self.logger.info(f"is_reactorless(): {is_reactorless()}") return yield diff --git a/tests/CrawlerProcess/reactorless.py b/tests/CrawlerProcess/reactorless.py new file mode 100644 index 000000000..6cfafe605 --- /dev/null +++ b/tests/CrawlerProcess/reactorless.py @@ -0,0 +1,12 @@ +from scrapy.crawler import CrawlerProcess + +CrawlerProcess( + settings={ + "TWISTED_ENABLED": False, + "DOWNLOAD_HANDLERS": { + "http": None, + "https": None, + "ftp": None, + }, + } +) diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py index 9e4ad70d9..d5a7ec53a 100644 --- a/tests/CrawlerProcess/simple.py +++ b/tests/CrawlerProcess/simple.py @@ -1,11 +1,13 @@ import scrapy from scrapy.crawler import CrawlerProcess +from scrapy.utils.reactorless import is_reactorless class NoRequestsSpider(scrapy.Spider): name = "no_request" async def start(self): + self.logger.info(f"is_reactorless(): {is_reactorless()}") return yield diff --git a/tests/CrawlerRunner/reactorless.py b/tests/CrawlerRunner/reactorless.py new file mode 100644 index 000000000..be4eb10fb --- /dev/null +++ b/tests/CrawlerRunner/reactorless.py @@ -0,0 +1,12 @@ +from scrapy.crawler import CrawlerRunner + +CrawlerRunner( + settings={ + "TWISTED_ENABLED": False, + "DOWNLOAD_HANDLERS": { + "http": None, + "https": None, + "ftp": None, + }, + } +) diff --git a/tests/CrawlerRunner/simple.py b/tests/CrawlerRunner/simple.py index d154dcde4..47c0fe04f 100644 --- a/tests/CrawlerRunner/simple.py +++ b/tests/CrawlerRunner/simple.py @@ -4,12 +4,14 @@ from scrapy import Spider from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging from scrapy.utils.reactor import install_reactor +from scrapy.utils.reactorless import is_reactorless class NoRequestsSpider(Spider): name = "no_request" async def start(self): + self.logger.info(f"is_reactorless(): {is_reactorless()}") return yield diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 350c82a0b..6d0c82825 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -800,6 +800,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" in log ) + assert "is_reactorless(): False" in log def test_multi(self): log = self.run_script("multi.py") @@ -1042,6 +1043,12 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): in log ) + def test_reactorless(self): + log = self.run_script("reactorless.py") + assert ( + "RuntimeError: CrawlerProcess doesn't support TWISTED_ENABLED=False" in log + ) + class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): @property @@ -1076,6 +1083,59 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): "setting (uvloop.Loop)" ) in log + def test_reactorless_simple(self): + log = self.run_script("reactorless_simple.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "is_reactorless(): True" in log + assert "ERROR: " not in log + assert "WARNING: " not in log + + def test_reactorless_datauri(self): + log = self.run_script("reactorless_datauri.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "{'data': 'foo'}" in log + assert "'item_scraped_count': 1" in log + assert "ERROR: " not in log + assert "WARNING: " not in log + + def test_reactorless_import_hook(self): + log = self.run_script("reactorless_import_hook.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "ImportError: Import of twisted.internet.reactor is forbidden" in log + + def test_reactorless_telnetconsole_default(self): + """By default TWISTED_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False.""" + log = self.run_script("reactorless_telnetconsole_default.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "The TelnetConsole extension requires a Twisted reactor" not in log + assert "scrapy.extensions.telnet.TelnetConsole" not in log + + def test_reactorless_telnetconsole_disabled(self): + """Explicit TELNETCONSOLE_ENABLED=False, there are no warnings.""" + log = self.run_script("reactorless_telnetconsole_disabled.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "The TelnetConsole extension requires a Twisted reactor" not in log + assert "scrapy.extensions.telnet.TelnetConsole" not in log + + def test_reactorless_telnetconsole_enabled(self): + """Explicit TELNETCONSOLE_ENABLED=True, the user gets a warning.""" + log = self.run_script("reactorless_telnetconsole_enabled.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "The TelnetConsole extension requires a Twisted reactor" in log + + def test_reactorless_reactor(self): + log = self.run_script("reactorless_reactor.py") + assert ( + "RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed" + in log + ) + class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): """Common tests between CrawlerRunner and AsyncCrawlerRunner, @@ -1089,6 +1149,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" in log ) + assert "is_reactorless(): False" in log def test_multi_parallel(self): log = self.run_script("multi_parallel.py") @@ -1164,6 +1225,12 @@ class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): ) assert "DEBUG: Using asyncio event loop" in log + def test_reactorless(self): + log = self.run_script("reactorless.py") + assert ( + "RuntimeError: CrawlerRunner doesn't support TWISTED_ENABLED=False" in log + ) + class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): @property @@ -1173,7 +1240,34 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): def test_simple_default_reactor(self): log = self.run_script("simple_default_reactor.py") assert "Spider closed (finished)" not in log - assert "RuntimeError: AsyncCrawlerRunner requires AsyncioSelectorReactor" in log + assert ( + "RuntimeError: When TWISTED_ENABLED is True, " + "AsyncCrawlerRunner requires that the installed Twisted reactor" + ) in log + + def test_reactorless_simple(self): + log = self.run_script("reactorless_simple.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "is_reactorless(): True" in log + assert "ERROR: " not in log + assert "WARNING: " not in log + + def test_reactorless_datauri(self): + log = self.run_script("reactorless_datauri.py") + assert "Not using a Twisted reactor" in log + assert "Spider closed (finished)" in log + assert "{'data': 'foo'}" in log + assert "'item_scraped_count': 1" in log + assert "ERROR: " not in log + assert "WARNING: " not in log + + def test_reactorless_reactor(self): + log = self.run_script("reactorless_reactor.py") + assert ( + "RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed" + in log + ) @pytest.mark.parametrize(