From b2b2d0b015948f8ca89fae9984ad67c4e7b33ea8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 2 Apr 2026 13:20:54 +0500 Subject: [PATCH] Rename TWISTED_ENABLED to TWISTED_REACTOR_ENABLED. (#7394) --- docs/topics/asyncio.rst | 41 ++++++++++--------- docs/topics/download-handlers.rst | 6 +-- docs/topics/practices.rst | 14 +++---- docs/topics/settings.rst | 28 ++++++------- docs/topics/shell.rst | 2 +- docs/topics/telnetconsole.rst | 2 +- scrapy/commands/shell.py | 4 +- scrapy/core/downloader/handlers/_httpx.py | 4 +- scrapy/core/downloader/handlers/ftp.py | 2 +- scrapy/core/downloader/handlers/http10.py | 2 +- scrapy/core/downloader/handlers/http11.py | 2 +- scrapy/core/downloader/handlers/http2.py | 2 +- scrapy/crawler.py | 39 +++++++++--------- scrapy/extensions/telnet.py | 2 +- scrapy/settings/default_settings.py | 4 +- scrapy/shell.py | 4 +- scrapy/utils/reactorless.py | 2 +- scrapy/utils/test.py | 2 +- .../reactorless_custom_settings.py | 2 +- .../reactorless_datauri.py | 11 +---- .../reactorless_import_hook.py | 2 +- .../reactorless_reactor.py | 2 +- .../AsyncCrawlerProcess/reactorless_simple.py | 2 +- .../reactorless_sleeping.py | 2 +- .../reactorless_telnetconsole_disabled.py | 2 +- .../reactorless_telnetconsole_enabled.py | 2 +- .../reactorless_custom_settings.py | 2 +- .../AsyncCrawlerRunner/reactorless_datauri.py | 2 +- .../AsyncCrawlerRunner/reactorless_reactor.py | 2 +- .../AsyncCrawlerRunner/reactorless_simple.py | 2 +- tests/CrawlerProcess/reactorless.py | 11 +---- tests/CrawlerRunner/reactorless.py | 11 +---- tests/test_addons.py | 4 +- tests/test_command_check.py | 2 +- tests/test_command_crawl.py | 4 +- tests/test_command_fetch.py | 2 +- tests/test_command_parse.py | 2 +- tests/test_command_runspider.py | 2 +- tests/test_command_shell.py | 10 +++-- tests/test_commands.py | 2 +- tests/test_crawl.py | 2 +- tests/test_crawler.py | 4 +- tests/test_crawler_subprocess.py | 32 +++++++++------ tests/test_downloader_handler_twisted_ftp.py | 2 +- .../test_downloader_handler_twisted_http11.py | 2 +- .../test_downloader_handler_twisted_http2.py | 2 +- 46 files changed, 139 insertions(+), 150 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index c6d26ea94..63bde8279 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -140,8 +140,8 @@ Using Scrapy without a Twisted reactor This is currently experimental and may not be suitable for production use. It's possible to use Scrapy without installing a Twisted reactor at all, by -setting the :setting:`TWISTED_ENABLED` setting to ``False``. In this mode -Scrapy will use the asyncio event loop directly, and most of the Scrapy +setting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In this +mode Scrapy will use the asyncio event loop directly, and most of the Scrapy functionality will work in the same way. Doing this provides several benefits in certain use cases: @@ -149,7 +149,8 @@ Doing this provides several benefits in certain use cases: * A Twisted reactor, once stopped, cannot be started again. This prevents, for example, using several instances of :class:`~scrapy.crawler.AsyncCrawlerProcess` in the same process when they - use a reactor, but with ``TWISTED_ENABLED=False`` it becomes possible. + use a reactor, but with ``TWISTED_REACTOR_ENABLED=False`` it becomes + possible. * There may be limitations imposed by :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` and related Twisted code, such as the requirement of using @@ -195,8 +196,8 @@ necessarily stop working. Other differences ----------------- -When :setting:`TWISTED_ENABLED` is set to ``False``, Scrapy will change the -defaults of some other settings: +When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``, Scrapy will change +the defaults of some other settings: * :setting:`TELNETCONSOLE_ENABLED` is set to ``False``. * The ``"http"`` and ``"https"`` keys in :setting:`DOWNLOAD_HANDLERS_BASE` are @@ -255,8 +256,8 @@ Scrapy provides unified helpers for some of these examples: .. autofunction:: scrapy.utils.asyncio.run_in_thread If your code needs to know whether the reactor is available, you can either -check for the value of the :setting:`TWISTED_ENABLED` setting (you need access -to the :class:`~scrapy.crawler.Crawler` instance to do this) or use the +check for the value of the :setting:`TWISTED_REACTOR_ENABLED` setting (you need +access to the :class:`~scrapy.crawler.Crawler` instance to do this) or use the following function: .. autofunction:: scrapy.utils.reactorless.is_reactorless @@ -273,26 +274,26 @@ Troubleshooting without a Twisted reactor [...]:** Scrapy is configured to run without a reactor, but some code imported :mod:`twisted.internet.reactor`, most likely because that code needs a reactor to be used. You need to stop using this code -or set :setting:`TWISTED_ENABLED` back to ``True``. It's also possible that the -reactor isn't really needed but was installed due to the problem described in -:ref:`asyncio-preinstalled-reactor`, in which case it should be enough to fix -the problematic imports. +or set :setting:`TWISTED_REACTOR_ENABLED` back to ``True``. It's also possible +that the reactor isn't really needed but was installed due to the problem +described in :ref:`asyncio-preinstalled-reactor`, in which case it should be +enough to fix the problematic imports. -**RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed:** -Scrapy is configured to run without a reactor, but a reactor is already -installed before the Scrapy code is executed. If you are trying to set -:setting:`TWISTED_ENABLED` via :ref:`per-spider settings `, -it's currently unsupported. +**RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is +installed:** Scrapy is configured to run without a reactor, but a reactor is +already installed before the Scrapy code is executed. If you are trying to set +:setting:`TWISTED_REACTOR_ENABLED` via :ref:`per-spider settings +`, it's currently unsupported. **RuntimeError: We expected a Twisted reactor to be installed but it isn't:** Scrapy is configured to run with a reactor and not to install one, but a reactor wasn't installed before the Scrapy code is executed. If you are trying -to set :setting:`TWISTED_ENABLED` via :ref:`per-spider settings +to set :setting:`TWISTED_REACTOR_ENABLED` via :ref:`per-spider settings `, it's currently unsupported. -**RuntimeError: doesn't support TWISTED_ENABLED=False:** The listed -class cannot be used with :setting:`TWISTED_ENABLED` set to ``False``. There -may be a replacement in the :ref:`documentation above +**RuntimeError: doesn't support TWISTED_REACTOR_ENABLED=False:** The +listed class cannot be used with :setting:`TWISTED_REACTOR_ENABLED` set to +``False``. There may be a replacement in the :ref:`documentation above ` or the documentation of the affected class. diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 29868bbaa..182ea613a 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -141,7 +141,7 @@ This handler supports ``ftp://host/path`` FTP URIs. It's implemented using :mod:`twisted.protocols.ftp`. .. note:: - This handler is not supported when :setting:`TWISTED_ENABLED` is ``False``. + This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. .. _twisted-http2-handler: @@ -197,7 +197,7 @@ If you want to use this handler you need to replace the default one for the .. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2 .. note:: - This handler is not supported when :setting:`TWISTED_ENABLED` is ``False``. + This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. HTTP11DownloadHandler --------------------- @@ -213,7 +213,7 @@ uses the HTTP/1.1 protocol for them. It's implemented using :mod:`twisted.web.client`. .. note:: - This handler is not supported when :setting:`TWISTED_ENABLED` is ``False``. + This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. HttpxDownloadHandler -------------------- diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 8176063d8..b0a2cd135 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -166,8 +166,8 @@ with :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`): .. seealso:: :doc:`twisted:core/howto/reactor-basics` -And here are examples of using these classes with :setting:`TWISTED_ENABLED` -set to ``False``. +And here are examples of using these classes with +:setting:`TWISTED_REACTOR_ENABLED` set to ``False``. Simple usage of :class:`~scrapy.crawler.AsyncCrawlerProcess`: @@ -184,14 +184,14 @@ Simple usage of :class:`~scrapy.crawler.AsyncCrawlerProcess`: process = AsyncCrawlerProcess( settings={ - "TWISTED_ENABLED": False, + "TWISTED_REACTOR_ENABLED": False, } ) process.crawl(MySpider) process.start() # the script will block here until the crawling is finished -With ``TWISTED_ENABLED=False`` you can use several instances of +With ``TWISTED_REACTOR_ENABLED=False`` you can use several instances of :class:`~scrapy.crawler.AsyncCrawlerProcess` in the same process: .. code-block:: python @@ -207,7 +207,7 @@ With ``TWISTED_ENABLED=False`` you can use several instances of process1 = AsyncCrawlerProcess( settings={ - "TWISTED_ENABLED": False, + "TWISTED_REACTOR_ENABLED": False, } ) process1.crawl(MySpider) @@ -215,7 +215,7 @@ With ``TWISTED_ENABLED=False`` you can use several instances of process2 = AsyncCrawlerProcess( settings={ - "TWISTED_ENABLED": False, + "TWISTED_REACTOR_ENABLED": False, } ) process2.crawl(MySpider) @@ -239,7 +239,7 @@ Using :func:`asyncio.run` with :class:`~scrapy.crawler.AsyncCrawlerRunner`: async def main(): configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) - runner = AsyncCrawlerRunner(settings={"TWISTED_ENABLED": False}) + runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) await runner.crawl(MySpider) # completes when the spider finishes diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 964ea99c0..f0456b4d5 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -305,7 +305,7 @@ These settings cannot be :ref:`set from a spider `. These settings are: -- :setting:`TWISTED_ENABLED` +- :setting:`TWISTED_REACTOR_ENABLED` - :setting:`SPIDER_LOADER_CLASS` and settings used by the corresponding spider loader class, e.g. :setting:`SPIDER_MODULES` and :setting:`SPIDER_LOADER_WARN_ONLY` for the default spider loader class. @@ -358,8 +358,8 @@ e.g. in :ref:`per-spider settings `, an exception will be raised. All of these settings, except for :setting:`ASYNCIO_EVENT_LOOP`, are only used -when the Twisted reactor is used, i.e. when :setting:`TWISTED_ENABLED` is -``True``. +when the Twisted reactor is used, i.e. when :setting:`TWISTED_REACTOR_ENABLED` +is ``True``. .. _topics-settings-ref: @@ -659,8 +659,8 @@ Whether to enable DNS in-memory cache. This setting is only used by :class:`~scrapy.resolver.CachingThreadedResolver` and :class:`~scrapy.resolver.CachingHostnameResolver`. It has no effect when - :setting:`TWISTED_ENABLED` is ``False``, and may have no effect either when - :setting:`DNS_RESOLVER` is set to a different resolver. + :setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect + either when :setting:`DNS_RESOLVER` is set to a different resolver. .. setting:: DNSCACHE_SIZE @@ -686,7 +686,7 @@ addresses. Scrapy provides an alternative resolver, take the :setting:`DNS_TIMEOUT` setting into account. .. note:: - This setting has no effect when :setting:`TWISTED_ENABLED` is ``False``. + This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. .. setting:: DNS_TIMEOUT @@ -700,8 +700,8 @@ Timeout for processing of DNS queries in seconds. Float is supported. .. note:: This setting is only used by :class:`~scrapy.resolver.CachingThreadedResolver`. It has no effect when - :setting:`TWISTED_ENABLED` is ``False``, and may have no effect either when - :setting:`DNS_RESOLVER` is set to a different resolver. + :setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect + either when :setting:`DNS_RESOLVER` is set to a different resolver. .. setting:: DOWNLOADER @@ -943,7 +943,7 @@ Default: "ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler", } -(when :setting:`TWISTED_ENABLED` is ``True``) +(when :setting:`TWISTED_REACTOR_ENABLED` is ``True``) .. code-block:: python @@ -956,7 +956,7 @@ Default: "ftp": None, } -(when :setting:`TWISTED_ENABLED` is ``False``) +(when :setting:`TWISTED_REACTOR_ENABLED` is ``False``) A dict containing the :ref:`download handlers ` enabled by default in Scrapy. You should never modify this setting in your @@ -1989,7 +1989,7 @@ For more info see: :ref:`topics-stats`. TELNETCONSOLE_ENABLED --------------------- -Default: ``True`` (``False`` when :setting:`TWISTED_ENABLED` is ``False``) +Default: ``True`` (``False`` when :setting:`TWISTED_REACTOR_ENABLED` is ``False``) A boolean which specifies if the :ref:`telnet console ` will be enabled (provided its extension is also enabled). @@ -2008,10 +2008,10 @@ command. The project name must not conflict with the name of custom files or directories in the ``project`` subdirectory. -.. setting:: TWISTED_ENABLED +.. setting:: TWISTED_REACTOR_ENABLED -TWISTED_ENABLED ---------------- +TWISTED_REACTOR_ENABLED +----------------------- Default: ``True`` diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index b59d45d1b..ca5457b2a 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -18,7 +18,7 @@ Once you get familiarized with the Scrapy shell, you'll see that it's an invaluable tool for developing and debugging your spiders. .. note:: - This feature is not supported when :setting:`TWISTED_ENABLED` is ``False``. + This feature is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. Configuring the shell ===================== diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 4f05057bf..e74afe49b 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -27,7 +27,7 @@ disable it if you want. For more information about the extension itself see or disable it completely using :setting:`TELNETCONSOLE_ENABLED` option. .. note:: - This feature is not supported when :setting:`TWISTED_ENABLED` is ``False``. + This feature is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. .. highlight:: none diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 9f4faba3a..772b2911c 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -83,9 +83,9 @@ class Command(ScrapyCommand): # crawling engine, so the set up in the crawl method won't work crawler = self.crawler_process._create_crawler(spidercls) crawler._apply_settings() - if not crawler.settings.getbool("TWISTED_ENABLED"): + if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): raise RuntimeError( - "scrapy shell currently doesn't support TWISTED_ENABLED=False" + "scrapy shell currently doesn't support TWISTED_REACTOR_ENABLED=False" ) # The Shell class needs a persistent engine in the crawler crawler.engine = crawler._create_engine() diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index 1787ec918..9bbcea7bb 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -81,8 +81,8 @@ class HttpxDownloadHandler(BaseHttpDownloadHandler): f"{type(self).__name__} requires the asyncio support. Make" f" sure that you have either enabled the asyncio Twisted" f" reactor in the TWISTED_REACTOR setting or disabled the" - f" TWISTED_ENABLED setting. See the asyncio documentation" - f" of Scrapy for more information." + f" TWISTED_REACTOR_ENABLED setting. See the asyncio" + f" documentation of Scrapy for more information." ) if httpx is None: # pragma: no cover raise NotConfigured( diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 65660d389..8327a3b0e 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -85,7 +85,7 @@ class FTPDownloadHandler(BaseDownloadHandler): } def __init__(self, crawler: Crawler): - if not crawler.settings.getbool("TWISTED_ENABLED"): + if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.") super().__init__(crawler) self.default_user = crawler.settings["FTP_USER"] diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index e720132f0..f5d1bbd76 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -33,7 +33,7 @@ class HTTP10DownloadHandler: category=ScrapyDeprecationWarning, stacklevel=2, ) - if not crawler.settings.getbool("TWISTED_ENABLED"): # pragma: no cover + if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): # pragma: no cover raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.") self.HTTPClientFactory: type[ScrapyHTTPClientFactory] = load_object( settings["DOWNLOADER_HTTPCLIENTFACTORY"] diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 6c9b699f6..9dd403aca 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -82,7 +82,7 @@ class _ResultT(TypedDict): class HTTP11DownloadHandler(BaseHttpDownloadHandler): def __init__(self, crawler: Crawler): - if not crawler.settings.getbool("TWISTED_ENABLED"): + if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.") super().__init__(crawler) self._crawler = crawler diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 52ca89b0f..f2774cc07 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -32,7 +32,7 @@ class H2DownloadHandler(BaseDownloadHandler): lazy = True def __init__(self, crawler: Crawler): - if not crawler.settings.getbool("TWISTED_ENABLED"): + if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.") super().__init__(crawler) self._crawler = crawler diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 31cb892a7..d622e407c 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -105,7 +105,7 @@ class Crawler: self, ) - use_reactor = self.settings.getbool("TWISTED_ENABLED") + use_reactor = self.settings.getbool("TWISTED_REACTOR_ENABLED") if use_reactor: # We either install a reactor or expect one to be installed. reactor_class: str = self.settings["TWISTED_REACTOR"] @@ -137,7 +137,7 @@ class Crawler: # We expect a reactor to not be installed. if is_reactor_installed(): raise RuntimeError( - "TWISTED_ENABLED is False but a Twisted reactor is installed." + "TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed." ) logger.debug("Not using a Twisted reactor") self._apply_reactorless_default_settings() @@ -411,9 +411,9 @@ class CrawlerRunner(CrawlerRunnerBase): def __init__(self, settings: dict[str, Any] | Settings | None = None): super().__init__(settings) - if not self.settings.getbool("TWISTED_ENABLED"): + if not self.settings.getbool("TWISTED_REACTOR_ENABLED"): raise RuntimeError( - f"{type(self).__name__} doesn't support TWISTED_ENABLED=False." + f"{type(self).__name__} doesn't support TWISTED_REACTOR_ENABLED=False." ) self._active: set[Deferred[None]] = set() @@ -499,10 +499,10 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): The AsyncCrawlerRunner object must be instantiated with a :class:`~scrapy.settings.Settings` object. - When the :setting:`TWISTED_ENABLED` setting is set to ``True``, this class - requires a reactor to be installed and uses it, otherwise it requires a - reactor to not be installed but requires an asyncio event loop to be - installed and uses it. + When the :setting:`TWISTED_REACTOR_ENABLED` setting is set to ``True``, + this class requires a reactor to be installed and uses it, otherwise it + requires a reactor to not be installed but requires an asyncio event loop + to be installed and uses it. This class shouldn't be needed (since Scrapy is responsible of using it accordingly) unless writing scripts that manually handle the crawling @@ -550,20 +550,20 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): "The crawler_or_spidercls argument cannot be a spider object, " "it must be a spider class (or a Crawler object)" ) - if self.settings.getbool("TWISTED_ENABLED"): + if self.settings.getbool("TWISTED_REACTOR_ENABLED"): if not is_reactor_installed(): raise RuntimeError( "We expected a Twisted reactor to be installed but it isn't." ) if not is_asyncio_reactor_installed(): raise RuntimeError( - f"When TWISTED_ENABLED is True, {type(self).__name__} " + f"When TWISTED_REACTOR_ENABLED is True, {type(self).__name__} " f"requires that the installed Twisted reactor is " f'"twisted.internet.asyncioreactor.AsyncioSelectorReactor".' ) elif is_reactor_installed(): raise RuntimeError( - "TWISTED_ENABLED is False but a Twisted reactor is installed." + "TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed." ) crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) @@ -784,9 +784,9 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): The AsyncCrawlerProcess object must be instantiated with a :class:`~scrapy.settings.Settings` object. - When the :setting:`TWISTED_ENABLED` setting is set to ``True``, this class - installs a reactor and uses it, otherwise it requires a reactor to not be - installed but installs an asyncio event loop and uses it. + When the :setting:`TWISTED_REACTOR_ENABLED` setting is set to ``True``, + this class installs a reactor and uses it, otherwise it requires a reactor + to not be installed but installs an asyncio event loop and uses it. :param install_root_handler: whether to install root logging handler (default: True) @@ -814,10 +814,10 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): # The ASYNCIO_EVENT_LOOP setting cannot be overridden by add-ons and # spiders when using AsyncCrawlerProcess. loop_path = self.settings["ASYNCIO_EVENT_LOOP"] - if not self.settings.getbool("TWISTED_ENABLED"): + if not self.settings.getbool("TWISTED_REACTOR_ENABLED"): if is_reactor_installed(): raise RuntimeError( - "TWISTED_ENABLED is False but a Twisted reactor is installed." + "TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed." ) self._reactorless_loop = set_asyncio_event_loop(loop_path) install_reactor_import_hook() @@ -840,8 +840,9 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): self, stop_after_crawl: bool = True, install_signal_handlers: bool = True ) -> None: """ - This method starts a :mod:`~twisted.internet.reactor`/asyncio event - loop, depending on the value of the :setting:`TWISTED_ENABLED` setting. + This method starts a :mod:`~twisted.internet.reactor` or an asyncio + event loop, depending on the value of the + :setting:`TWISTED_REACTOR_ENABLED` setting. When using a reactor it adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE` and installs a DNS resolver based @@ -857,7 +858,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): handlers from Twisted and Scrapy (default: True) """ - if not self.settings.getbool("TWISTED_ENABLED"): + if not self.settings.getbool("TWISTED_REACTOR_ENABLED"): self._start_asyncio(stop_after_crawl, install_signal_handlers) else: self._start_twisted(stop_after_crawl, install_signal_handlers) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index fec6b2d56..3be24c53f 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -44,7 +44,7 @@ class TelnetConsole(protocol.ServerFactory): if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"): raise NotConfigured - if not crawler.settings.getbool("TWISTED_ENABLED"): + if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): raise NotConfigured( "The TelnetConsole extension requires a Twisted reactor." " You can set the TELNETCONSOLE_ENABLED setting to False to remove this warning." diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a4768a4bf..462240133 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -185,8 +185,8 @@ __all__ = [ "TELNETCONSOLE_PORT", "TELNETCONSOLE_USERNAME", "TEMPLATES_DIR", - "TWISTED_ENABLED", "TWISTED_REACTOR", + "TWISTED_REACTOR_ENABLED", "URLLENGTH_LIMIT", "USER_AGENT", "WARN_ON_GENERATOR_RETURN_VALUE", @@ -526,7 +526,7 @@ TELNETCONSOLE_PASSWORD = None TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) -TWISTED_ENABLED = True +TWISTED_REACTOR_ENABLED = True TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" URLLENGTH_LIMIT = 2083 diff --git a/scrapy/shell.py b/scrapy/shell.py index 4ce6e20c4..6a9464d74 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -45,9 +45,9 @@ class Shell: code: str | None = None, ): self.crawler: Crawler = crawler - if not crawler.settings.getbool("TWISTED_ENABLED"): # pragma: no cover + if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"): # pragma: no cover raise RuntimeError( - f"{global_object_name(self.__class__)} currently doesn't support TWISTED_ENABLED=False." + f"{global_object_name(self.__class__)} currently doesn't support TWISTED_REACTOR_ENABLED=False." ) self.update_vars: Callable[[dict[str, Any]], None] = update_vars or ( lambda x: None diff --git a/scrapy/utils/reactorless.py b/scrapy/utils/reactorless.py index cd60e6c7d..27dc69d93 100644 --- a/scrapy/utils/reactorless.py +++ b/scrapy/utils/reactorless.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: def is_reactorless() -> bool: """Check if we are running in the reactorless mode, i.e. with - :setting:`TWISTED_ENABLED` set to ``False``. + :setting:`TWISTED_REACTOR_ENABLED` set to ``False``. As this checks the runtime state and not the setting itself, it can be wrong when executed very early, before the reactor and/or the asyncio event diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 75f357d50..5c69dc76d 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -128,7 +128,7 @@ def get_reactor_settings() -> dict[str, Any]: # running some 3rd-party library tests without initializing a reactor # properly. The first two cases are fine, but we cannot distinguish the # last one from them. - settings["TWISTED_ENABLED"] = False + settings["TWISTED_REACTOR_ENABLED"] = False settings["DOWNLOAD_HANDLERS"] = { "ftp": None, "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", diff --git a/tests/AsyncCrawlerProcess/reactorless_custom_settings.py b/tests/AsyncCrawlerProcess/reactorless_custom_settings.py index b0d4d47e5..93259f696 100644 --- a/tests/AsyncCrawlerProcess/reactorless_custom_settings.py +++ b/tests/AsyncCrawlerProcess/reactorless_custom_settings.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: class NoRequestsSpider(scrapy.Spider): name = "no_request" custom_settings = { - "TWISTED_ENABLED": False, + "TWISTED_REACTOR_ENABLED": False, } async def start(self): diff --git a/tests/AsyncCrawlerProcess/reactorless_datauri.py b/tests/AsyncCrawlerProcess/reactorless_datauri.py index 7906cc9de..5270b7b83 100644 --- a/tests/AsyncCrawlerProcess/reactorless_datauri.py +++ b/tests/AsyncCrawlerProcess/reactorless_datauri.py @@ -12,16 +12,7 @@ class DataSpider(Spider): return {"data": response.text} -process = AsyncCrawlerProcess( - settings={ - "TWISTED_ENABLED": False, - "DOWNLOAD_HANDLERS": { - "http": None, - "https": None, - "ftp": None, - }, - } -) +process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) process.crawl(DataSpider) process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_import_hook.py b/tests/AsyncCrawlerProcess/reactorless_import_hook.py index c3ef6389a..57e2a5c26 100644 --- a/tests/AsyncCrawlerProcess/reactorless_import_hook.py +++ b/tests/AsyncCrawlerProcess/reactorless_import_hook.py @@ -12,7 +12,7 @@ class NoRequestsSpider(scrapy.Spider): yield -process = AsyncCrawlerProcess(settings={"TWISTED_ENABLED": False}) +process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) process.crawl(NoRequestsSpider) process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_reactor.py b/tests/AsyncCrawlerProcess/reactorless_reactor.py index 59bbc9f0a..481112ca7 100644 --- a/tests/AsyncCrawlerProcess/reactorless_reactor.py +++ b/tests/AsyncCrawlerProcess/reactorless_reactor.py @@ -3,4 +3,4 @@ from scrapy.utils.reactor import install_reactor install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") -AsyncCrawlerProcess(settings={"TWISTED_ENABLED": False}) +AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) diff --git a/tests/AsyncCrawlerProcess/reactorless_simple.py b/tests/AsyncCrawlerProcess/reactorless_simple.py index fa8cb64b0..33b4e8cb2 100644 --- a/tests/AsyncCrawlerProcess/reactorless_simple.py +++ b/tests/AsyncCrawlerProcess/reactorless_simple.py @@ -12,7 +12,7 @@ class NoRequestsSpider(scrapy.Spider): yield -process = AsyncCrawlerProcess(settings={"TWISTED_ENABLED": False}) +process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) process.crawl(NoRequestsSpider) process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_sleeping.py b/tests/AsyncCrawlerProcess/reactorless_sleeping.py index 6bfe64686..12101d221 100644 --- a/tests/AsyncCrawlerProcess/reactorless_sleeping.py +++ b/tests/AsyncCrawlerProcess/reactorless_sleeping.py @@ -14,7 +14,7 @@ class SleepingSpider(scrapy.Spider): await asyncio.sleep(int(sys.argv[1])) -process = AsyncCrawlerProcess(settings={"TWISTED_ENABLED": False}) +process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) process.crawl(SleepingSpider) process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_telnetconsole_disabled.py b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_disabled.py index 8403986fb..e59a365bc 100644 --- a/tests/AsyncCrawlerProcess/reactorless_telnetconsole_disabled.py +++ b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_disabled.py @@ -12,7 +12,7 @@ class NoRequestsSpider(scrapy.Spider): process = AsyncCrawlerProcess( settings={ - "TWISTED_ENABLED": False, + "TWISTED_REACTOR_ENABLED": False, "TELNETCONSOLE_ENABLED": False, } ) diff --git a/tests/AsyncCrawlerProcess/reactorless_telnetconsole_enabled.py b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_enabled.py index 34f0c69ba..34daf8779 100644 --- a/tests/AsyncCrawlerProcess/reactorless_telnetconsole_enabled.py +++ b/tests/AsyncCrawlerProcess/reactorless_telnetconsole_enabled.py @@ -12,7 +12,7 @@ class NoRequestsSpider(scrapy.Spider): process = AsyncCrawlerProcess( settings={ - "TWISTED_ENABLED": False, + "TWISTED_REACTOR_ENABLED": False, "TELNETCONSOLE_ENABLED": True, } ) diff --git a/tests/AsyncCrawlerRunner/reactorless_custom_settings.py b/tests/AsyncCrawlerRunner/reactorless_custom_settings.py index 0545ca3dc..9b5efc3f9 100644 --- a/tests/AsyncCrawlerRunner/reactorless_custom_settings.py +++ b/tests/AsyncCrawlerRunner/reactorless_custom_settings.py @@ -9,7 +9,7 @@ from scrapy.utils.reactorless import is_reactorless class NoRequestsSpider(Spider): name = "no_request" custom_settings = { - "TWISTED_ENABLED": False, + "TWISTED_REACTOR_ENABLED": False, } async def start(self): diff --git a/tests/AsyncCrawlerRunner/reactorless_datauri.py b/tests/AsyncCrawlerRunner/reactorless_datauri.py index 7915fbdcb..d964f9c1c 100644 --- a/tests/AsyncCrawlerRunner/reactorless_datauri.py +++ b/tests/AsyncCrawlerRunner/reactorless_datauri.py @@ -17,7 +17,7 @@ class DataSpider(Spider): async def main() -> None: configure_logging() - runner = AsyncCrawlerRunner(settings={"TWISTED_ENABLED": False}) + runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) await runner.crawl(DataSpider) diff --git a/tests/AsyncCrawlerRunner/reactorless_reactor.py b/tests/AsyncCrawlerRunner/reactorless_reactor.py index 9a77b9c44..4889b2ddc 100644 --- a/tests/AsyncCrawlerRunner/reactorless_reactor.py +++ b/tests/AsyncCrawlerRunner/reactorless_reactor.py @@ -16,7 +16,7 @@ class NoRequestsSpider(Spider): async def main() -> None: configure_logging() - runner = AsyncCrawlerRunner(settings={"TWISTED_ENABLED": False}) + runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) await runner.crawl(NoRequestsSpider) diff --git a/tests/AsyncCrawlerRunner/reactorless_simple.py b/tests/AsyncCrawlerRunner/reactorless_simple.py index 6f42600ad..e91c7c89b 100644 --- a/tests/AsyncCrawlerRunner/reactorless_simple.py +++ b/tests/AsyncCrawlerRunner/reactorless_simple.py @@ -17,7 +17,7 @@ class NoRequestsSpider(Spider): async def main() -> None: configure_logging() - runner = AsyncCrawlerRunner(settings={"TWISTED_ENABLED": False}) + runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) await runner.crawl(NoRequestsSpider) diff --git a/tests/CrawlerProcess/reactorless.py b/tests/CrawlerProcess/reactorless.py index 6cfafe605..c9384010e 100644 --- a/tests/CrawlerProcess/reactorless.py +++ b/tests/CrawlerProcess/reactorless.py @@ -1,12 +1,3 @@ from scrapy.crawler import CrawlerProcess -CrawlerProcess( - settings={ - "TWISTED_ENABLED": False, - "DOWNLOAD_HANDLERS": { - "http": None, - "https": None, - "ftp": None, - }, - } -) +CrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) diff --git a/tests/CrawlerRunner/reactorless.py b/tests/CrawlerRunner/reactorless.py index be4eb10fb..bad2ae8d9 100644 --- a/tests/CrawlerRunner/reactorless.py +++ b/tests/CrawlerRunner/reactorless.py @@ -1,12 +1,3 @@ from scrapy.crawler import CrawlerRunner -CrawlerRunner( - settings={ - "TWISTED_ENABLED": False, - "DOWNLOAD_HANDLERS": { - "http": None, - "https": None, - "ftp": None, - }, - } -) +CrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False}) diff --git a/tests/test_addons.py b/tests/test_addons.py index 5c26e4651..3ad8cf7ff 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -110,7 +110,7 @@ class TestAddonManager: runner_cls = ( CrawlerRunner - if settings_dict.get("TWISTED_ENABLED", True) + if settings_dict.get("TWISTED_REACTOR_ENABLED", True) else AsyncCrawlerRunner ) @@ -201,7 +201,7 @@ class TestAddonManager: settings.set("KEY", "default", priority="default") runner_cls = ( CrawlerRunner - if settings.getbool("TWISTED_ENABLED", True) + if settings.getbool("TWISTED_REACTOR_ENABLED", True) else AsyncCrawlerRunner ) runner = runner_cls(settings) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index c34e75624..84486e717 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -55,7 +55,7 @@ class CheckSpider(scrapy.Spider): self._write_contract(proj_path, contracts, parse_def) args = ["check"] if not use_reactor: - args += ["-s", "TWISTED_ENABLED=False"] + args += ["-s", "TWISTED_REACTOR_ENABLED=False"] ret, out, err = proc(*args, cwd=proj_path) assert "F" not in out assert "OK" in err diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py index 1b5dee961..dd69d36ba 100644 --- a/tests/test_command_crawl.py +++ b/tests/test_command_crawl.py @@ -137,7 +137,9 @@ class MySpider(scrapy.Spider): return yield """ - log = self.get_log(spider_code, proj_path, args=("-s", "TWISTED_ENABLED=False")) + log = self.get_log( + spider_code, proj_path, args=("-s", "TWISTED_REACTOR_ENABLED=False") + ) assert "[myspider] DEBUG: It works!" in log assert "Not using a Twisted reactor" in log assert "Spider closed (finished)" in log diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 57db7dc50..d98dac968 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -33,6 +33,6 @@ class TestFetchCommand: def test_no_reactor(self, mockserver: MockServer) -> None: _, out, _ = proc( - "fetch", "-s", "TWISTED_ENABLED=False", mockserver.url("/text") + "fetch", "-s", "TWISTED_REACTOR_ENABLED=False", mockserver.url("/text") ) assert out.strip() == "Works" diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index d612b705b..c210a06e8 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -523,7 +523,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} "parse", mockserver.url("/html"), "-s", - "TWISTED_ENABLED=False", + "TWISTED_REACTOR_ENABLED=False", cwd=proj_path, ) assert "INFO: Got response 200" in stderr diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index 9bc65f152..b1455611e 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -219,7 +219,7 @@ class MySpider(scrapy.Spider): self.debug_log_spider, args=[ "-s", - "TWISTED_ENABLED=False", + "TWISTED_REACTOR_ENABLED=False", ], ) assert "Not using a Twisted reactor" in log diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 5699a64a1..6a12f28bb 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -129,15 +129,19 @@ class TestShellCommand: def test_shell_fetch_no_reactor(self, mockserver: MockServer) -> None: url = mockserver.url("/html") code = f"fetch('{url}')" - ret, _, err = proc("shell", "-c", code, "--set", "TWISTED_ENABLED=False") + ret, _, err = proc( + "shell", "-c", code, "--set", "TWISTED_REACTOR_ENABLED=False" + ) assert ret == 0, err def test_no_reactor_unsupported(self) -> None: # to be removed when it's supported - ret, out, err = proc("shell", "-c", "item", "--set", "TWISTED_ENABLED=False") + ret, out, err = proc( + "shell", "-c", "item", "--set", "TWISTED_REACTOR_ENABLED=False" + ) assert ret == 1, out or err assert ( - "RuntimeError: scrapy shell currently doesn't support TWISTED_ENABLED=False" + "RuntimeError: scrapy shell currently doesn't support TWISTED_REACTOR_ENABLED=False" in err ) diff --git a/tests/test_commands.py b/tests/test_commands.py index 209ffdc08..1e91aa0b0 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -355,7 +355,7 @@ class TestBenchCommand: "CLOSESPIDER_TIMEOUT=0.01", ] if not use_reactor: - args += ["-s", "TWISTED_ENABLED=False"] + args += ["-s", "TWISTED_REACTOR_ENABLED=False"] _, _, err = proc(*args) assert "INFO: Crawled" in err assert "Unhandled Error" not in err diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 2bc50aad4..811ce1d18 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -407,7 +407,7 @@ with multiples lines settings_dict = get_reactor_settings() runner_cls = ( CrawlerRunner - if settings_dict.get("TWISTED_ENABLED", True) + if settings_dict.get("TWISTED_REACTOR_ENABLED", True) else AsyncCrawlerRunner ) runner = runner_cls(settings_dict) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f4b4fbec9..cbcb7e274 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -638,13 +638,13 @@ class TestCrawlerProcess(TestBaseCrawler): class TestAsyncCrawlerProcess(TestBaseCrawler): def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None: runner = AsyncCrawlerProcess( - {"foo": "bar", "TWISTED_ENABLED": reactor_pytest != "none"}, + {"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"}, install_root_handler=False, ) assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - @pytest.mark.requires_reactor # can't pass TWISTED_ENABLED=False + @pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=False def test_crawler_process_accepts_None(self) -> None: runner = AsyncCrawlerProcess(install_root_handler=False) self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index c13bc0c9d..beae4f277 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -309,7 +309,8 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): def test_reactorless(self): log = self.run_script("reactorless.py") assert ( - "RuntimeError: CrawlerProcess doesn't support TWISTED_ENABLED=False" in log + "RuntimeError: CrawlerProcess doesn't support TWISTED_REACTOR_ENABLED=False" + in log ) @@ -356,12 +357,16 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): assert log.count("WARNING: ") == 2 def test_reactorless_custom_settings(self): - """Setting TWISTED_ENABLED=False in spider settings is not currently supported, - AsyncCrawlerProcess will install a reactor in this case. + """Setting TWISTED_REACTOR_ENABLED=False in spider settings is not + currently supported, AsyncCrawlerProcess will install a reactor in this + case. """ log = self.run_script("reactorless_custom_settings.py") assert "Spider closed (finished)" not in log - assert "TWISTED_ENABLED is False but a Twisted reactor is installed." in log + assert ( + "TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed." + in log + ) def test_reactorless_datauri(self): log = self.run_script("reactorless_datauri.py") @@ -370,7 +375,8 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): assert "{'data': 'foo'}" in log assert "'item_scraped_count': 1" in log assert "ERROR: " not in log - assert "WARNING: " not in log + assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 + assert log.count("WARNING: ") == 2 def test_reactorless_import_hook(self): log = self.run_script("reactorless_import_hook.py") @@ -379,7 +385,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): assert "ImportError: Import of twisted.internet.reactor is forbidden" in log def test_reactorless_telnetconsole_default(self): - """By default TWISTED_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False.""" + """By default TWISTED_REACTOR_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False.""" log = self.run_script("reactorless_simple.py") # no need for a separate script assert "Not using a Twisted reactor" in log assert "Spider closed (finished)" in log @@ -404,7 +410,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): def test_reactorless_reactor(self): log = self.run_script("reactorless_reactor.py") assert ( - "RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed" + "RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed" in log ) @@ -515,7 +521,8 @@ class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): def test_reactorless(self): log = self.run_script("reactorless.py") assert ( - "RuntimeError: CrawlerRunner doesn't support TWISTED_ENABLED=False" in log + "RuntimeError: CrawlerRunner doesn't support TWISTED_REACTOR_ENABLED=False" + in log ) @@ -528,7 +535,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): log = self.run_script("simple_default_reactor.py") assert "Spider closed (finished)" not in log assert ( - "RuntimeError: When TWISTED_ENABLED is True, " + "RuntimeError: When TWISTED_REACTOR_ENABLED is True, " "AsyncCrawlerRunner requires that the installed Twisted reactor" ) in log @@ -542,8 +549,9 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): assert log.count("WARNING: ") == 2 def test_reactorless_custom_settings(self): - """Setting TWISTED_ENABLED=False in spider settings is not currently supported, - AsyncCrawlerRunner will expect a reactor installed by the user. + """Setting TWISTED_REACTOR_ENABLED=False in spider settings is not + currently supported, AsyncCrawlerRunner will expect a reactor installed + by the user. """ log = self.run_script("reactorless_custom_settings.py") assert "Spider closed (finished)" not in log @@ -562,6 +570,6 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): def test_reactorless_reactor(self): log = self.run_script("reactorless_reactor.py") assert ( - "RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed" + "RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed" in log ) diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py index 8f43d8071..361e91382 100644 --- a/tests/test_downloader_handler_twisted_ftp.py +++ b/tests/test_downloader_handler_twisted_ftp.py @@ -206,6 +206,6 @@ class TestAnonymousFTP(TestFTPBase): def test_not_configured_without_reactor() -> None: - crawler = Crawler(Spider, {"TWISTED_ENABLED": False}) + crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False}) with pytest.raises(NotConfigured): FTPDownloadHandler.from_crawler(crawler) diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index 5d093cc17..f6e86d3dc 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -36,7 +36,7 @@ class HTTP11DownloadHandlerMixin: def test_not_configured_without_reactor() -> None: - crawler = Crawler(Spider, {"TWISTED_ENABLED": False}) + crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False}) with pytest.raises(NotConfigured): HTTP11DownloadHandler.from_crawler(crawler) diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 7e2c06d6b..53f0210ff 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -52,7 +52,7 @@ class H2DownloadHandlerMixin: def test_not_configured_without_reactor() -> None: from scrapy.core.downloader.handlers.http2 import H2DownloadHandler # noqa: PLC0415 - crawler = Crawler(Spider, {"TWISTED_ENABLED": False}) + crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False}) with pytest.raises(NotConfigured): H2DownloadHandler.from_crawler(crawler)