RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Foundations for the reactorless mode. (#7199) 

* Foundations for the reactorless mode.

* Add simple subprocess tests for reactorless AsyncCrawler*.

* More reactorless tests.

* Refactor AsyncCrawlerProcess.start().

* More checks.

* Fix test_reactorless_import_hook.

* More tests.

* Call install_reactor() before asyncio.run().

* Cleanup.

* Rephrase.

* Rephrasing.

* Set TELNETCONSOLE_ENABLED=False in the reactorless mode.
This commit is contained in:
Andrey Rakhmatullin 2026-01-31 19:48:54 +04:00 committed by GitHub
parent 49930dfec5
commit 4e1faf883d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 572 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
scrapy/crawler.py
View File

@ -34,9 +34,11 @@ from scrapy.utils.reactor import (
install_reactor,
is_asyncio_reactor_installed,
is_reactor_installed,
set_asyncio_event_loop,
verify_installed_asyncio_event_loop,
verify_installed_reactor,
)
from scrapy.utils.reactorless import install_reactor_import_hook
if TYPE_CHECKING:
from collections.abc import Awaitable, Generator, Iterable
@ -103,22 +105,27 @@ class Crawler:
self,
)
reactor_class: str = self.settings["TWISTED_REACTOR"]
event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"]
if self._init_reactor:
# this needs to be done after the spider settings are merged,
# but before something imports twisted.internet.reactor
use_reactor = self.settings.getbool("TWISTED_ENABLED")
if use_reactor:
reactor_class: str = self.settings["TWISTED_REACTOR"]
event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"]
if self._init_reactor:
# this needs to be done after the spider settings are merged,
# but before something imports twisted.internet.reactor
if reactor_class:
install_reactor(reactor_class, event_loop)
else:
from twisted.internet import reactor # noqa: F401
if reactor_class:
install_reactor(reactor_class, event_loop)
else:
from twisted.internet import reactor # noqa: F401
if reactor_class:
verify_installed_reactor(reactor_class)
if is_asyncio_reactor_installed() and event_loop:
verify_installed_asyncio_event_loop(event_loop)
verify_installed_reactor(reactor_class)
if is_asyncio_reactor_installed() and event_loop:
verify_installed_asyncio_event_loop(event_loop)
if self._init_reactor or reactor_class:
log_reactor_info()
if self._init_reactor or reactor_class:
log_reactor_info()
else:
logger.debug("Not using a Twisted reactor")
self._apply_reactorless_default_settings()
self.extensions = ExtensionManager.from_crawler(self)
self.settings.freeze()
@ -128,6 +135,15 @@ class Crawler:
"Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
)
def _apply_reactorless_default_settings(self):
"""Change some setting defaults when not using a Twisted reactor.
Some settings need different defaults when using and not using a
reactor, but as we can't put this logic into default_settings.py we
change them here when the reactor is not used.
"""
self.settings.set("TELNETCONSOLE_ENABLED", False, priority="default")
# Cannot use @deferred_f_from_coro_f because that relies on the reactor
# being installed already, which is done within _apply_settings(), inside
# this method.
@ -375,6 +391,10 @@ class CrawlerRunner(CrawlerRunnerBase):
def __init__(self, settings: dict[str, Any] | Settings | None = None):
super().__init__(settings)
if not self.settings.getbool("TWISTED_ENABLED"):
raise RuntimeError(
f"{type(self).__name__} doesn't support TWISTED_ENABLED=False."
)
self._active: set[Deferred[None]] = set()
def crawl(
@ -499,9 +519,16 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
"The crawler_or_spidercls argument cannot be a spider object, "
"it must be a spider class (or a Crawler object)"
)
if not is_asyncio_reactor_installed():
if self.settings.getbool("TWISTED_ENABLED"):
if not is_asyncio_reactor_installed():
raise RuntimeError(
f"When TWISTED_ENABLED is True, {type(self).__name__} "
f"requires that the installed Twisted reactor is "
f'"twisted.internet.asyncioreactor.AsyncioSelectorReactor".'
)
elif is_reactor_installed():
raise RuntimeError(
f"{type(self).__name__} requires AsyncioSelectorReactor."
"TWISTED_ENABLED is False but a Twisted reactor is installed."
)
crawler = self.create_crawler(crawler_or_spidercls)
return self._crawl(crawler, *args, **kwargs)
@ -730,7 +757,14 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
# The ASYNCIO_EVENT_LOOP setting cannot be overridden by add-ons and
# spiders when using AsyncCrawlerProcess.
loop_path = self.settings["ASYNCIO_EVENT_LOOP"]
if is_reactor_installed():
if not self.settings.getbool("TWISTED_ENABLED"):
if is_reactor_installed():
raise RuntimeError(
"TWISTED_ENABLED is False but a Twisted reactor is installed."
)
set_asyncio_event_loop(loop_path)
install_reactor_import_hook()
elif is_reactor_installed():
# The user could install a reactor before this class is instantiated.
# We need to make sure the reactor is the correct one and the loop
# type matches the setting.
@ -761,6 +795,33 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
:param bool install_signal_handlers: whether to install the OS signal
handlers from Twisted and Scrapy (default: True)
"""
if not self.settings.getbool("TWISTED_ENABLED"):
self._start_asyncio(stop_after_crawl, install_signal_handlers)
else:
self._start_twisted(stop_after_crawl, install_signal_handlers)
def _start_asyncio(
self, stop_after_crawl: bool, install_signal_handlers: bool
) -> None:
# Very basic and will need multiple improvements.
# TODO https://docs.python.org/3/library/asyncio-runner.html#handling-keyboard-interruption
# TODO various exception handling
# TODO consider asyncio.run()
loop = asyncio.get_event_loop()
if stop_after_crawl:
join_task = loop.create_task(self.join())
join_task.add_done_callback(lambda _: loop.stop())
try:
loop.run_forever() # blocking call
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
def _start_twisted(
self, stop_after_crawl: bool, install_signal_handlers: bool
) -> None:
from twisted.internet import reactor
if stop_after_crawl:

6
scrapy/extensions/telnet.py
View File

@ -44,6 +44,12 @@ class TelnetConsole(protocol.ServerFactory):
if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"):
raise NotConfigured
if not crawler.settings.getbool("TWISTED_ENABLED"):
raise NotConfigured(
"The TelnetConsole extension requires a Twisted reactor."
" You can set the TELNETCONSOLE_ENABLED setting to False to remove this warning."
)
self.crawler: Crawler = crawler
self.noisy: bool = False
self.portrange: list[int] = [

2
scrapy/settings/default_settings.py
View File

@ -184,6 +184,7 @@ __all__ = [
"TELNETCONSOLE_PORT",
"TELNETCONSOLE_USERNAME",
"TEMPLATES_DIR",
"TWISTED_ENABLED",
"TWISTED_REACTOR",
"URLLENGTH_LIMIT",
"USER_AGENT",
@ -522,6 +523,7 @@ TELNETCONSOLE_PASSWORD = None
TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve())
TWISTED_ENABLED = True
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
URLLENGTH_LIMIT = 2083

42
scrapy/utils/asyncio.py
View File

@ -35,15 +35,16 @@ def is_asyncio_available() -> bool:
.. versionadded:: 2.14
Currently this function is identical to
:func:`scrapy.utils.reactor.is_asyncio_reactor_installed`: it returns
``True`` if the Twisted reactor that is installed is
This function returns ``True`` if there is a running asyncio event loop. If
there is no such loop, it returns ``True`` if the Twisted reactor that is
installed is
:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, returns
``False`` if a different reactor is installed, and raises a
:exc:`RuntimeError` if no reactor is installed. In a future Scrapy version,
when Scrapy supports running without a Twisted reactor, this function will
also return ``True`` when running in that mode, so code that doesn't
directly require a Twisted reactor should use this function instead of
:exc:`RuntimeError` if no reactor is installed.
Code that doesn't directly require a Twisted reactor should use this
function while code that requires
:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` should use
:func:`~scrapy.utils.reactor.is_asyncio_reactor_installed`.
When this returns ``True``, an asyncio loop is installed and used by
@ -56,10 +57,35 @@ def is_asyncio_available() -> bool:
loop or await on :class:`asyncio.Future` objects in Scrapy-related code,
but it's possible to await on :class:`~twisted.internet.defer.Deferred`
objects.
.. note:: As this function uses :func:`asyncio.get_running_loop()`, it will
only detect the event loop if called in the same thread and from the
code that runs inside that loop (this shouldn't be a problem when
calling it from code such as spiders and Scrapy components, if Scrapy
is run using one of the supported ways).
.. versionchanged:: VERSION
This function now also returns ``True`` if there is a running asyncio
loop, even if no Twisted reactor is installed.
"""
# Check if there is a running asyncio loop.
# Can't easily check for an installed but not running one, and if we
# checked that there could be false positives due to some 3rd-party code
# installing it as a side effect (e.g. by calling get_event_loop()).
try:
asyncio.get_running_loop()
except RuntimeError:
pass
else:
return True
# Check if there is an installed asyncio reactor (it doesn't need to be
# running).
if not is_reactor_installed():
raise RuntimeError(
"is_asyncio_available() called without an installed reactor."
"is_asyncio_available() called without an installed reactor"
" or running asyncio loop."
)
return is_asyncio_reactor_installed()

53
scrapy/utils/reactorless.py Normal file
View File

@ -0,0 +1,53 @@
from __future__ import annotations
import sys
from importlib.abc import MetaPathFinder
from typing import TYPE_CHECKING
from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.reactor import is_reactor_installed
if TYPE_CHECKING:
from collections.abc import Sequence
from importlib.machinery import ModuleSpec
from types import ModuleType
def is_reactorless() -> bool:
"""Check if we are running in the reactorless mode, i.e. with
:setting:`TWISTED_ENABLED` set to ``False``.
As this checks the runtime state and not the setting itself, it can be
wrong when executed very early, before the reactor and/or the asyncio event
loop are initialized.
.. note:: As this function uses
:func:`scrapy.utils.asyncio.is_asyncio_available()`, it has the same
limitations for detecting a running asyncio event loop as that one.
.. versionadded:: VERSION
"""
return is_asyncio_available() and not is_reactor_installed()
class ReactorImportHook(MetaPathFinder):
"""Hook that prevents importing :mod:`twisted.internet.reactor`."""
def find_spec(
self,
fullname: str,
path: Sequence[str] | None,
target: ModuleType | None = None,
) -> ModuleSpec | None:
if fullname == "twisted.internet.reactor":
raise ImportError(
f"Import of {fullname} is forbidden when running without a Twisted reactor,"
f" as importing it installs the reactor, which can lead to unexpected behavior."
)
return None
def install_reactor_import_hook() -> None:
"""Prevent importing :mod:`twisted.internet.reactor`."""
sys.meta_path.insert(0, ReactorImportHook())

27
tests/AsyncCrawlerProcess/reactorless_datauri.py Normal file
View File

@ -0,0 +1,27 @@
from scrapy import Request, Spider
from scrapy.crawler import AsyncCrawlerProcess
class DataSpider(Spider):
name = "data"
async def start(self):
yield Request("data:,foo")
def parse(self, response):
return {"data": response.text}
process = AsyncCrawlerProcess(
settings={
"TWISTED_ENABLED": False,
"DOWNLOAD_HANDLERS": {
"http": None,
"https": None,
"ftp": None,
},
}
)
process.crawl(DataSpider)
process.start()

27
tests/AsyncCrawlerProcess/reactorless_import_hook.py Normal file
View File

@ -0,0 +1,27 @@
import scrapy
from scrapy.crawler import AsyncCrawlerProcess
class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def start(self):
import twisted.internet.reactor # noqa: F401
return
yield
process = AsyncCrawlerProcess(
settings={
"TWISTED_ENABLED": False,
"DOWNLOAD_HANDLERS": {
"http": None,
"https": None,
"ftp": None,
},
}
)
process.crawl(NoRequestsSpider)
process.start()

15
tests/AsyncCrawlerProcess/reactorless_reactor.py Normal file
View File

@ -0,0 +1,15 @@
from scrapy.crawler import AsyncCrawlerProcess
from scrapy.utils.reactor import install_reactor
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
AsyncCrawlerProcess(
settings={
"TWISTED_ENABLED": False,
"DOWNLOAD_HANDLERS": {
"http": None,
"https": None,
"ftp": None,
},
}
)

27
tests/AsyncCrawlerProcess/reactorless_simple.py Normal file
View File

@ -0,0 +1,27 @@
import scrapy
from scrapy.crawler import AsyncCrawlerProcess
from scrapy.utils.reactorless import is_reactorless
class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def start(self):
self.logger.info(f"is_reactorless(): {is_reactorless()}")
return
yield
process = AsyncCrawlerProcess(
settings={
"TWISTED_ENABLED": False,
"DOWNLOAD_HANDLERS": {
"http": None,
"https": None,
"ftp": None,
},
}
)
process.crawl(NoRequestsSpider)
process.start()

25
tests/AsyncCrawlerProcess/reactorless_telnetconsole_default.py Normal file
View File

@ -0,0 +1,25 @@
import scrapy
from scrapy.crawler import AsyncCrawlerProcess
class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def start(self):
return
yield
process = AsyncCrawlerProcess(
settings={
"TWISTED_ENABLED": False,
"DOWNLOAD_HANDLERS": {
"http": None,
"https": None,
"ftp": None,
},
}
)
process.crawl(NoRequestsSpider)
process.start()

26
tests/AsyncCrawlerProcess/reactorless_telnetconsole_disabled.py Normal file
View File

@ -0,0 +1,26 @@
import scrapy
from scrapy.crawler import AsyncCrawlerProcess
class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def start(self):
return
yield
process = AsyncCrawlerProcess(
settings={
"TWISTED_ENABLED": False,
"DOWNLOAD_HANDLERS": {
"http": None,
"https": None,
"ftp": None,
},
"TELNETCONSOLE_ENABLED": False,
}
)
process.crawl(NoRequestsSpider)
process.start()

26
tests/AsyncCrawlerProcess/reactorless_telnetconsole_enabled.py Normal file
View File

@ -0,0 +1,26 @@
import scrapy
from scrapy.crawler import AsyncCrawlerProcess
class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def start(self):
return
yield
process = AsyncCrawlerProcess(
settings={
"TWISTED_ENABLED": False,
"DOWNLOAD_HANDLERS": {
"http": None,
"https": None,
"ftp": None,
},
"TELNETCONSOLE_ENABLED": True,
}
)
process.crawl(NoRequestsSpider)
process.start()

2
tests/AsyncCrawlerProcess/simple.py
View File

@ -1,11 +1,13 @@
import scrapy
from scrapy.crawler import AsyncCrawlerProcess
from scrapy.utils.reactorless import is_reactorless
class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def start(self):
self.logger.info(f"is_reactorless(): {is_reactorless()}")
return
yield

33
tests/AsyncCrawlerRunner/reactorless_datauri.py Normal file
View File

@ -0,0 +1,33 @@
import asyncio
from scrapy import Request, Spider
from scrapy.crawler import AsyncCrawlerRunner
from scrapy.utils.log import configure_logging
class DataSpider(Spider):
name = "data"
async def start(self):
yield Request("data:,foo")
def parse(self, response):
return {"data": response.text}
async def main():
configure_logging()
runner = AsyncCrawlerRunner(
settings={
"TWISTED_ENABLED": False,
"DOWNLOAD_HANDLERS": {
"http": None,
"https": None,
"ftp": None,
},
}
)
await runner.crawl(DataSpider)
asyncio.run(main())

33
tests/AsyncCrawlerRunner/reactorless_reactor.py Normal file
View File

@ -0,0 +1,33 @@
import asyncio
from scrapy import Spider
from scrapy.crawler import AsyncCrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.reactor import install_reactor
class NoRequestsSpider(Spider):
name = "no_request"
async def start(self):
return
yield
async def main():
configure_logging()
runner = AsyncCrawlerRunner(
settings={
"TWISTED_ENABLED": False,
"DOWNLOAD_HANDLERS": {
"http": None,
"https": None,
"ftp": None,
},
}
)
await runner.crawl(NoRequestsSpider)
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
asyncio.run(main())

33
tests/AsyncCrawlerRunner/reactorless_simple.py Normal file
View File

@ -0,0 +1,33 @@
import asyncio
from scrapy import Spider
from scrapy.crawler import AsyncCrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.reactorless import is_reactorless
class NoRequestsSpider(Spider):
name = "no_request"
async def start(self):
self.logger.info(f"is_reactorless(): {is_reactorless()}")
return
yield
async def main():
configure_logging()
runner = AsyncCrawlerRunner(
settings={
"TWISTED_ENABLED": False,
"DOWNLOAD_HANDLERS": {
"http": None,
"https": None,
"ftp": None,
},
}
)
await runner.crawl(NoRequestsSpider)
asyncio.run(main())

2
tests/AsyncCrawlerRunner/simple.py
View File

@ -5,12 +5,14 @@ from scrapy.crawler import AsyncCrawlerRunner
from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.log import configure_logging
from scrapy.utils.reactor import install_reactor
from scrapy.utils.reactorless import is_reactorless
class NoRequestsSpider(Spider):
name = "no_request"
async def start(self):
self.logger.info(f"is_reactorless(): {is_reactorless()}")
return
yield

12
tests/CrawlerProcess/reactorless.py Normal file
View File

@ -0,0 +1,12 @@
from scrapy.crawler import CrawlerProcess
CrawlerProcess(
settings={
"TWISTED_ENABLED": False,
"DOWNLOAD_HANDLERS": {
"http": None,
"https": None,
"ftp": None,
},
}
)

2
tests/CrawlerProcess/simple.py
View File

@ -1,11 +1,13 @@
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.utils.reactorless import is_reactorless
class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def start(self):
self.logger.info(f"is_reactorless(): {is_reactorless()}")
return
yield

12
tests/CrawlerRunner/reactorless.py Normal file
View File

@ -0,0 +1,12 @@
from scrapy.crawler import CrawlerRunner
CrawlerRunner(
settings={
"TWISTED_ENABLED": False,
"DOWNLOAD_HANDLERS": {
"http": None,
"https": None,
"ftp": None,
},
}
)

2
tests/CrawlerRunner/simple.py
View File

@ -4,12 +4,14 @@ from scrapy import Spider
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.reactor import install_reactor
from scrapy.utils.reactorless import is_reactorless
class NoRequestsSpider(Spider):
name = "no_request"
async def start(self):
self.logger.info(f"is_reactorless(): {is_reactorless()}")
return
yield

96
tests/test_crawler.py
View File

@ -800,6 +800,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "is_reactorless(): False" in log
def test_multi(self):
log = self.run_script("multi.py")
@ -1042,6 +1043,12 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
in log
)
def test_reactorless(self):
log = self.run_script("reactorless.py")
assert (
"RuntimeError: CrawlerProcess doesn't support TWISTED_ENABLED=False" in log
)
class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
@property
@ -1076,6 +1083,59 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
"setting (uvloop.Loop)"
) in log
def test_reactorless_simple(self):
log = self.run_script("reactorless_simple.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "is_reactorless(): True" in log
assert "ERROR: " not in log
assert "WARNING: " not in log
def test_reactorless_datauri(self):
log = self.run_script("reactorless_datauri.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "{'data': 'foo'}" in log
assert "'item_scraped_count': 1" in log
assert "ERROR: " not in log
assert "WARNING: " not in log
def test_reactorless_import_hook(self):
log = self.run_script("reactorless_import_hook.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "ImportError: Import of twisted.internet.reactor is forbidden" in log
def test_reactorless_telnetconsole_default(self):
"""By default TWISTED_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False."""
log = self.run_script("reactorless_telnetconsole_default.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "The TelnetConsole extension requires a Twisted reactor" not in log
assert "scrapy.extensions.telnet.TelnetConsole" not in log
def test_reactorless_telnetconsole_disabled(self):
"""Explicit TELNETCONSOLE_ENABLED=False, there are no warnings."""
log = self.run_script("reactorless_telnetconsole_disabled.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "The TelnetConsole extension requires a Twisted reactor" not in log
assert "scrapy.extensions.telnet.TelnetConsole" not in log
def test_reactorless_telnetconsole_enabled(self):
"""Explicit TELNETCONSOLE_ENABLED=True, the user gets a warning."""
log = self.run_script("reactorless_telnetconsole_enabled.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "The TelnetConsole extension requires a Twisted reactor" in log
def test_reactorless_reactor(self):
log = self.run_script("reactorless_reactor.py")
assert (
"RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed"
in log
)
class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
"""Common tests between CrawlerRunner and AsyncCrawlerRunner,
@ -1089,6 +1149,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
"Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
in log
)
assert "is_reactorless(): False" in log
def test_multi_parallel(self):
log = self.run_script("multi_parallel.py")
@ -1164,6 +1225,12 @@ class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
)
assert "DEBUG: Using asyncio event loop" in log
def test_reactorless(self):
log = self.run_script("reactorless.py")
assert (
"RuntimeError: CrawlerRunner doesn't support TWISTED_ENABLED=False" in log
)
class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
@property
@ -1173,7 +1240,34 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
def test_simple_default_reactor(self):
log = self.run_script("simple_default_reactor.py")
assert "Spider closed (finished)" not in log
assert "RuntimeError: AsyncCrawlerRunner requires AsyncioSelectorReactor" in log
assert (
"RuntimeError: When TWISTED_ENABLED is True, "
"AsyncCrawlerRunner requires that the installed Twisted reactor"
) in log
def test_reactorless_simple(self):
log = self.run_script("reactorless_simple.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "is_reactorless(): True" in log
assert "ERROR: " not in log
assert "WARNING: " not in log
def test_reactorless_datauri(self):
log = self.run_script("reactorless_datauri.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "{'data': 'foo'}" in log
assert "'item_scraped_count': 1" in log
assert "ERROR: " not in log
assert "WARNING: " not in log
def test_reactorless_reactor(self):
log = self.run_script("reactorless_reactor.py")
assert (
"RuntimeError: TWISTED_ENABLED is False but a Twisted reactor is installed"
in log
)
@pytest.mark.parametrize(