Add change reactor test to CrawlerRunner

This commit is contained in:
Laerte Pereira 2024-05-26 19:55:54 -03:00
parent 62c89aaf05
commit 2facdd4fb0
4 changed files with 42 additions and 0 deletions

View File

@ -9,6 +9,7 @@ exclude =
per-file-ignores =
# Exclude files that are meant to provide top-level imports
# E402: Module level import not at top of file
tests/CrawlerRunner/change_reactor.py:E402
# F401: Module imported but unused
scrapy/__init__.py:E402
scrapy/core/downloader/handlers/http.py:F401

View File

@ -129,6 +129,8 @@ class Crawler:
if is_asyncio_reactor_installed() and event_loop:
verify_installed_asyncio_event_loop(event_loop)
log_reactor_info()
self.extensions = ExtensionManager.from_crawler(self)
self.settings.freeze()

View File

@ -0,0 +1,31 @@
from scrapy import Spider
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
class NoRequestsSpider(Spider):
name = "no_request"
custom_settings = {
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
}
def start_requests(self):
return []
configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"})
from scrapy.utils.reactor import install_reactor
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
runner = CrawlerRunner()
d = runner.crawl(NoRequestsSpider)
from twisted.internet import reactor
d.addBoth(callback=lambda _: reactor.stop())
reactor.run()

View File

@ -926,3 +926,11 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
self.assertIn("INFO: Host: not.a.real.domain", log)
self.assertIn("INFO: Type: <class 'ipaddress.IPv4Address'>", log)
self.assertIn("INFO: IP address: 127.0.0.1", log)
def test_change_default_reactor(self):
log = self.run_script("change_reactor.py")
self.assertIn(
"DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor",
log,
)
self.assertIn("DEBUG: Using asyncio event loop", log)