From 2facdd4fb08ec3edaf1752047dd86d5b565621a1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Sun, 26 May 2024 19:55:54 -0300 Subject: [PATCH] Add change reactor test to CrawlerRunner --- .flake8 | 1 + scrapy/crawler.py | 2 ++ tests/CrawlerRunner/change_reactor.py | 31 +++++++++++++++++++++++++++ tests/test_crawler.py | 8 +++++++ 4 files changed, 42 insertions(+) create mode 100644 tests/CrawlerRunner/change_reactor.py diff --git a/.flake8 b/.flake8 index 62ccad9cf..0e43b9b56 100644 --- a/.flake8 +++ b/.flake8 @@ -9,6 +9,7 @@ exclude = per-file-ignores = # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file + tests/CrawlerRunner/change_reactor.py:E402 # F401: Module imported but unused scrapy/__init__.py:E402 scrapy/core/downloader/handlers/http.py:F401 diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ccfe78891..4fe5987a7 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -129,6 +129,8 @@ class Crawler: if is_asyncio_reactor_installed() and event_loop: verify_installed_asyncio_event_loop(event_loop) + log_reactor_info() + self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py new file mode 100644 index 000000000..b20aa0c7c --- /dev/null +++ b/tests/CrawlerRunner/change_reactor.py @@ -0,0 +1,31 @@ +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + def start_requests(self): + return [] + + +configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"}) + + +from scrapy.utils.reactor import install_reactor + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + +runner = CrawlerRunner() + +d = runner.crawl(NoRequestsSpider) + +from twisted.internet import reactor + +d.addBoth(callback=lambda _: reactor.stop()) +reactor.run() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 989208694..791ea1faa 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -926,3 +926,11 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("INFO: Host: not.a.real.domain", log) self.assertIn("INFO: Type: ", log) self.assertIn("INFO: IP address: 127.0.0.1", log) + + def test_change_default_reactor(self): + log = self.run_script("change_reactor.py") + self.assertIn( + "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", + log, + ) + self.assertIn("DEBUG: Using asyncio event loop", log)