From 3ecbea4b876ed084f30bb4063e1993dd9c3cdb8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 2 Mar 2022 16:06:49 +0100 Subject: [PATCH 1/3] CrawlerProcess: initiate the reactor only once --- scrapy/crawler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index a638254f1..9939a19eb 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -290,6 +290,7 @@ class CrawlerProcess(CrawlerRunner): super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) + self._initiated_reactor = False def _signal_shutdown(self, signum, _): from twisted.internet import reactor @@ -310,7 +311,9 @@ class CrawlerProcess(CrawlerRunner): def _create_crawler(self, spidercls): if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings, init_reactor=True) + init_reactor = not self._initiated_reactor + self._initiated_reactor = True + return Crawler(spidercls, self.settings, init_reactor=init_reactor) def start(self, stop_after_crawl=True, install_signal_handlers=True): """ From 96fc4dae15181695c58040389fa502857a2b0df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 2 Mar 2022 16:14:39 +0100 Subject: [PATCH 2/3] CrawlerProcess: test a multi-spider scenario --- tests/CrawlerProcess/multi.py | 16 ++++++++++++++++ tests/test_crawler.py | 6 ++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/CrawlerProcess/multi.py diff --git a/tests/CrawlerProcess/multi.py b/tests/CrawlerProcess/multi.py new file mode 100644 index 000000000..aaa1af5c5 --- /dev/null +++ b/tests/CrawlerProcess/multi.py @@ -0,0 +1,16 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 8f6227109..957525382 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -302,6 +302,12 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn('Spider closed (finished)', log) self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + def test_multi(self): + log = self.run_script('multi.py') + self.assertIn('Spider closed (finished)', log) + self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + self.assertNotIn("ReactorAlreadyInstalledError", log) + def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) From 3bf6baeaa705ddc2d3417206f3db816dabadf5a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 2 Mar 2022 17:03:41 +0100 Subject: [PATCH 3/3] =?UTF-8?q?initiated=20=E2=86=92=20initialized?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/crawler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 9939a19eb..d669d93a8 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -290,7 +290,7 @@ class CrawlerProcess(CrawlerRunner): super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) - self._initiated_reactor = False + self._initialized_reactor = False def _signal_shutdown(self, signum, _): from twisted.internet import reactor @@ -311,8 +311,8 @@ class CrawlerProcess(CrawlerRunner): def _create_crawler(self, spidercls): if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - init_reactor = not self._initiated_reactor - self._initiated_reactor = True + init_reactor = not self._initialized_reactor + self._initialized_reactor = True return Crawler(spidercls, self.settings, init_reactor=init_reactor) def start(self, stop_after_crawl=True, install_signal_handlers=True):