From f35970778b93033ee36379e0b8b55035aacaf918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= Date: Tue, 29 Jun 2021 13:21:38 +0100 Subject: [PATCH] Test case for raising CloseSpider on spider idle signal handler --- tests/test_engine.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index c200ded90..dc24f50fa 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -26,7 +26,7 @@ from twisted.web import server, static, util from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.exceptions import ScrapyDeprecationWarning, CloseSpider from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor @@ -113,6 +113,18 @@ class ItemZeroDivisionErrorSpider(TestSpider): } +class ChangeCloseReasonSpider(TestSpider): + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = cls(*args, **kwargs) + spider._set_crawler(crawler) + crawler.signals.connect(spider.spider_idle, signals.spider_idle) + return spider + + def spider_idle(self): + raise CloseSpider(reason="custom_reason") + + def start_test_site(debug=False): root_dir = os.path.join(tests_datadir, "test_site") r = static.File(root_dir) @@ -251,6 +263,13 @@ class EngineTest(unittest.TestCase): yield self.run.run() self._assert_items_error() + @defer.inlineCallbacks + def test_crawler_change_close_reason_on_idle(self): + self.run = CrawlerRun(ChangeCloseReasonSpider) + yield self.run.run() + self.assertEqual({'spider': self.run.spider, 'reason': 'custom_reason'}, + self.run.signals_caught[signals.spider_closed]) + def _assert_visited_urls(self): must_be_visited = ["/", "/redirect", "/redirected", "/item1.html", "/item2.html", "/item999.html"]