Merge pull request #5191 from ivanprado/master

CloseSpider can be raised on spider_idle signal handler to set the closing reason

docs/topics/signals.rst

@@ -268,6 +268,13 @@ spider_idle
     You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to
     prevent the spider from being closed.
 
+    Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider`
+    exception to provide a custom spider closing reason. An
+    idle handler is the perfect place to put some code that assesses
+    the final spider results and update the final closing reason
+    accordingly (e.g. setting it to 'too_few_results' instead of
+    'finished').
+
     This signal does not support returning deferreds from its handlers.
 
     :param spider: the spider which has gone idle

scrapy/core/engine.py

@@ -15,7 +15,11 @@ from twisted.python.failure import Failure
 
 from scrapy import signals
 from scrapy.core.scraper import Scraper
-from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning
+from scrapy.exceptions import (
+    CloseSpider,
+    DontCloseSpider,
+    ScrapyDeprecationWarning,
+)
 from scrapy.http import Response, Request
 from scrapy.settings import BaseSettings
 from scrapy.spiders import Spider
@@ -325,14 +329,23 @@ class ExecutionEngine:
         Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule.
         It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider
         exception, the spider is not closed until the next loop and this function is guaranteed to be called
-        (at least) once again.
+        (at least) once again. A handler can raise CloseSpider to provide a custom closing reason.
         """
         assert self.spider is not None  # typing
-        res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=DontCloseSpider)
-        if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
+        expected_ex = (DontCloseSpider, CloseSpider)
+        res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex)
+        detected_ex = {
+            ex: x.value
+            for _, x in res
+            for ex in expected_ex
+            if isinstance(x, Failure) and isinstance(x.value, ex)
+        }
+        if DontCloseSpider in detected_ex:
             return None
         if self.spider_is_idle():
-            self.close_spider(self.spider, reason='finished')
+            ex = detected_ex.get(CloseSpider, CloseSpider(reason='finished'))
+            assert isinstance(ex, CloseSpider)  # typing
+            self.close_spider(self.spider, reason=ex.reason)
 
     def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred:
         """Close (cancel) spider and clear all its outstanding requests"""

scrapy/utils/signal.py

@@ -1,5 +1,5 @@
 """Helper functions for working with signals"""
-
+import collections
 import logging
 
 from twisted.internet.defer import DeferredList, Deferred
@@ -16,15 +16,13 @@ from scrapy.utils.log import failure_to_exc_info
 logger = logging.getLogger(__name__)
 
 
-class _IgnoredException(Exception):
-    pass
-
-
 def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
     """Like pydispatcher.robust.sendRobust but it also logs errors and returns
     Failures instead of exceptions.
     """
-    dont_log = (named.pop('dont_log', _IgnoredException), StopDownload)
+    dont_log = named.pop('dont_log', ())
+    dont_log = tuple(dont_log) if isinstance(dont_log, collections.Sequence) else (dont_log,)
+    dont_log += (StopDownload, )
     spider = named.get('spider', None)
     responses = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):

tests/test_engine.py

@@ -26,7 +26,7 @@ from twisted.web import server, static, util
 
 from scrapy import signals
 from scrapy.core.engine import ExecutionEngine
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.item import Item, Field
 from scrapy.linkextractors import LinkExtractor
@@ -113,6 +113,18 @@ class ItemZeroDivisionErrorSpider(TestSpider):
     }
 
 
+class ChangeCloseReasonSpider(TestSpider):
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        spider = cls(*args, **kwargs)
+        spider._set_crawler(crawler)
+        crawler.signals.connect(spider.spider_idle, signals.spider_idle)
+        return spider
+
+    def spider_idle(self):
+        raise CloseSpider(reason="custom_reason")
+
+
 def start_test_site(debug=False):
     root_dir = os.path.join(tests_datadir, "test_site")
     r = static.File(root_dir)
@@ -251,6 +263,13 @@ class EngineTest(unittest.TestCase):
         yield self.run.run()
         self._assert_items_error()
 
+    @defer.inlineCallbacks
+    def test_crawler_change_close_reason_on_idle(self):
+        self.run = CrawlerRun(ChangeCloseReasonSpider)
+        yield self.run.run()
+        self.assertEqual({'spider': self.run.spider, 'reason': 'custom_reason'},
+                         self.run.signals_caught[signals.spider_closed])
+
     def _assert_visited_urls(self):
         must_be_visited = ["/", "/redirect", "/redirected",
                            "/item1.html", "/item2.html", "/item999.html"]