CloseSpider can be raised on spider_idle signal handler

docs/topics/signals.rst

@@ -268,6 +268,9 @@ spider_idle
     You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to
     prevent the spider from being closed.
 
+    Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider`
+    exception to provide a custom spider closing reason.
+
     This signal does not support returning deferreds from its handlers.
 
     :param spider: the spider which has gone idle

scrapy/core/engine.py

@@ -15,7 +15,8 @@ from twisted.python.failure import Failure
 
 from scrapy import signals
 from scrapy.core.scraper import Scraper
-from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning
+from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning, \
+    CloseSpider
 from scrapy.http import Response, Request
 from scrapy.settings import BaseSettings
 from scrapy.spiders import Spider
@@ -325,14 +326,19 @@ class ExecutionEngine:
         Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule.
         It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider
         exception, the spider is not closed until the next loop and this function is guaranteed to be called
-        (at least) once again.
+        (at least) once again. A handler can raise CloseSpider to provide a custom closing reason
         """
         assert self.spider is not None  # typing
-        res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=DontCloseSpider)
-        if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
+        expected_ex = (DontCloseSpider, CloseSpider)
+        res = self.signals.send_catch_log(signals.spider_idle, spider=self.spider, dont_log=expected_ex)
+        detected_ex = {ex: x.value
+                       for _, x in res for ex in expected_ex
+                       if isinstance(x, Failure) and isinstance(x.value, ex)}
+        if DontCloseSpider in detected_ex:
             return None
         if self.spider_is_idle():
-            self.close_spider(self.spider, reason='finished')
+            reason = detected_ex[CloseSpider].reason if CloseSpider in detected_ex else 'finished'
+            self.close_spider(self.spider, reason=reason)
 
     def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred:
         """Close (cancel) spider and clear all its outstanding requests"""

scrapy/utils/signal.py

@@ -1,5 +1,5 @@
 """Helper functions for working with signals"""
-
+import collections
 import logging
 
 from twisted.internet.defer import DeferredList, Deferred
@@ -16,15 +16,13 @@ from scrapy.utils.log import failure_to_exc_info
 logger = logging.getLogger(__name__)
 
 
-class _IgnoredException(Exception):
-    pass
-
-
 def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
     """Like pydispatcher.robust.sendRobust but it also logs errors and returns
     Failures instead of exceptions.
     """
-    dont_log = (named.pop('dont_log', _IgnoredException), StopDownload)
+    dont_log = named.pop('dont_log', ())
+    dont_log = tuple(dont_log) if isinstance(dont_log, collections.Sequence) else (dont_log,)
+    dont_log += (StopDownload, )
     spider = named.get('spider', None)
     responses = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):