From e94d3ac173d2fcd269bdd8f06b6b1633953a225e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado=20Alonso?= Date: Tue, 29 Jun 2021 13:40:43 +0100 Subject: [PATCH] Expanded doc for idle signal --- docs/topics/signals.rst | 6 +++++- scrapy/core/engine.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index ee9d95836..530af1e37 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -269,7 +269,11 @@ spider_idle prevent the spider from being closed. Alternatively, you may raise a :exc:`~scrapy.exceptions.CloseSpider` - exception to provide a custom spider closing reason. + exception to provide a custom spider closing reason. An + idle handler is the perfect place to put some code that assesses + the final spider results and update the final closing reason + accordingly (e.g. setting it to 'too_few_results' instead of + 'finished'). This signal does not support returning deferreds from its handlers. diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 81a7a5068..0b34d213d 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -326,7 +326,7 @@ class ExecutionEngine: Called when a spider gets idle, i.e. when there are no remaining requests to download or schedule. It can be called multiple times. If a handler for the spider_idle signal raises a DontCloseSpider exception, the spider is not closed until the next loop and this function is guaranteed to be called - (at least) once again. A handler can raise CloseSpider to provide a custom closing reason + (at least) once again. A handler can raise CloseSpider to provide a custom closing reason. """ assert self.spider is not None # typing expected_ex = (DontCloseSpider, CloseSpider)