Merge pull request #5979 from BurnzZ/closespider-timeout-no-item

docs/topics/extensions.rst

@@ -258,6 +258,7 @@ The conditions for closing a spider can be configured through the following
 settings:
 
 * :setting:`CLOSESPIDER_TIMEOUT`
+* :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`
 * :setting:`CLOSESPIDER_ITEMCOUNT`
 * :setting:`CLOSESPIDER_PAGECOUNT`
 * :setting:`CLOSESPIDER_ERRORCOUNT`
@@ -280,6 +281,18 @@ more than that number of second, it will be automatically closed with the
 reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by
 timeout.
 
+.. setting:: CLOSESPIDER_TIMEOUT_NO_ITEM
+
+CLOSESPIDER_TIMEOUT_NO_ITEM
+"""""""""""""""""""""""""""
+
+Default: ``0``
+
+An integer which specifies a number of seconds. If the spider has not produced
+any items in the last number of seconds, it will be closed with the reason
+``closespider_timeout_no_item``. If zero (or non set), spiders won't be closed
+regardless if it hasn't produced any items.
+
 .. setting:: CLOSESPIDER_ITEMCOUNT
 
 CLOSESPIDER_ITEMCOUNT

scrapy/extensions/closespider.py

@@ -4,11 +4,14 @@ conditions are met.
 See documentation in docs/topics/extensions.rst
 """
 
+import logging
 from collections import defaultdict
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
 
+logger = logging.getLogger(__name__)
+
 
 class CloseSpider:
     def __init__(self, crawler):
@@ -19,6 +22,7 @@ class CloseSpider:
             "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"),
             "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"),
             "errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"),
+            "timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"),
         }
 
         if not any(self.close_on.values()):
@@ -34,6 +38,15 @@ class CloseSpider:
             crawler.signals.connect(self.spider_opened, signal=signals.spider_opened)
         if self.close_on.get("itemcount"):
             crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
+        if self.close_on.get("timeout_no_item"):
+            self.timeout_no_item = self.close_on["timeout_no_item"]
+            self.items_in_period = 0
+            crawler.signals.connect(
+                self.spider_opened_no_item, signal=signals.spider_opened
+            )
+            crawler.signals.connect(
+                self.item_scraped_no_item, signal=signals.item_scraped
+            )
         crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
 
     @classmethod
@@ -69,3 +82,31 @@ class CloseSpider:
         task = getattr(self, "task", False)
         if task and task.active():
             task.cancel()
+
+        task_no_item = getattr(self, "task_no_item", False)
+        if task_no_item and task_no_item.running:
+            task_no_item.stop()
+
+    def spider_opened_no_item(self, spider):
+        from twisted.internet import task
+
+        self.task_no_item = task.LoopingCall(self._count_items_produced, spider)
+        self.task_no_item.start(self.timeout_no_item, now=False)
+
+        logger.info(
+            f"Spider will stop when no items are produced after "
+            f"{self.timeout_no_item} seconds."
+        )
+
+    def item_scraped_no_item(self, item, spider):
+        self.items_in_period += 1
+
+    def _count_items_produced(self, spider):
+        if self.items_in_period >= 1:
+            self.items_in_period = 0
+        else:
+            logger.info(
+                f"Closing spider since no items were produced in the last "
+                f"{self.timeout_no_item} seconds."
+            )
+            self.crawler.engine.close_spider(spider, "closespider_timeout_no_item")

tests/spiders.py

@@ -77,6 +77,22 @@ class DelaySpider(MetaSpider):
         self.t2_err = time.time()
 
 
+class SlowSpider(DelaySpider):
+    name = "slow"
+
+    def start_requests(self):
+        # 1st response is fast
+        url = self.mockserver.url("/delay?n=0&b=0")
+        yield Request(url, callback=self.parse, errback=self.errback)
+
+        # 2nd response is slow
+        url = self.mockserver.url(f"/delay?n={self.n}&b={self.b}")
+        yield Request(url, callback=self.parse, errback=self.errback)
+
+    def parse(self, response):
+        yield Item()
+
+
 class SimpleSpider(MetaSpider):
     name = "simple"
 

tests/test_closespider.py

@@ -3,7 +3,7 @@ from twisted.trial.unittest import TestCase
 
 from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
-from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider
+from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider, SlowSpider
 
 
 class TestCloseSpider(TestCase):
@@ -54,3 +54,13 @@ class TestCloseSpider(TestCase):
         self.assertEqual(reason, "closespider_timeout")
         total_seconds = crawler.stats.get_value("elapsed_time_seconds")
         self.assertTrue(total_seconds >= close_on)
+
+    @defer.inlineCallbacks
+    def test_closespider_timeout_no_item(self):
+        timeout = 1
+        crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout})
+        yield crawler.crawl(n=3, mockserver=self.mockserver)
+        reason = crawler.spider.meta["close_reason"]
+        self.assertEqual(reason, "closespider_timeout_no_item")
+        total_seconds = crawler.stats.get_value("elapsed_time_seconds")
+        self.assertTrue(total_seconds >= timeout)