From 0b8604bb5d8bffbbd5c78783022965fa2606c131 Mon Sep 17 00:00:00 2001
From: guillermo-bondonno <guillermo.bondonno@zyte.com>
Date: Tue, 9 Jul 2024 15:52:49 -0300
Subject: [PATCH] add CLOSESPIDER_PAGECOUNT_NO_ITEM to CloseSpider extension

---
 scrapy/extensions/closespider.py | 25 +++++++++++++++++++++++++
 tests/keys/mitmproxy-dhparam.pem | 14 ++++++++++++++
 tests/test_closespider.py        | 13 +++++++++++++
 3 files changed, 52 insertions(+)
 create mode 100644 tests/keys/mitmproxy-dhparam.pem

diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index 4627e7f98..6ebf98e65 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, DefaultDict, Dict
 
 from scrapy import Request, Spider, signals
 from scrapy.exceptions import NotConfigured
+from scrapy.signalmanager import dispatcher
 
 if TYPE_CHECKING:
     from twisted.python.failure import Failure
@@ -36,6 +37,9 @@ class CloseSpider:
             "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"),
             "errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"),
             "timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"),
+            "pagecount_no_item": crawler.settings.getint(
+                "CLOSESPIDER_PAGECOUNT_NO_ITEM"
+            ),
         }
 
         if not any(self.close_on.values()):
@@ -60,6 +64,19 @@ class CloseSpider:
             crawler.signals.connect(
                 self.item_scraped_no_item, signal=signals.item_scraped
             )
+        if self.close_on.get("pagecount_no_item"):
+            if self.page_count not in dispatcher.getReceivers(
+                signal=signals.response_received
+            ):
+                crawler.signals.connect(
+                    self.page_count, signal=signals.response_received
+                )
+
+            if self.item_scraped not in dispatcher.getReceivers(
+                signal=signals.item_scraped
+            ):
+                crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
+
         crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
 
     @classmethod
@@ -74,9 +91,16 @@ class CloseSpider:
 
     def page_count(self, response: Response, request: Request, spider: Spider) -> None:
         self.counter["pagecount"] += 1
+        self.counter["pagecount_since_last_item"] += 1
         if self.counter["pagecount"] == self.close_on["pagecount"]:
             assert self.crawler.engine
             self.crawler.engine.close_spider(spider, "closespider_pagecount")
+        if self.close_on["pagecount_no_item"] and (
+            self.counter["pagecount_since_last_item"]
+            >= self.close_on["pagecount_no_item"]
+        ):
+            assert self.crawler.engine
+            self.crawler.engine.close_spider(spider, "closespider_pagecount_no_item")
 
     def spider_opened(self, spider: Spider) -> None:
         from twisted.internet import reactor
@@ -91,6 +115,7 @@ class CloseSpider:
 
     def item_scraped(self, item: Any, spider: Spider) -> None:
         self.counter["itemcount"] += 1
+        self.counter["pagecount_since_last_item"] = 0
         if self.counter["itemcount"] == self.close_on["itemcount"]:
             assert self.crawler.engine
             self.crawler.engine.close_spider(spider, "closespider_itemcount")
diff --git a/tests/keys/mitmproxy-dhparam.pem b/tests/keys/mitmproxy-dhparam.pem
new file mode 100644
index 000000000..c10121fbf
--- /dev/null
+++ b/tests/keys/mitmproxy-dhparam.pem
@@ -0,0 +1,14 @@
+
+-----BEGIN DH PARAMETERS-----
+MIICCAKCAgEAyT6LzpwVFS3gryIo29J5icvgxCnCebcdSe/NHMkD8dKJf8suFCg3
+O2+dguLakSVif/t6dhImxInJk230HmfC8q93hdcg/j8rLGJYDKu3ik6H//BAHKIv
+j5O9yjU3rXCfmVJQic2Nne39sg3CreAepEts2TvYHhVv3TEAzEqCtOuTjgDv0ntJ
+Gwpj+BJBRQGG9NvprX1YGJ7WOFBP/hWU7d6tgvE6Xa7T/u9QIKpYHMIkcN/l3ZFB
+chZEqVlyrcngtSXCROTPcDOQ6Q8QzhaBJS+Z6rcsd7X+haiQqvoFcmaJ08Ks6LQC
+ZIL2EtYJw8V8z7C0igVEBIADZBI6OTbuuhDwRw//zU1uq52Oc48CIZlGxTYG/Evq
+o9EWAXUYVzWkDSTeBH1r4z/qLPE2cnhtMxbFxuvK53jGB0emy2y1Ei6IhKshJ5qX
+IB/aE7SSHyQ3MDHHkCmQJCsOd4Mo26YX61NZ+n501XjqpCBQ2+DfZCBh8Va2wDyv
+A2Ryg9SUz8j0AXViRNMJgJrr446yro/FuJZwnQcO3WQnXeqSBnURqKjmqkeFP+d8
+6mk2tqJaY507lRNqtGlLnj7f5RNoBFJDCLBNurVgfvq9TCVWKDIFD4vZRjCrnl6I
+rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI=
+-----END DH PARAMETERS-----
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 38ede70e4..caaa9f183 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -34,6 +34,19 @@ class TestCloseSpider(TestCase):
         pagecount = crawler.stats.get_value("response_received_count")
         self.assertTrue(pagecount >= close_on)
 
+    @defer.inlineCallbacks
+    def test_closespider_pagecount_no_item(self):
+        close_on = 5
+        crawler = get_crawler(
+            FollowAllSpider,
+            {"CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on},
+        )
+        yield crawler.crawl(mockserver=self.mockserver)
+        reason = crawler.spider.meta["close_reason"]
+        self.assertEqual(reason, "closespider_pagecount_no_item")
+        pagecount = crawler.stats.get_value("response_received_count")
+        self.assertTrue(pagecount >= close_on)
+
     @defer.inlineCallbacks
     def test_closespider_errorcount(self):
         close_on = 5