improve test_closespider_pagecount_no_item and MaxItemsSpider

tests/spiders.py

@@ -283,21 +283,18 @@ class ItemSpider(FollowAllSpider):
             yield {}
 
 
-class MaxItemsSpider(ItemSpider):
+class MaxItemsKeepCrawlingSpider(FollowAllSpider):
     def __init__(self, max_items=10, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.max_items = max_items
         self.items_scraped = 0
 
     def parse(self, response):
-        for item_or_req in super().parse(response):
-            if isinstance(item_or_req, Request):
-                yield item_or_req
-            else:
-                if self.items_scraped >= self.max_items:
-                    continue
+        for request in super().parse(response):
+            yield request
+            if self.items_scraped < self.max_items:
+                yield Item()
                 self.items_scraped += 1
-                yield item_or_req
 
 
 class DefaultError(Exception):

tests/test_closespider.py

@@ -7,7 +7,7 @@ from tests.spiders import (
     ErrorSpider,
     FollowAllSpider,
     ItemSpider,
-    MaxItemsSpider,
+    MaxItemsKeepCrawlingSpider,
     SlowSpider,
 )
 
@@ -43,13 +43,11 @@ class TestCloseSpider(TestCase):
     @defer.inlineCallbacks
     def test_closespider_pagecount_no_item(self):
         close_on = 5
-        close_on_pagecount = 20
         max_items = 5
         crawler = get_crawler(
-            MaxItemsSpider,
+            MaxItemsKeepCrawlingSpider,
             {
                 "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on,
-                "CLOSESPIDER_PAGECOUNT": close_on_pagecount,
             },
         )
         yield crawler.crawl(max_items=max_items, mockserver=self.mockserver)
@@ -58,8 +56,8 @@ class TestCloseSpider(TestCase):
         pagecount = crawler.stats.get_value("response_received_count")
         itemcount = crawler.stats.get_value("item_scraped_count")
         self.assertEqual(itemcount, max_items)
-        self.assertLess(pagecount, close_on_pagecount)
-        self.assertTrue((pagecount - itemcount) >= close_on)
+        self.assertLessEqual(pagecount, close_on + itemcount)
+        self.assertGreater(pagecount, itemcount)
 
     @defer.inlineCallbacks
     def test_closespider_errorcount(self):