diff --git a/tests/spiders.py b/tests/spiders.py index d1998ca69..743811893 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -283,15 +283,19 @@ class ItemSpider(FollowAllSpider): yield {} -class MaxItemsKeepCrawlingSpider(FollowAllSpider): - def __init__(self, max_items=10, *args, **kwargs): +class MaxItemsAndRequestsSpider(FollowAllSpider): + def __init__(self, max_items=10, max_requests=10, *args, **kwargs): super().__init__(*args, **kwargs) self.max_items = max_items - self.items_scraped = 0 + self.max_requests = max_requests def parse(self, response): + self.items_scraped = 0 + self.pages_crawled = 1 # account for the start url for request in super().parse(response): - yield request + if self.pages_crawled < self.max_requests: + yield request + self.pages_crawled += 1 if self.items_scraped < self.max_items: yield Item() self.items_scraped += 1 diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 50b483a74..9810d10fb 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -7,7 +7,7 @@ from tests.spiders import ( ErrorSpider, FollowAllSpider, ItemSpider, - MaxItemsKeepCrawlingSpider, + MaxItemsAndRequestsSpider, SlowSpider, ) @@ -44,20 +44,21 @@ class TestCloseSpider(TestCase): def test_closespider_pagecount_no_item(self): close_on = 5 max_items = 5 + max_requests = close_on + max_items crawler = get_crawler( - MaxItemsKeepCrawlingSpider, + MaxItemsAndRequestsSpider, { "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on, }, ) - yield crawler.crawl(max_items=max_items, mockserver=self.mockserver) + yield crawler.crawl( + max_items=max_items, max_requests=max_requests, mockserver=self.mockserver + ) reason = crawler.spider.meta["close_reason"] self.assertEqual(reason, "closespider_pagecount_no_item") pagecount = crawler.stats.get_value("response_received_count") itemcount = crawler.stats.get_value("item_scraped_count") - self.assertEqual(itemcount, max_items) self.assertLessEqual(pagecount, close_on + itemcount) - self.assertGreater(pagecount, itemcount) @defer.inlineCallbacks def test_closespider_errorcount(self):