diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 4dc04ab66..a56fa9577 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -161,12 +161,24 @@ class CrawlTestCase(TestCase): self.assertIsNotNone(record.exc_info) self.assertIs(record.exc_info[0], ZeroDivisionError) - @defer.inlineCallbacks - def test_start_requests_eagerness(self): + def is_eager(_, requests_in_order): """ All start requests(depth=0) are scheduled before any other requests(depth!=0) """ + depths_in_order = [r.meta.get('depth', 0) for r in requests_in_order] + order_of_start_requests = [ + o for o, d in enumerate(depths_in_order) if d == 0 + ] + order_of_other_requests = [ + o for o, d in enumerate(depths_in_order) if d != 0 + ] + last_start_request = max(order_of_start_requests) + first_other_request = min(order_of_other_requests) + return last_start_request < first_other_request + + @defer.inlineCallbacks + def test_start_requests_eagerness(self): class EagerSpider(YeldingRequestsSpider): def start_requests_with_control(self): yield from self.start_requests() @@ -186,17 +198,9 @@ class CrawlTestCase(TestCase): mockserver=self.mockserver, number_of_start_requests=100, ) - requests_in_order = crawler.spider.requests_in_order_of_scheduling - depths_in_order = [r.meta.get('depth', 0) for r in requests_in_order] - order_of_start_requests = [ - o for o, d in enumerate(depths_in_order) if d == 0 - ] - order_of_other_requests = [ - o for o, d in enumerate(depths_in_order) if d != 0 - ] - last_start_request = max(order_of_start_requests) - first_other_request = min(order_of_other_requests) - assert last_start_request < first_other_request + self.assertTrue( + self.is_eager(crawler.spider.requests_in_order_of_scheduling) + ) @defer.inlineCallbacks def test_start_requests_lazyness(self): @@ -218,17 +222,9 @@ class CrawlTestCase(TestCase): mockserver=self.mockserver, number_of_start_requests=100, ) - requests_in_order = crawler.spider.requests_in_order_of_scheduling - depths_in_order = [r.meta.get('depth', 0) for r in requests_in_order] - order_of_start_requests = [ - o for o, d in enumerate(depths_in_order) if d == 0 - ] - order_of_other_requests = [ - o for o, d in enumerate(depths_in_order) if d != 0 - ] - last_start_request = max(order_of_start_requests) - first_other_request = min(order_of_other_requests) - assert last_start_request > first_other_request + self.assertFalse( + self.is_eager(crawler.spider.requests_in_order_of_scheduling) + ) @defer.inlineCallbacks def test_start_requests_dupes(self):