separate yielding from brokenness

tests/spiders.py

@@ -177,33 +177,29 @@ class ErrorSpider(FollowAllSpider):
             self.raise_exception()
 
 
+class YeldingRequestsSpider(FollowAllSpider):
+    number_of_start_requests = 10
+
+    def start_requests(self):
+        for s in range(self.number_of_start_requests):
+            qargs = {'total': 10, 'seed': s}
+            url = self.mockserver.url("/follow?%s") % urlencode(qargs, doseq=1)
+            yield Request(url, meta={'seed': s})
+
+
 class BrokenStartRequestsSpider(FollowAllSpider):
-
-    fail_before_yield = False
-    fail_yielding = False
-
-    def __init__(self, *a, **kw):
-        super(BrokenStartRequestsSpider, self).__init__(*a, **kw)
-        self.seedsseen = []
+    fail_before_yield = True
+    fail_yielding = True
 
     def start_requests(self):
         if self.fail_before_yield:
             1 / 0
 
-        for s in range(100):
-            qargs = {'total': 10, 'seed': s}
-            url = self.mockserver.url("/follow?%s") % urlencode(qargs, doseq=1)
-            yield Request(url, meta={'seed': s})
+        for r in super().start_requests():
+            yield r
             if self.fail_yielding:
                 2 / 0
 
-        assert self.seedsseen, 'All start requests consumed before any download happened'
-
-    def parse(self, response):
-        self.seedsseen.append(response.meta.get('seed'))
-        for req in super(BrokenStartRequestsSpider, self).parse(response):
-            yield req
-
 
 class SingleRequestSpider(MetaSpider):
 

tests/test_crawl.py

@@ -34,6 +34,7 @@ from tests.spiders import (
     FollowAllSpider,
     SimpleSpider,
     SingleRequestSpider,
+    YeldingRequestsSpider,
 )
 
 
@@ -166,7 +167,7 @@ class CrawlTestCase(TestCase):
             All start requests(depth=0) are scheduled before
             any other requests(depth!=0)
         """
-        class EagerSpider(BrokenStartRequestsSpider):
+        class EagerSpider(YeldingRequestsSpider):
             def start_requests_with_control(self):
                 yield from self.start_requests()
 
@@ -177,7 +178,14 @@ class CrawlTestCase(TestCase):
                     }
                 }
         crawler = CrawlerRunner(settings).create_crawler(EagerSpider)
-        yield crawler.crawl(mockserver=self.mockserver)
+        """
+            number_of_start_requests should be big enough, so scheduling such amount
+            of requests takes longer than crawling first of them
+        """
+        yield crawler.crawl(
+                mockserver=self.mockserver,
+                number_of_start_requests=100,
+                )
         requests_in_order = crawler.spider.requests_in_order_of_scheduling
         depths_in_order = [r.meta.get('depth', 0) for r in requests_in_order]
         order_of_start_requests = [
@@ -195,18 +203,21 @@ class CrawlTestCase(TestCase):
         """
             lazyness as a negation of eagerness
         """
-        class LazySpider(BrokenStartRequestsSpider):
-            pass
-
-
         settings = {
                 "SPIDER_MIDDLEWARES": {
                     "scrapy.spidermiddlewares.depth.DepthMiddleware" : 0,
                     "tests.middlewares.RequestInOrderMiddleware" : 1,
                     }
                 }
-        crawler = CrawlerRunner(settings).create_crawler(LazySpider)
-        yield crawler.crawl(mockserver=self.mockserver)
+        crawler = CrawlerRunner(settings).create_crawler(YeldingRequestsSpider)
+        """
+            number_of_start_requests should be big enough, so scheduling such amount
+            of requests takes longer than crawling first of them
+        """
+        yield crawler.crawl(
+                mockserver=self.mockserver,
+                number_of_start_requests=100,
+                )
         requests_in_order = crawler.spider.requests_in_order_of_scheduling
         depths_in_order = [r.meta.get('depth', 0) for r in requests_in_order]
         order_of_start_requests = [