simple tests for lazyness and eagerness

tests/middlewares.py

@@ -0,0 +1,25 @@
+from scrapy.http import Request
+
+
+class RequestInOrderMiddleware:
+    def process_spider_output(self, response, result, spider):
+        return (self._preserve_in_order(r, spider) for r in result or ())
+
+    def process_start_requests(self, start_requests, spider):
+        return (self._preserve_in_order(r, spider) for r in start_requests or ())
+
+    def _preserve_in_order(self, smth, spider):
+        self.__preserve_in_order(smth, spider)
+        return smth
+
+    def __preserve_in_order(self, smth, spider):
+
+        if not isinstance(smth, Request):
+            return
+
+        request = smth
+
+        if getattr(spider, 'requests_in_order_of_scheduling', None) is None:
+            setattr(spider, 'requests_in_order_of_scheduling', list())
+
+        spider.requests_in_order_of_scheduling.append(request)

tests/test_crawl.py

@@ -161,13 +161,63 @@ class CrawlTestCase(TestCase):
         self.assertIs(record.exc_info[0], ZeroDivisionError)
 
     @defer.inlineCallbacks
-    def test_start_requests_lazyness(self):
-        settings = {"CONCURRENT_REQUESTS": 1}
-        crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider)
+    def test_start_requests_eagerness(self):
+        """
+            All start requests(depth=0) are scheduled before
+            any other requests(depth!=0)
+        """
+        class EagerSpider(BrokenStartRequestsSpider):
+            def start_requests_with_control(self):
+                yield from self.start_requests()
+
+        settings = {
+                "SPIDER_MIDDLEWARES": {
+                    "scrapy.spidermiddlewares.depth.DepthMiddleware" : 0,
+                    "tests.middlewares.RequestInOrderMiddleware" : 1,
+                    }
+                }
+        crawler = CrawlerRunner(settings).create_crawler(EagerSpider)
         yield crawler.crawl(mockserver=self.mockserver)
-        self.assertTrue(
-            crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
-            crawler.spider.seedsseen)
+        requests_in_order = crawler.spider.requests_in_order_of_scheduling
+        depths_in_order = [r.meta.get('depth', 0) for r in requests_in_order]
+        order_of_start_requests = [
+                o for o, d in enumerate(depths_in_order) if d == 0
+                ]
+        order_of_other_requests = [
+                o for o, d in enumerate(depths_in_order) if d != 0
+                ]
+        last_start_request = max(order_of_start_requests)
+        first_other_request = min(order_of_other_requests)
+        assert last_start_request < first_other_request
+
+    @defer.inlineCallbacks
+    def test_start_requests_lazyness(self):
+        """
+            lazyness as a negation of eagerness
+        """
+        class LazySpider(BrokenStartRequestsSpider):
+            pass
+
+
+        settings = {
+                "SPIDER_MIDDLEWARES": {
+                    "scrapy.spidermiddlewares.depth.DepthMiddleware" : 0,
+                    "tests.middlewares.RequestInOrderMiddleware" : 1,
+                    }
+                }
+        crawler = CrawlerRunner(settings).create_crawler(LazySpider)
+        yield crawler.crawl(mockserver=self.mockserver)
+        requests_in_order = crawler.spider.requests_in_order_of_scheduling
+        depths_in_order = [r.meta.get('depth', 0) for r in requests_in_order]
+        order_of_start_requests = [
+                o for o, d in enumerate(depths_in_order) if d == 0
+                ]
+        order_of_other_requests = [
+                o for o, d in enumerate(depths_in_order) if d != 0
+                ]
+        last_start_request = max(order_of_start_requests)
+        first_other_request = min(order_of_other_requests)
+        assert last_start_request > first_other_request
 
     @defer.inlineCallbacks
     def test_start_requests_dupes(self):