From 6db0f1e914a49cfe02741f5df111b68ec1a767d8 Mon Sep 17 00:00:00 2001
From: Vostretsov Nikita <whalebot.helmsman@gmail.com>
Date: Tue, 30 Jun 2020 06:24:00 +0000
Subject: [PATCH] simple tests for lazyness and eagerness

---
 tests/middlewares.py | 25 ++++++++++++++++++
 tests/test_crawl.py  | 62 +++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 81 insertions(+), 6 deletions(-)
 create mode 100644 tests/middlewares.py

diff --git a/tests/middlewares.py b/tests/middlewares.py
new file mode 100644
index 000000000..983147642
--- /dev/null
+++ b/tests/middlewares.py
@@ -0,0 +1,25 @@
+from scrapy.http import Request
+
+
+class RequestInOrderMiddleware:
+    def process_spider_output(self, response, result, spider):
+        return (self._preserve_in_order(r, spider) for r in result or ())
+
+    def process_start_requests(self, start_requests, spider):
+        return (self._preserve_in_order(r, spider) for r in start_requests or ())
+
+    def _preserve_in_order(self, smth, spider):
+        self.__preserve_in_order(smth, spider)
+        return smth
+
+    def __preserve_in_order(self, smth, spider):
+
+        if not isinstance(smth, Request):
+            return
+
+        request = smth
+
+        if getattr(spider, 'requests_in_order_of_scheduling', None) is None:
+            setattr(spider, 'requests_in_order_of_scheduling', list())
+
+        spider.requests_in_order_of_scheduling.append(request)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index df920f2a2..ec59b07c6 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -161,13 +161,63 @@ class CrawlTestCase(TestCase):
         self.assertIs(record.exc_info[0], ZeroDivisionError)
 
     @defer.inlineCallbacks
-    def test_start_requests_lazyness(self):
-        settings = {"CONCURRENT_REQUESTS": 1}
-        crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider)
+    def test_start_requests_eagerness(self):
+        """
+            All start requests(depth=0) are scheduled before
+            any other requests(depth!=0)
+        """
+        class EagerSpider(BrokenStartRequestsSpider):
+            def start_requests_with_control(self):
+                yield from self.start_requests()
+
+        settings = {
+                "SPIDER_MIDDLEWARES": {
+                    "scrapy.spidermiddlewares.depth.DepthMiddleware" : 0,
+                    "tests.middlewares.RequestInOrderMiddleware" : 1,
+                    }
+                }
+        crawler = CrawlerRunner(settings).create_crawler(EagerSpider)
         yield crawler.crawl(mockserver=self.mockserver)
-        self.assertTrue(
-            crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
-            crawler.spider.seedsseen)
+        requests_in_order = crawler.spider.requests_in_order_of_scheduling
+        depths_in_order = [r.meta.get('depth', 0) for r in requests_in_order]
+        order_of_start_requests = [
+                o for o, d in enumerate(depths_in_order) if d == 0
+                ]
+        order_of_other_requests = [
+                o for o, d in enumerate(depths_in_order) if d != 0
+                ]
+        last_start_request = max(order_of_start_requests)
+        first_other_request = min(order_of_other_requests)
+        assert last_start_request < first_other_request
+
+    @defer.inlineCallbacks
+    def test_start_requests_lazyness(self):
+        """
+            lazyness as a negation of eagerness
+        """
+        class LazySpider(BrokenStartRequestsSpider):
+            pass
+
+
+        settings = {
+                "SPIDER_MIDDLEWARES": {
+                    "scrapy.spidermiddlewares.depth.DepthMiddleware" : 0,
+                    "tests.middlewares.RequestInOrderMiddleware" : 1,
+                    }
+                }
+        crawler = CrawlerRunner(settings).create_crawler(LazySpider)
+        yield crawler.crawl(mockserver=self.mockserver)
+        requests_in_order = crawler.spider.requests_in_order_of_scheduling
+        depths_in_order = [r.meta.get('depth', 0) for r in requests_in_order]
+        order_of_start_requests = [
+                o for o, d in enumerate(depths_in_order) if d == 0
+                ]
+        order_of_other_requests = [
+                o for o, d in enumerate(depths_in_order) if d != 0
+                ]
+        last_start_request = max(order_of_start_requests)
+        first_other_request = min(order_of_other_requests)
+        assert last_start_request > first_other_request
 
     @defer.inlineCallbacks
     def test_start_requests_dupes(self):