Merge back tests/py36/_test_crawl.py.

conftest.py

@@ -14,8 +14,6 @@ collect_ignore = [
     *_py_files("tests/CrawlerProcess"),
     # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
     *_py_files("tests/CrawlerRunner"),
-    # Py36-only parts of respective tests
-    *_py_files("tests/py36"),
 ]
 
 for line in open('tests/ignores.txt'):

tests/py36/_test_crawl.py

@@ -1,57 +0,0 @@
-import asyncio
-
-from scrapy import Request
-from tests.spiders import SimpleSpider
-
-
-class AsyncDefAsyncioGenSpider(SimpleSpider):
-
-    name = 'asyncdef_asyncio_gen'
-
-    async def parse(self, response):
-        await asyncio.sleep(0.2)
-        yield {'foo': 42}
-        self.logger.info("Got response %d" % response.status)
-
-
-class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
-
-    name = 'asyncdef_asyncio_gen_loop'
-
-    async def parse(self, response):
-        for i in range(10):
-            await asyncio.sleep(0.1)
-            yield {'foo': i}
-        self.logger.info("Got response %d" % response.status)
-
-
-class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
-
-    name = 'asyncdef_asyncio_gen_complex'
-    initial_reqs = 4
-    following_reqs = 3
-    depth = 2
-
-    def _get_req(self, index, cb=None):
-        return Request(self.mockserver.url("/status?n=200&request=%d" % index),
-                       meta={'index': index},
-                       dont_filter=True,
-                       callback=cb)
-
-    def start_requests(self):
-        for i in range(1, self.initial_reqs + 1):
-            yield self._get_req(i)
-
-    async def parse(self, response):
-        index = response.meta['index']
-        yield {'index': index}
-        if index < 10 ** self.depth:
-            for new_index in range(10 * index, 10 * index + self.following_reqs):
-                yield self._get_req(new_index)
-        yield self._get_req(index, cb=self.parse2)
-        await asyncio.sleep(0.1)
-        yield {'index': index + 5}
-
-    async def parse2(self, response):
-        await asyncio.sleep(0.1)
-        yield {'index2': response.meta['index']}

tests/spiders.py

@@ -148,6 +148,59 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
         return reqs
 
 
+class AsyncDefAsyncioGenSpider(SimpleSpider):
+
+    name = 'asyncdef_asyncio_gen'
+
+    async def parse(self, response):
+        await asyncio.sleep(0.2)
+        yield {'foo': 42}
+        self.logger.info("Got response %d" % response.status)
+
+
+class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
+
+    name = 'asyncdef_asyncio_gen_loop'
+
+    async def parse(self, response):
+        for i in range(10):
+            await asyncio.sleep(0.1)
+            yield {'foo': i}
+        self.logger.info("Got response %d" % response.status)
+
+
+class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
+
+    name = 'asyncdef_asyncio_gen_complex'
+    initial_reqs = 4
+    following_reqs = 3
+    depth = 2
+
+    def _get_req(self, index, cb=None):
+        return Request(self.mockserver.url("/status?n=200&request=%d" % index),
+                       meta={'index': index},
+                       dont_filter=True,
+                       callback=cb)
+
+    def start_requests(self):
+        for i in range(1, self.initial_reqs + 1):
+            yield self._get_req(i)
+
+    async def parse(self, response):
+        index = response.meta['index']
+        yield {'index': index}
+        if index < 10 ** self.depth:
+            for new_index in range(10 * index, 10 * index + self.following_reqs):
+                yield self._get_req(new_index)
+        yield self._get_req(index, cb=self.parse2)
+        await asyncio.sleep(0.1)
+        yield {'index': index + 5}
+
+    async def parse2(self, response):
+        await asyncio.sleep(0.1)
+        yield {'index2': response.meta['index']}
+
+
 class ItemSpider(FollowAllSpider):
 
     name = 'item'

tests/test_crawl.py

@@ -19,6 +19,9 @@ from scrapy.http.response import Response
 from scrapy.utils.python import to_unicode
 from tests.mockserver import MockServer
 from tests.spiders import (
+    AsyncDefAsyncioGenComplexSpider,
+    AsyncDefAsyncioGenLoopSpider,
+    AsyncDefAsyncioGenSpider,
     AsyncDefAsyncioReqsReturnSpider,
     AsyncDefAsyncioReturnSingleElementSpider,
     AsyncDefAsyncioReturnSpider,
@@ -407,7 +410,6 @@ class CrawlSpiderTestCase(TestCase):
     @mark.only_asyncio()
     @defer.inlineCallbacks
     def test_async_def_asyncgen_parse(self):
-        from tests.py36._test_crawl import AsyncDefAsyncioGenSpider
         crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider)
         with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
@@ -423,7 +425,6 @@ class CrawlSpiderTestCase(TestCase):
         def _on_item_scraped(item):
             items.append(item)
 
-        from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpider
         crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider)
         crawler.signals.connect(_on_item_scraped, signals.item_scraped)
         with LogCapture() as log:
@@ -442,7 +443,6 @@ class CrawlSpiderTestCase(TestCase):
         def _on_item_scraped(item):
             items.append(item)
 
-        from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpider
         crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider)
         crawler.signals.connect(_on_item_scraped, signals.item_scraped)
         yield crawler.crawl(mockserver=self.mockserver)