Support yield in async def callbacks.

conftest.py

@@ -11,7 +11,9 @@ collect_ignore = [
     # not a test, but looks like a test
     "scrapy/utils/testsite.py",
     # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
-    *_py_files("tests/CrawlerProcess")
+    *_py_files("tests/CrawlerProcess"),
+    # Py36-only parts of respective tests
+    *_py_files("tests/py36"),
 ]
 
 for line in open('tests/ignores.txt'):

scrapy/utils/py36.py

@@ -0,0 +1,10 @@
+"""
+Helpers using Python 3.6+ syntax (ignore SyntaxError on import).
+"""
+
+
+async def collect_asyncgen(result):
+    results = []
+    async for x in result:
+        results.append(x)
+    return results

scrapy/utils/spider.py

@@ -4,12 +4,20 @@ import inspect
 from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.misc import arg_to_iter
+try:
+    from scrapy.utils.py36 import collect_asyncgen
+except SyntaxError:
+    collect_asyncgen = None
 
 
 logger = logging.getLogger(__name__)
 
 
 def iterate_spider_output(result):
+    if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result):
+        d = deferred_from_coro(collect_asyncgen(result))
+        d.addCallback(iterate_spider_output)
+        return d
     return arg_to_iter(deferred_from_coro(result))
 
 

tests/py36/_test_crawl.py

@@ -0,0 +1,50 @@
+import asyncio
+
+from scrapy import Request
+from tests.spiders import SimpleSpider
+
+
+class AsyncDefAsyncioGenSpider(SimpleSpider):
+
+    name = 'asyncdef_asyncio_gen'
+
+    async def parse(self, response):
+        await asyncio.sleep(0.2)
+        yield {'foo': 42}
+        self.logger.info("Got response %d" % response.status)
+
+
+class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
+
+    name = 'asyncdef_asyncio_gen_loop'
+
+    async def parse(self, response):
+        for i in range(10):
+            await asyncio.sleep(0.1)
+            yield {'foo': i}
+        self.logger.info("Got response %d" % response.status)
+
+
+class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
+
+    name = 'asyncdef_asyncio_gen_complex'
+    initial_reqs = 4
+    following_reqs = 3
+    depth = 2
+
+    def _get_req(self, index):
+        return Request(self.mockserver.url("/status?n=200&request=%d" % index),
+                       meta={'index': index})
+
+    def start_requests(self):
+        for i in range(self.initial_reqs):
+            yield self._get_req(i)
+
+    async def parse(self, response):
+        index = response.meta['index']
+        yield {'index': index}
+        if index < 10 ** self.depth:
+            for new_index in range(10 * index, 10 * index + self.following_reqs):
+                yield self._get_req(new_index)
+        await asyncio.sleep(0.1)
+        yield {'index': index + 5}

tests/test_crawl.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import sys
 
 from pytest import mark
 from testfixtures import LogCapture
@@ -343,3 +344,53 @@ with multiples lines
         self.assertIn("Got response 200", str(log))
         self.assertIn({'id': 1}, items)
         self.assertIn({'id': 2}, items)
+
+    @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_asyncgen_parse(self):
+        from tests.py36._test_crawl import AsyncDefAsyncioGenSpider
+        crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        self.assertIn("Got response 200", str(log))
+        itemcount = crawler.stats.get_value('item_scraped_count')
+        self.assertEqual(itemcount, 1)
+
+    @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_asyncgen_parse_loop(self):
+        items = []
+
+        def _on_item_scraped(item):
+            items.append(item)
+
+        from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpider
+        crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider)
+        crawler.signals.connect(_on_item_scraped, signals.item_scraped)
+        with LogCapture() as log:
+            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        self.assertIn("Got response 200", str(log))
+        itemcount = crawler.stats.get_value('item_scraped_count')
+        self.assertEqual(itemcount, 10)
+        for i in range(10):
+            self.assertIn({'foo': i}, items)
+
+    @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_asyncgen_parse_complex(self):
+        items = []
+
+        def _on_item_scraped(item):
+            items.append(item)
+
+        from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpider
+        crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider)
+        crawler.signals.connect(_on_item_scraped, signals.item_scraped)
+        yield crawler.crawl(mockserver=self.mockserver)
+        itemcount = crawler.stats.get_value('item_scraped_count')
+        self.assertEqual(itemcount, 80)
+        for i in [0, 3, 21, 22, 207, 311]:  # some random items
+            self.assertIn({'index': i}, items)