CrawlSpider: add support for async def callbacks

scrapy/spiders/crawl.py

@@ -6,7 +6,7 @@ See documentation in docs/topics/spiders.rst
 """
 
 import copy
-from typing import Sequence
+from typing import Awaitable, Sequence
 
 from scrapy.http import Request, HtmlResponse
 from scrapy.linkextractors import LinkExtractor
@@ -109,9 +109,11 @@ class CrawlSpider(Spider):
         rule = self._rules[failure.request.meta['rule']]
         return self._handle_failure(failure, rule.errback)
 
-    def _parse_response(self, response, callback, cb_kwargs, follow=True):
+    async def _parse_response(self, response, callback, cb_kwargs, follow=True):
         if callback:
             cb_res = callback(response, **cb_kwargs) or ()
+            if isinstance(cb_res, Awaitable):
+                cb_res = await cb_res
             cb_res = self.process_results(response, cb_res)
             for request_or_item in iterate_spider_output(cb_res):
                 yield request_or_item

tests/spiders.py

@@ -369,6 +369,18 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
         yield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"})
 
 
+class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod):
+    """A CrawlSpider with an async def callback"""
+    name = 'crawl_spider_with_async_callback'
+    rules = (
+        Rule(LinkExtractor(), callback='parse_async', follow=True),
+    )
+
+    async def parse_async(self, response, foo=None):
+        self.logger.info('[parse_async] status %i (foo: %s)', response.status, foo)
+        return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"})
+
+
 class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
     name = 'crawl_spider_with_errback'
     rules = (

tests/test_crawl.py

@@ -37,6 +37,7 @@ from tests.spiders import (
     BrokenStartRequestsSpider,
     BytesReceivedCallbackSpider,
     BytesReceivedErrbackSpider,
+    CrawlSpiderWithAsyncCallback,
     CrawlSpiderWithErrback,
     CrawlSpiderWithParseMethod,
     DelaySpider,
@@ -391,6 +392,16 @@ class CrawlSpiderTestCase(TestCase):
         self.assertIn("[parse] status 201 (foo: None)", str(log))
         self.assertIn("[parse] status 202 (foo: bar)", str(log))
 
+    @defer.inlineCallbacks
+    def test_crawlspider_with_async_callback(self):
+        crawler = get_crawler(CrawlSpiderWithAsyncCallback)
+        with LogCapture() as log:
+            yield crawler.crawl(mockserver=self.mockserver)
+
+        self.assertIn("[parse_async] status 200 (foo: None)", str(log))
+        self.assertIn("[parse_async] status 201 (foo: None)", str(log))
+        self.assertIn("[parse_async] status 202 (foo: bar)", str(log))
+
     @defer.inlineCallbacks
     def test_crawlspider_with_errback(self):
         crawler = get_crawler(CrawlSpiderWithErrback)