Merge pull request #5657 from elacuesta/crawlspider-support-async-def-callback

CrawlSpider: add support for async callbacks

scrapy/spiders/crawl.py

@@ -6,11 +6,12 @@ See documentation in docs/topics/spiders.rst
 """
 
 import copy
-from typing import Sequence
+from typing import AsyncIterable, Awaitable, Sequence
 
-from scrapy.http import Request, HtmlResponse
+from scrapy.http import Request, Response, HtmlResponse
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
+from scrapy.utils.asyncgen import collect_asyncgen
 from scrapy.utils.spider import iterate_spider_output
 
 
@@ -78,7 +79,7 @@ class CrawlSpider(Spider):
     def parse_start_url(self, response, **kwargs):
         return []
 
-    def process_results(self, response, results):
+    def process_results(self, response: Response, results: list):
         return results
 
     def _build_request(self, rule_index, link):
@@ -109,9 +110,13 @@ class CrawlSpider(Spider):
         rule = self._rules[failure.request.meta['rule']]
         return self._handle_failure(failure, rule.errback)
 
-    def _parse_response(self, response, callback, cb_kwargs, follow=True):
+    async def _parse_response(self, response, callback, cb_kwargs, follow=True):
         if callback:
             cb_res = callback(response, **cb_kwargs) or ()
+            if isinstance(cb_res, AsyncIterable):
+                cb_res = await collect_asyncgen(cb_res)
+            elif isinstance(cb_res, Awaitable):
+                cb_res = await cb_res
             cb_res = self.process_results(response, cb_res)
             for request_or_item in iterate_spider_output(cb_res):
                 yield request_or_item

scrapy/utils/asyncgen.py

@@ -1,7 +1,7 @@
 from typing import AsyncGenerator, AsyncIterable, Iterable, Union
 
 
-async def collect_asyncgen(result: AsyncIterable):
+async def collect_asyncgen(result: AsyncIterable) -> list:
     results = []
     async for x in result:
         results.append(x)

tests/spiders.py

@@ -369,6 +369,30 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
         yield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"})
 
 
+class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod):
+    """A CrawlSpider with an async def callback"""
+    name = 'crawl_spider_with_async_callback'
+    rules = (
+        Rule(LinkExtractor(), callback='parse_async', follow=True),
+    )
+
+    async def parse_async(self, response, foo=None):
+        self.logger.info('[parse_async] status %i (foo: %s)', response.status, foo)
+        return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"})
+
+
+class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod):
+    """A CrawlSpider with an async generator callback"""
+    name = 'crawl_spider_with_async_generator_callback'
+    rules = (
+        Rule(LinkExtractor(), callback='parse_async_gen', follow=True),
+    )
+
+    async def parse_async_gen(self, response, foo=None):
+        self.logger.info('[parse_async_gen] status %i (foo: %s)', response.status, foo)
+        yield Request(self.mockserver.url("/status?n=202"), self.parse_async_gen, cb_kwargs={"foo": "bar"})
+
+
 class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
     name = 'crawl_spider_with_errback'
     rules = (

tests/test_crawl.py

@@ -37,6 +37,8 @@ from tests.spiders import (
     BrokenStartRequestsSpider,
     BytesReceivedCallbackSpider,
     BytesReceivedErrbackSpider,
+    CrawlSpiderWithAsyncCallback,
+    CrawlSpiderWithAsyncGeneratorCallback,
     CrawlSpiderWithErrback,
     CrawlSpiderWithParseMethod,
     DelaySpider,
@@ -391,6 +393,26 @@ class CrawlSpiderTestCase(TestCase):
         self.assertIn("[parse] status 201 (foo: None)", str(log))
         self.assertIn("[parse] status 202 (foo: bar)", str(log))
 
+    @defer.inlineCallbacks
+    def test_crawlspider_with_async_callback(self):
+        crawler = get_crawler(CrawlSpiderWithAsyncCallback)
+        with LogCapture() as log:
+            yield crawler.crawl(mockserver=self.mockserver)
+
+        self.assertIn("[parse_async] status 200 (foo: None)", str(log))
+        self.assertIn("[parse_async] status 201 (foo: None)", str(log))
+        self.assertIn("[parse_async] status 202 (foo: bar)", str(log))
+
+    @defer.inlineCallbacks
+    def test_crawlspider_with_async_generator_callback(self):
+        crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback)
+        with LogCapture() as log:
+            yield crawler.crawl(mockserver=self.mockserver)
+
+        self.assertIn("[parse_async_gen] status 200 (foo: None)", str(log))
+        self.assertIn("[parse_async_gen] status 201 (foo: None)", str(log))
+        self.assertIn("[parse_async_gen] status 202 (foo: bar)", str(log))
+
     @defer.inlineCallbacks
     def test_crawlspider_with_errback(self):
         crawler = get_crawler(CrawlSpiderWithErrback)