diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index d860ae0b4..edac082d0 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,11 +6,12 @@ See documentation in docs/topics/spiders.rst """ import copy -from typing import Awaitable, Sequence +from typing import AsyncIterable, Awaitable, Sequence -from scrapy.http import Request, HtmlResponse +from scrapy.http import Request, Response, HtmlResponse from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider +from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.spider import iterate_spider_output @@ -78,7 +79,7 @@ class CrawlSpider(Spider): def parse_start_url(self, response, **kwargs): return [] - def process_results(self, response, results): + def process_results(self, response: Response, results: list): return results def _build_request(self, rule_index, link): @@ -112,7 +113,9 @@ class CrawlSpider(Spider): async def _parse_response(self, response, callback, cb_kwargs, follow=True): if callback: cb_res = callback(response, **cb_kwargs) or () - if isinstance(cb_res, Awaitable): + if isinstance(cb_res, AsyncIterable): + cb_res = await collect_asyncgen(cb_res) + elif isinstance(cb_res, Awaitable): cb_res = await cb_res cb_res = self.process_results(response, cb_res) for request_or_item in iterate_spider_output(cb_res): diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 9f794de92..c84b51e8c 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,7 +1,7 @@ from typing import AsyncGenerator, AsyncIterable, Iterable, Union -async def collect_asyncgen(result: AsyncIterable): +async def collect_asyncgen(result: AsyncIterable) -> list: results = [] async for x in result: results.append(x) diff --git a/tests/spiders.py b/tests/spiders.py index 2b78e1f7c..5ea8a4a21 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -381,6 +381,18 @@ class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod): return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"}) +class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod): + """A CrawlSpider with an async generator callback""" + name = 'crawl_spider_with_async_generator_callback' + rules = ( + Rule(LinkExtractor(), callback='parse_async_gen', follow=True), + ) + + async def parse_async_gen(self, response, foo=None): + self.logger.info('[parse_async_gen] status %i (foo: %s)', response.status, foo) + yield Request(self.mockserver.url("/status?n=202"), self.parse_async_gen, cb_kwargs={"foo": "bar"}) + + class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = 'crawl_spider_with_errback' rules = ( diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d14021319..8be4b6fe1 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -38,6 +38,7 @@ from tests.spiders import ( BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, CrawlSpiderWithAsyncCallback, + CrawlSpiderWithAsyncGeneratorCallback, CrawlSpiderWithErrback, CrawlSpiderWithParseMethod, DelaySpider, @@ -402,6 +403,16 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("[parse_async] status 201 (foo: None)", str(log)) self.assertIn("[parse_async] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks + def test_crawlspider_with_async_generator_callback(self): + crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback) + with LogCapture() as log: + yield crawler.crawl(mockserver=self.mockserver) + + self.assertIn("[parse_async_gen] status 200 (foo: None)", str(log)) + self.assertIn("[parse_async_gen] status 201 (foo: None)", str(log)) + self.assertIn("[parse_async_gen] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks def test_crawlspider_with_errback(self): crawler = get_crawler(CrawlSpiderWithErrback)