From 80194f1c0374b2aa429de4c7ea70a683c946db95 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 2 Oct 2022 15:22:06 -0300 Subject: [PATCH 1/2] CrawlSpider: add support for async def callbacks --- scrapy/spiders/crawl.py | 6 ++++-- tests/spiders.py | 12 ++++++++++++ tests/test_crawl.py | 11 +++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 1dcf2e6ab..d860ae0b4 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,7 +6,7 @@ See documentation in docs/topics/spiders.rst """ import copy -from typing import Sequence +from typing import Awaitable, Sequence from scrapy.http import Request, HtmlResponse from scrapy.linkextractors import LinkExtractor @@ -109,9 +109,11 @@ class CrawlSpider(Spider): rule = self._rules[failure.request.meta['rule']] return self._handle_failure(failure, rule.errback) - def _parse_response(self, response, callback, cb_kwargs, follow=True): + async def _parse_response(self, response, callback, cb_kwargs, follow=True): if callback: cb_res = callback(response, **cb_kwargs) or () + if isinstance(cb_res, Awaitable): + cb_res = await cb_res cb_res = self.process_results(response, cb_res) for request_or_item in iterate_spider_output(cb_res): yield request_or_item diff --git a/tests/spiders.py b/tests/spiders.py index 3b69aa7ae..2b78e1f7c 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -369,6 +369,18 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): yield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"}) +class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod): + """A CrawlSpider with an async def callback""" + name = 'crawl_spider_with_async_callback' + rules = ( + Rule(LinkExtractor(), callback='parse_async', follow=True), + ) + + async def parse_async(self, response, foo=None): + self.logger.info('[parse_async] status %i (foo: %s)', response.status, foo) + return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"}) + + class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = 'crawl_spider_with_errback' rules = ( diff --git a/tests/test_crawl.py b/tests/test_crawl.py index c11871745..d14021319 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -37,6 +37,7 @@ from tests.spiders import ( BrokenStartRequestsSpider, BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, + CrawlSpiderWithAsyncCallback, CrawlSpiderWithErrback, CrawlSpiderWithParseMethod, DelaySpider, @@ -391,6 +392,16 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("[parse] status 201 (foo: None)", str(log)) self.assertIn("[parse] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks + def test_crawlspider_with_async_callback(self): + crawler = get_crawler(CrawlSpiderWithAsyncCallback) + with LogCapture() as log: + yield crawler.crawl(mockserver=self.mockserver) + + self.assertIn("[parse_async] status 200 (foo: None)", str(log)) + self.assertIn("[parse_async] status 201 (foo: None)", str(log)) + self.assertIn("[parse_async] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks def test_crawlspider_with_errback(self): crawler = get_crawler(CrawlSpiderWithErrback) From da8f915091a769b3c3aca0f60d2370c2240213a1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 2 Oct 2022 17:37:10 -0300 Subject: [PATCH 2/2] Adapt for asyng generator callbacks --- scrapy/spiders/crawl.py | 11 +++++++---- scrapy/utils/asyncgen.py | 2 +- tests/spiders.py | 12 ++++++++++++ tests/test_crawl.py | 11 +++++++++++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index d860ae0b4..edac082d0 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,11 +6,12 @@ See documentation in docs/topics/spiders.rst """ import copy -from typing import Awaitable, Sequence +from typing import AsyncIterable, Awaitable, Sequence -from scrapy.http import Request, HtmlResponse +from scrapy.http import Request, Response, HtmlResponse from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider +from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.spider import iterate_spider_output @@ -78,7 +79,7 @@ class CrawlSpider(Spider): def parse_start_url(self, response, **kwargs): return [] - def process_results(self, response, results): + def process_results(self, response: Response, results: list): return results def _build_request(self, rule_index, link): @@ -112,7 +113,9 @@ class CrawlSpider(Spider): async def _parse_response(self, response, callback, cb_kwargs, follow=True): if callback: cb_res = callback(response, **cb_kwargs) or () - if isinstance(cb_res, Awaitable): + if isinstance(cb_res, AsyncIterable): + cb_res = await collect_asyncgen(cb_res) + elif isinstance(cb_res, Awaitable): cb_res = await cb_res cb_res = self.process_results(response, cb_res) for request_or_item in iterate_spider_output(cb_res): diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 9f794de92..c84b51e8c 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,7 +1,7 @@ from typing import AsyncGenerator, AsyncIterable, Iterable, Union -async def collect_asyncgen(result: AsyncIterable): +async def collect_asyncgen(result: AsyncIterable) -> list: results = [] async for x in result: results.append(x) diff --git a/tests/spiders.py b/tests/spiders.py index 2b78e1f7c..5ea8a4a21 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -381,6 +381,18 @@ class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod): return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"}) +class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod): + """A CrawlSpider with an async generator callback""" + name = 'crawl_spider_with_async_generator_callback' + rules = ( + Rule(LinkExtractor(), callback='parse_async_gen', follow=True), + ) + + async def parse_async_gen(self, response, foo=None): + self.logger.info('[parse_async_gen] status %i (foo: %s)', response.status, foo) + yield Request(self.mockserver.url("/status?n=202"), self.parse_async_gen, cb_kwargs={"foo": "bar"}) + + class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod): name = 'crawl_spider_with_errback' rules = ( diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d14021319..8be4b6fe1 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -38,6 +38,7 @@ from tests.spiders import ( BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, CrawlSpiderWithAsyncCallback, + CrawlSpiderWithAsyncGeneratorCallback, CrawlSpiderWithErrback, CrawlSpiderWithParseMethod, DelaySpider, @@ -402,6 +403,16 @@ class CrawlSpiderTestCase(TestCase): self.assertIn("[parse_async] status 201 (foo: None)", str(log)) self.assertIn("[parse_async] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks + def test_crawlspider_with_async_generator_callback(self): + crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback) + with LogCapture() as log: + yield crawler.crawl(mockserver=self.mockserver) + + self.assertIn("[parse_async_gen] status 200 (foo: None)", str(log)) + self.assertIn("[parse_async_gen] status 201 (foo: None)", str(log)) + self.assertIn("[parse_async_gen] status 202 (foo: bar)", str(log)) + @defer.inlineCallbacks def test_crawlspider_with_errback(self): crawler = get_crawler(CrawlSpiderWithErrback)