From a91a13b4434ae60be6fded94e4ed08ba322b4f69 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 12 Nov 2019 23:09:00 +0500 Subject: [PATCH 1/2] Support for async def callbacks. --- scrapy/utils/spider.py | 5 +++-- tests/spiders.py | 23 +++++++++++++++++++++++ tests/test_crawl.py | 20 +++++++++++++++++++- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 4061d1ea3..72775df5c 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -2,14 +2,15 @@ import logging import inspect from scrapy.spiders import Spider -from scrapy.utils.misc import arg_to_iter +from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.misc import arg_to_iter logger = logging.getLogger(__name__) def iterate_spider_output(result): - return arg_to_iter(result) + return arg_to_iter(deferred_from_coro(result)) def iter_spider_classes(module): diff --git a/tests/spiders.py b/tests/spiders.py index 39c8da0b6..e4f2d5474 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -1,14 +1,18 @@ """ Some spiders used for testing and benchmarking """ +import asyncio import time from urllib.parse import urlencode +from twisted.internet import defer + from scrapy.http import Request from scrapy.item import Item from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.spiders.crawl import CrawlSpider, Rule +from scrapy.utils.test import get_from_asyncio_queue class MockServerSpider(Spider): @@ -83,6 +87,25 @@ class SimpleSpider(MetaSpider): self.logger.info("Got response %d" % response.status) +class AsyncDefSpider(SimpleSpider): + + name = 'asyncdef' + + async def parse(self, response): + await defer.succeed(42) + self.logger.info("Got response %d" % response.status) + + +class AsyncDefAsyncioSpider(SimpleSpider): + + name = 'asyncdef_asyncio' + + async def parse(self, response): + await asyncio.sleep(0.2) + status = await get_from_asyncio_queue(response.status) + self.logger.info("Got response %d" % status) + + class ItemSpider(FollowAllSpider): name = 'item' diff --git a/tests/test_crawl.py b/tests/test_crawl.py index f433fcea6..99b887ff6 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,6 +1,7 @@ import json import logging +from pytest import mark from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase @@ -10,7 +11,8 @@ from scrapy.http import Request from scrapy.utils.python import to_unicode from tests.mockserver import MockServer from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider, - SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback) + SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback, + AsyncDefSpider, AsyncDefAsyncioSpider) class CrawlTestCase(TestCase): @@ -308,3 +310,19 @@ with multiples lines self.assertIn("[callback] status 201", str(log)) self.assertIn("[errback] status 404", str(log)) self.assertIn("[errback] status 500", str(log)) + + @defer.inlineCallbacks + def test_async_def_parse(self): + self.runner.crawl(AsyncDefSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + with LogCapture() as log: + yield self.runner.join() + self.assertIn("Got response 200", str(log)) + + @mark.only_asyncio() + @defer.inlineCallbacks + def test_async_def_asyncio_parse(self): + runner = CrawlerRunner({"ASYNCIO_REACTOR": True}) + runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + with LogCapture() as log: + yield runner.join() + self.assertIn("Got response 200", str(log)) From cc825c21deaa56875f2abf1b30b53abb60c566c7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 30 Jan 2020 16:17:06 +0500 Subject: [PATCH 2/2] Test returning items from an async def callback. --- tests/spiders.py | 11 +++++++++++ tests/test_crawl.py | 19 ++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/spiders.py b/tests/spiders.py index e4f2d5474..3b1ee94b8 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -106,6 +106,17 @@ class AsyncDefAsyncioSpider(SimpleSpider): self.logger.info("Got response %d" % status) +class AsyncDefAsyncioReturnSpider(SimpleSpider): + + name = 'asyncdef_asyncio_return' + + async def parse(self, response): + await asyncio.sleep(0.2) + status = await get_from_asyncio_queue(response.status) + self.logger.info("Got response %d" % status) + return [{'id': 1}, {'id': 2}] + + class ItemSpider(FollowAllSpider): name = 'item' diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 99b887ff6..85005eba4 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -6,13 +6,14 @@ from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase +from scrapy import signals from scrapy.crawler import CrawlerRunner from scrapy.http import Request from scrapy.utils.python import to_unicode from tests.mockserver import MockServer from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback, - AsyncDefSpider, AsyncDefAsyncioSpider) + AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider) class CrawlTestCase(TestCase): @@ -326,3 +327,19 @@ with multiples lines with LogCapture() as log: yield runner.join() self.assertIn("Got response 200", str(log)) + + @mark.only_asyncio() + @defer.inlineCallbacks + def test_async_def_asyncio_parse_list(self): + items = [] + + def _on_item_scraped(item): + items.append(item) + + crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSpider) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + with LogCapture() as log: + yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + self.assertIn("Got response 200", str(log)) + self.assertIn({'id': 1}, items) + self.assertIn({'id': 2}, items)