diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index efb93c844..7713b1af1 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -98,56 +98,6 @@ Futures. Scrapy provides two helpers for this: into your own code. -Inline requests -=============== - -The spider below shows how to send a request and await its response all from -within a spider callback: - -.. code-block:: python - - from scrapy.utils.defer import maybe_deferred_to_future - - class SingleRequestSpider(scrapy.Spider): - name = "single" - start_urls = ["https://example.org/product"] - - async def parse(self, response, **kwargs): - additional_request = scrapy.Request('https://example.org/price') - deferred = self.crawler.engine.download(additional_request) - additional_response = await maybe_deferred_to_future(deferred) - yield { - 'h1': response.css('h1').get(), - 'price': additional_response.css('#price').get(), - } - -You can also send multiple requests in parallel: - -.. code-block:: python - - from scrapy.utils.defer import DeferredList, maybe_deferred_to_future - - class MultipleRequestsSpider(scrapy.Spider): - name = "multiple" - start_urls = ["https://example.com/product"] - - async def parse(self, response, **kwargs): - additional_requests = [ - scrapy.Request("https://example.com/price"), - scrapy.Request("https://example.com/color"), - ] - deferreds = [] - for r in additional_requests: - deferred = self.crawler.engine.download(r) - deferreds.append(deferred) - responses = await maybe_deferred_to_future(DeferredList(deferreds)) - yield { - 'h1': response.css('h1::text').get(), - 'price': responses[0][1].css('.price::text').get(), - 'price2': responses[1][1].css('.color::text').get(), - } - - .. _enforce-asyncio-requirement: Enforcing asyncio as a requirement diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 3916bd295..a65bab3ca 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -134,6 +134,63 @@ Common use cases for asynchronous code include: .. _aio-libs: https://github.com/aio-libs +.. _inline-requests: + +Inline requests +=============== + +The spider below shows how to send a request and await its response all from +within a spider callback: + +.. code-block:: python + + from scrapy import Spider, Request + from scrapy.utils.defer import maybe_deferred_to_future + + + class SingleRequestSpider(Spider): + name = "single" + start_urls = ["https://example.org/product"] + + async def parse(self, response, **kwargs): + additional_request = Request("https://example.org/price") + deferred = self.crawler.engine.download(additional_request) + additional_response = await maybe_deferred_to_future(deferred) + yield { + "h1": response.css("h1").get(), + "price": additional_response.css("#price").get(), + } + +You can also send multiple requests in parallel: + +.. code-block:: python + + from scrapy import Spider, Request + from scrapy.utils.defer import maybe_deferred_to_future + from twisted.internet.defer import DeferredList + + + class MultipleRequestsSpider(Spider): + name = "multiple" + start_urls = ["https://example.com/product"] + + async def parse(self, response, **kwargs): + additional_requests = [ + Request("https://example.com/price"), + Request("https://example.com/color"), + ] + deferreds = [] + for r in additional_requests: + deferred = self.crawler.engine.download(r) + deferreds.append(deferred) + responses = await maybe_deferred_to_future(DeferredList(deferreds)) + yield { + "h1": response.css("h1::text").get(), + "price": responses[0][1].css(".price::text").get(), + "price2": responses[1][1].css(".color::text").get(), + } + + .. _sync-async-spider-middleware: Mixing synchronous and asynchronous spider middlewares