Move inline request examples to the coroutines documentation

This commit is contained in:
Adrián Chaves 2023-05-16 13:39:02 +02:00
parent e4cf8fc121
commit d362699fa3
2 changed files with 57 additions and 50 deletions

View File

@ -98,56 +98,6 @@ Futures. Scrapy provides two helpers for this:
into your own code.
Inline requests
===============
The spider below shows how to send a request and await its response all from
within a spider callback:
.. code-block:: python
from scrapy.utils.defer import maybe_deferred_to_future
class SingleRequestSpider(scrapy.Spider):
name = "single"
start_urls = ["https://example.org/product"]
async def parse(self, response, **kwargs):
additional_request = scrapy.Request('https://example.org/price')
deferred = self.crawler.engine.download(additional_request)
additional_response = await maybe_deferred_to_future(deferred)
yield {
'h1': response.css('h1').get(),
'price': additional_response.css('#price').get(),
}
You can also send multiple requests in parallel:
.. code-block:: python
from scrapy.utils.defer import DeferredList, maybe_deferred_to_future
class MultipleRequestsSpider(scrapy.Spider):
name = "multiple"
start_urls = ["https://example.com/product"]
async def parse(self, response, **kwargs):
additional_requests = [
scrapy.Request("https://example.com/price"),
scrapy.Request("https://example.com/color"),
]
deferreds = []
for r in additional_requests:
deferred = self.crawler.engine.download(r)
deferreds.append(deferred)
responses = await maybe_deferred_to_future(DeferredList(deferreds))
yield {
'h1': response.css('h1::text').get(),
'price': responses[0][1].css('.price::text').get(),
'price2': responses[1][1].css('.color::text').get(),
}
.. _enforce-asyncio-requirement:
Enforcing asyncio as a requirement

View File

@ -134,6 +134,63 @@ Common use cases for asynchronous code include:
.. _aio-libs: https://github.com/aio-libs
.. _inline-requests:
Inline requests
===============
The spider below shows how to send a request and await its response all from
within a spider callback:
.. code-block:: python
from scrapy import Spider, Request
from scrapy.utils.defer import maybe_deferred_to_future
class SingleRequestSpider(Spider):
name = "single"
start_urls = ["https://example.org/product"]
async def parse(self, response, **kwargs):
additional_request = Request("https://example.org/price")
deferred = self.crawler.engine.download(additional_request)
additional_response = await maybe_deferred_to_future(deferred)
yield {
"h1": response.css("h1").get(),
"price": additional_response.css("#price").get(),
}
You can also send multiple requests in parallel:
.. code-block:: python
from scrapy import Spider, Request
from scrapy.utils.defer import maybe_deferred_to_future
from twisted.internet.defer import DeferredList
class MultipleRequestsSpider(Spider):
name = "multiple"
start_urls = ["https://example.com/product"]
async def parse(self, response, **kwargs):
additional_requests = [
Request("https://example.com/price"),
Request("https://example.com/color"),
]
deferreds = []
for r in additional_requests:
deferred = self.crawler.engine.download(r)
deferreds.append(deferred)
responses = await maybe_deferred_to_future(DeferredList(deferreds))
yield {
"h1": response.css("h1::text").get(),
"price": responses[0][1].css(".price::text").get(),
"price2": responses[1][1].css(".color::text").get(),
}
.. _sync-async-spider-middleware:
Mixing synchronous and asynchronous spider middlewares