add additional requests examples.

This commit is contained in:
bulat 2023-05-09 18:04:18 +05:00
parent 92f86fab06
commit c327a92e97
2 changed files with 53 additions and 4 deletions

View File

@ -98,6 +98,53 @@ Futures. Scrapy provides two helpers for this:
into your own code.
Async additional requests
=====================
The spider below shows a single use-case of scraping page and gathering price from a separate url::
class SingleRequestSpider(scrapy.Spider):
name = "single"
start_urls = ["https://example.org/product"]
async def parse(self, response, **kwargs):
additional_request = scrapy.Request('https://example.org/price')
deferred = self.crawler.engine.download(additional_request)
additional_response = await maybe_deferred_to_future(deferred)
yield {
'h1': response.css('h1').get(),
'price': additional_response.css('#price').get(),
}
Spider with gathering batch requests::
class BatchRequestsSpider(scrapy.Spider):
name = "batch"
start_urls = ["https://example.com/product"]
async def parse(self, response, **kwargs):
additional_requests = [
scrapy.Request("https://example.com/price1"),
scrapy.Request("https://example.com/price2"),
]
coroutines = []
for r in additional_requests:
deffered = self.crawler.engine.download(r)
coroutines.append(maybe_deferred_to_future(deffered))
responses = await asyncio.gather(
*coroutines, return_exceptions=True
)
yield {
'h1': response.css('h1::text').get(),
'price': responses[0].css('.price_color::text').get(),
'price2': responses[1].css('.price_color::text').get(),
}
.. _enforce-asyncio-requirement:
Enforcing asyncio as a requirement

View File

@ -340,8 +340,9 @@ def deferred_to_future(d: Deferred) -> Future:
class MySpider(Spider):
...
async def parse(self, response):
d = treq.get('https://example.com/additional')
additional_response = await deferred_to_future(d)
additional_request = scrapy.Request('https://example.org/price')
deferred = self.crawler.engine.download(additional_request)
additional_response = await deferred_to_future(deferred)
"""
return d.asFuture(_get_asyncio_event_loop())
@ -368,8 +369,9 @@ def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
class MySpider(Spider):
...
async def parse(self, response):
d = treq.get('https://example.com/additional')
extra_response = await maybe_deferred_to_future(d)
additional_request = scrapy.Request('https://example.org/price')
deferred = self.crawler.engine.download(additional_request)
additional_response = await maybe_deferred_to_future(deferred)
"""
if not is_asyncio_reactor_installed():
return d