mirror of https://github.com/scrapy/scrapy.git
add additional requests examples.
This commit is contained in:
parent
92f86fab06
commit
c327a92e97
|
|
@ -98,6 +98,53 @@ Futures. Scrapy provides two helpers for this:
|
|||
into your own code.
|
||||
|
||||
|
||||
Async additional requests
|
||||
=====================
|
||||
|
||||
The spider below shows a single use-case of scraping page and gathering price from a separate url::
|
||||
|
||||
|
||||
class SingleRequestSpider(scrapy.Spider):
|
||||
name = "single"
|
||||
start_urls = ["https://example.org/product"]
|
||||
|
||||
async def parse(self, response, **kwargs):
|
||||
additional_request = scrapy.Request('https://example.org/price')
|
||||
deferred = self.crawler.engine.download(additional_request)
|
||||
additional_response = await maybe_deferred_to_future(deferred)
|
||||
yield {
|
||||
'h1': response.css('h1').get(),
|
||||
'price': additional_response.css('#price').get(),
|
||||
}
|
||||
|
||||
|
||||
Spider with gathering batch requests::
|
||||
|
||||
class BatchRequestsSpider(scrapy.Spider):
|
||||
name = "batch"
|
||||
start_urls = ["https://example.com/product"]
|
||||
|
||||
async def parse(self, response, **kwargs):
|
||||
additional_requests = [
|
||||
scrapy.Request("https://example.com/price1"),
|
||||
scrapy.Request("https://example.com/price2"),
|
||||
]
|
||||
coroutines = []
|
||||
for r in additional_requests:
|
||||
deffered = self.crawler.engine.download(r)
|
||||
coroutines.append(maybe_deferred_to_future(deffered))
|
||||
|
||||
responses = await asyncio.gather(
|
||||
*coroutines, return_exceptions=True
|
||||
)
|
||||
yield {
|
||||
'h1': response.css('h1::text').get(),
|
||||
'price': responses[0].css('.price_color::text').get(),
|
||||
'price2': responses[1].css('.price_color::text').get(),
|
||||
}
|
||||
|
||||
|
||||
|
||||
.. _enforce-asyncio-requirement:
|
||||
|
||||
Enforcing asyncio as a requirement
|
||||
|
|
|
|||
|
|
@ -340,8 +340,9 @@ def deferred_to_future(d: Deferred) -> Future:
|
|||
class MySpider(Spider):
|
||||
...
|
||||
async def parse(self, response):
|
||||
d = treq.get('https://example.com/additional')
|
||||
additional_response = await deferred_to_future(d)
|
||||
additional_request = scrapy.Request('https://example.org/price')
|
||||
deferred = self.crawler.engine.download(additional_request)
|
||||
additional_response = await deferred_to_future(deferred)
|
||||
"""
|
||||
return d.asFuture(_get_asyncio_event_loop())
|
||||
|
||||
|
|
@ -368,8 +369,9 @@ def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
|
|||
class MySpider(Spider):
|
||||
...
|
||||
async def parse(self, response):
|
||||
d = treq.get('https://example.com/additional')
|
||||
extra_response = await maybe_deferred_to_future(d)
|
||||
additional_request = scrapy.Request('https://example.org/price')
|
||||
deferred = self.crawler.engine.download(additional_request)
|
||||
additional_response = await maybe_deferred_to_future(deferred)
|
||||
"""
|
||||
if not is_asyncio_reactor_installed():
|
||||
return d
|
||||
|
|
|
|||
Loading…
Reference in New Issue