mirror of https://github.com/scrapy/scrapy.git
Move inline request examples to the coroutines documentation
This commit is contained in:
parent
e4cf8fc121
commit
d362699fa3
|
|
@ -98,56 +98,6 @@ Futures. Scrapy provides two helpers for this:
|
|||
into your own code.
|
||||
|
||||
|
||||
Inline requests
|
||||
===============
|
||||
|
||||
The spider below shows how to send a request and await its response all from
|
||||
within a spider callback:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
|
||||
class SingleRequestSpider(scrapy.Spider):
|
||||
name = "single"
|
||||
start_urls = ["https://example.org/product"]
|
||||
|
||||
async def parse(self, response, **kwargs):
|
||||
additional_request = scrapy.Request('https://example.org/price')
|
||||
deferred = self.crawler.engine.download(additional_request)
|
||||
additional_response = await maybe_deferred_to_future(deferred)
|
||||
yield {
|
||||
'h1': response.css('h1').get(),
|
||||
'price': additional_response.css('#price').get(),
|
||||
}
|
||||
|
||||
You can also send multiple requests in parallel:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy.utils.defer import DeferredList, maybe_deferred_to_future
|
||||
|
||||
class MultipleRequestsSpider(scrapy.Spider):
|
||||
name = "multiple"
|
||||
start_urls = ["https://example.com/product"]
|
||||
|
||||
async def parse(self, response, **kwargs):
|
||||
additional_requests = [
|
||||
scrapy.Request("https://example.com/price"),
|
||||
scrapy.Request("https://example.com/color"),
|
||||
]
|
||||
deferreds = []
|
||||
for r in additional_requests:
|
||||
deferred = self.crawler.engine.download(r)
|
||||
deferreds.append(deferred)
|
||||
responses = await maybe_deferred_to_future(DeferredList(deferreds))
|
||||
yield {
|
||||
'h1': response.css('h1::text').get(),
|
||||
'price': responses[0][1].css('.price::text').get(),
|
||||
'price2': responses[1][1].css('.color::text').get(),
|
||||
}
|
||||
|
||||
|
||||
.. _enforce-asyncio-requirement:
|
||||
|
||||
Enforcing asyncio as a requirement
|
||||
|
|
|
|||
|
|
@ -134,6 +134,63 @@ Common use cases for asynchronous code include:
|
|||
.. _aio-libs: https://github.com/aio-libs
|
||||
|
||||
|
||||
.. _inline-requests:
|
||||
|
||||
Inline requests
|
||||
===============
|
||||
|
||||
The spider below shows how to send a request and await its response all from
|
||||
within a spider callback:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy import Spider, Request
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
|
||||
|
||||
class SingleRequestSpider(Spider):
|
||||
name = "single"
|
||||
start_urls = ["https://example.org/product"]
|
||||
|
||||
async def parse(self, response, **kwargs):
|
||||
additional_request = Request("https://example.org/price")
|
||||
deferred = self.crawler.engine.download(additional_request)
|
||||
additional_response = await maybe_deferred_to_future(deferred)
|
||||
yield {
|
||||
"h1": response.css("h1").get(),
|
||||
"price": additional_response.css("#price").get(),
|
||||
}
|
||||
|
||||
You can also send multiple requests in parallel:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy import Spider, Request
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
from twisted.internet.defer import DeferredList
|
||||
|
||||
|
||||
class MultipleRequestsSpider(Spider):
|
||||
name = "multiple"
|
||||
start_urls = ["https://example.com/product"]
|
||||
|
||||
async def parse(self, response, **kwargs):
|
||||
additional_requests = [
|
||||
Request("https://example.com/price"),
|
||||
Request("https://example.com/color"),
|
||||
]
|
||||
deferreds = []
|
||||
for r in additional_requests:
|
||||
deferred = self.crawler.engine.download(r)
|
||||
deferreds.append(deferred)
|
||||
responses = await maybe_deferred_to_future(DeferredList(deferreds))
|
||||
yield {
|
||||
"h1": response.css("h1::text").get(),
|
||||
"price": responses[0][1].css(".price::text").get(),
|
||||
"price2": responses[1][1].css(".color::text").get(),
|
||||
}
|
||||
|
||||
|
||||
.. _sync-async-spider-middleware:
|
||||
|
||||
Mixing synchronous and asynchronous spider middlewares
|
||||
|
|
|
|||
Loading…
Reference in New Issue