mirror of https://github.com/scrapy/scrapy.git
58 lines
1.6 KiB
Python
58 lines
1.6 KiB
Python
import asyncio
|
|
|
|
from scrapy import Request
|
|
from tests.spiders import SimpleSpider
|
|
|
|
|
|
class AsyncDefAsyncioGenSpider(SimpleSpider):
|
|
|
|
name = 'asyncdef_asyncio_gen'
|
|
|
|
async def parse(self, response):
|
|
await asyncio.sleep(0.2)
|
|
yield {'foo': 42}
|
|
self.logger.info("Got response %d" % response.status)
|
|
|
|
|
|
class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
|
|
|
|
name = 'asyncdef_asyncio_gen_loop'
|
|
|
|
async def parse(self, response):
|
|
for i in range(10):
|
|
await asyncio.sleep(0.1)
|
|
yield {'foo': i}
|
|
self.logger.info("Got response %d" % response.status)
|
|
|
|
|
|
class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
|
|
|
|
name = 'asyncdef_asyncio_gen_complex'
|
|
initial_reqs = 4
|
|
following_reqs = 3
|
|
depth = 2
|
|
|
|
def _get_req(self, index, cb=None):
|
|
return Request(self.mockserver.url("/status?n=200&request=%d" % index),
|
|
meta={'index': index},
|
|
dont_filter=True,
|
|
callback=cb)
|
|
|
|
def start_requests(self):
|
|
for i in range(1, self.initial_reqs + 1):
|
|
yield self._get_req(i)
|
|
|
|
async def parse(self, response):
|
|
index = response.meta['index']
|
|
yield {'index': index}
|
|
if index < 10 ** self.depth:
|
|
for new_index in range(10 * index, 10 * index + self.following_reqs):
|
|
yield self._get_req(new_index)
|
|
yield self._get_req(index, cb=self.parse2)
|
|
await asyncio.sleep(0.1)
|
|
yield {'index': index + 5}
|
|
|
|
async def parse2(self, response):
|
|
await asyncio.sleep(0.1)
|
|
yield {'index2': response.meta['index']}
|