RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Forbid running ExecutionEngine._next_request in parallel.

This commit is contained in:
Andrey Rakhmatullin 2020-06-03 13:52:51 +05:00
parent 9b16ba8a59
commit 2f00a66237
2 changed files with 26 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
scrapy/core/engine.py
View File

@ -73,6 +73,7 @@ class ExecutionEngine:
self.downloader = downloader_cls(crawler)
self.scraper = Scraper(crawler)
self._spider_closed_callback = spider_closed_callback
self._waiting_for_request = False
@defer.inlineCallbacks
def start(self):
@ -125,30 +126,33 @@ class ExecutionEngine:
if self.paused:
return
while not self._needs_backout(spider):
if not self._next_request_from_scheduler(spider):
break
if self._waiting_for_request:
return
self._waiting_for_request = True
try:
while not self._needs_backout(spider):
if not self._next_request_from_scheduler(spider):
break
if slot.start_requests and not self._needs_backout(spider):
try:
if _isasyncgen(slot.start_requests):
request = yield deferred_from_coro(slot.start_requests.__anext__())
if slot.start_requests and not self._needs_backout(spider):
try:
if _isasyncgen(slot.start_requests):
request = yield deferred_from_coro(slot.start_requests.__anext__())
else:
request = next(slot.start_requests)
except (StopIteration, StopAsyncIteration):
slot.start_requests = None
except Exception:
slot.start_requests = None
logger.error('Error while obtaining start requests',
exc_info=True, extra={'spider': spider})
else:
request = next(slot.start_requests)
except (StopIteration, StopAsyncIteration):
if slot.start_requests is None:
# the crawl is already stopping
return
slot.start_requests = None
except Exception:
slot.start_requests = None
logger.error('Error while obtaining start requests',
exc_info=True, extra={'spider': spider})
else:
self.crawl(request, spider)
self.crawl(request, spider)
if self.spider_is_idle(spider) and slot.close_if_idle:
self._spider_idle(spider)
if self.spider_is_idle(spider) and slot.close_if_idle:
self._spider_idle(spider)
finally:
self._waiting_for_request = False
def _needs_backout(self, spider):
slot = self.slot

2
tests/py36/_test_engine.py
View File

@ -16,4 +16,4 @@ class StartRequestsAsyncGenAsyncioSpider(TestSpider):
async def start_requests(self):
for url in self.start_urls:
yield Request(url, dont_filter=True)
await asyncio.sleep(0.1)
await asyncio.sleep(1)