Fix setting ExecutionEngine._slot to None on close. (#6910)

* Fix setting ExecutionEngine._slot to None on close.

* Check self._slot in needs_backout().
This commit is contained in:
Andrey Rakhmatullin 2025-06-25 17:29:29 +05:00 committed by GitHub
parent d70f8a3f14
commit ff7795b159
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 10 additions and 4 deletions

View File

@ -266,10 +266,10 @@ class ExecutionEngine:
See :ref:`start-requests-lazy` for an example.
"""
assert self._slot is not None # typing
assert self.scraper.slot is not None # typing
return (
not self.running
or not self._slot
or bool(self._slot.closing)
or self.downloader.needs_backout()
or self.scraper.slot.needs_backout()
@ -530,10 +530,16 @@ class ExecutionEngine:
)
)
dfd.addBoth(lambda _: setattr(self, "slot", None))
def unassign_slot(_: Any) -> None:
self._slot = None
dfd.addBoth(unassign_slot)
dfd.addErrback(log_failure("Error while unassigning slot"))
dfd.addBoth(lambda _: setattr(self, "spider", None))
def unassign_spider(_: Any) -> None:
self.spider = None
dfd.addBoth(unassign_spider)
dfd.addErrback(log_failure("Error while unassigning spider"))
dfd.addBoth(lambda _: self._spider_closed_callback(spider))

View File

@ -112,7 +112,7 @@ class TestMain(TestCase):
with LogCapture(level=ERROR) as log:
await maybe_deferred_to_future(crawler.crawl())
assert not log.records, f"{log.records=}"
assert not log.records
finish_reason = crawler.stats.get_value("finish_reason")
assert finish_reason == "shutdown", f"{finish_reason=}"
expected_urls = []