Restore and deprecate the spider arg of Downloader._get_slot(). (#7178)

This commit is contained in:
Andrey Rakhmatullin 2025-12-15 15:42:24 +05:00 committed by GitHub
parent c49ae2115a
commit 5a7e132486
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 23 additions and 1 deletions

View File

@ -151,7 +151,10 @@ class Downloader:
def needs_backout(self) -> bool:
return len(self.active) >= self.total_concurrency
def _get_slot(self, request: Request) -> tuple[str, Slot]:
@_warn_spider_arg
def _get_slot(
self, request: Request, spider: Spider | None = None
) -> tuple[str, Slot]:
key = self.get_slot_key(request)
if key not in self.slots:
assert self.crawler.spider

View File

@ -7,6 +7,7 @@ from twisted.internet.defer import inlineCallbacks
from scrapy import Request
from scrapy.core.downloader import Downloader, Slot
from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
@ -108,6 +109,24 @@ def test_params():
)
def test_get_slot_deprecated_spider_arg():
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
downloader = Downloader(crawler)
downloader._slot_gc_loop.stop() # Prevent an unclean reactor.
request = Request("https://example.com")
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a 'spider' argument to Downloader\._get_slot\(\) is deprecated",
):
key1, slot1 = downloader._get_slot(request, spider=crawler.spider)
key2, slot2 = downloader._get_slot(request)
assert key1 == key2
assert slot1 == slot2
@pytest.mark.parametrize(
"priority_queue_class",
[