Restore and deprecate the spider arg of Downloader._get_slot(). (#7178)

This commit is contained in:
Andrey Rakhmatullin 2025-12-15 15:42:24 +05:00 committed by GitHub
parent c49ae2115a
commit 5a7e132486
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 23 additions and 1 deletions

View File

@ -151,7 +151,10 @@ class Downloader:
def needs_backout(self) -> bool: def needs_backout(self) -> bool:
return len(self.active) >= self.total_concurrency return len(self.active) >= self.total_concurrency
def _get_slot(self, request: Request) -> tuple[str, Slot]: @_warn_spider_arg
def _get_slot(
self, request: Request, spider: Spider | None = None
) -> tuple[str, Slot]:
key = self.get_slot_key(request) key = self.get_slot_key(request)
if key not in self.slots: if key not in self.slots:
assert self.crawler.spider assert self.crawler.spider

View File

@ -7,6 +7,7 @@ from twisted.internet.defer import inlineCallbacks
from scrapy import Request from scrapy import Request
from scrapy.core.downloader import Downloader, Slot from scrapy.core.downloader import Downloader, Slot
from scrapy.crawler import CrawlerRunner from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.spider import DefaultSpider from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
@ -108,6 +109,24 @@ def test_params():
) )
def test_get_slot_deprecated_spider_arg():
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
downloader = Downloader(crawler)
downloader._slot_gc_loop.stop() # Prevent an unclean reactor.
request = Request("https://example.com")
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a 'spider' argument to Downloader\._get_slot\(\) is deprecated",
):
key1, slot1 = downloader._get_slot(request, spider=crawler.spider)
key2, slot2 = downloader._get_slot(request)
assert key1 == key2
assert slot1 == slot2
@pytest.mark.parametrize( @pytest.mark.parametrize(
"priority_queue_class", "priority_queue_class",
[ [