mirror of https://github.com/scrapy/scrapy.git
Restore and deprecate the spider arg of Downloader._get_slot(). (#7178)
This commit is contained in:
parent
c49ae2115a
commit
5a7e132486
|
|
@ -151,7 +151,10 @@ class Downloader:
|
|||
def needs_backout(self) -> bool:
|
||||
return len(self.active) >= self.total_concurrency
|
||||
|
||||
def _get_slot(self, request: Request) -> tuple[str, Slot]:
|
||||
@_warn_spider_arg
|
||||
def _get_slot(
|
||||
self, request: Request, spider: Spider | None = None
|
||||
) -> tuple[str, Slot]:
|
||||
key = self.get_slot_key(request)
|
||||
if key not in self.slots:
|
||||
assert self.crawler.spider
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from twisted.internet.defer import inlineCallbacks
|
|||
from scrapy import Request
|
||||
from scrapy.core.downloader import Downloader, Slot
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
|
||||
from scrapy.utils.spider import DefaultSpider
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
|
@ -108,6 +109,24 @@ def test_params():
|
|||
)
|
||||
|
||||
|
||||
def test_get_slot_deprecated_spider_arg():
|
||||
crawler = get_crawler(DefaultSpider)
|
||||
crawler.spider = crawler._create_spider()
|
||||
downloader = Downloader(crawler)
|
||||
downloader._slot_gc_loop.stop() # Prevent an unclean reactor.
|
||||
request = Request("https://example.com")
|
||||
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match=r"Passing a 'spider' argument to Downloader\._get_slot\(\) is deprecated",
|
||||
):
|
||||
key1, slot1 = downloader._get_slot(request, spider=crawler.spider)
|
||||
key2, slot2 = downloader._get_slot(request)
|
||||
|
||||
assert key1 == key2
|
||||
assert slot1 == slot2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"priority_queue_class",
|
||||
[
|
||||
|
|
|
|||
Loading…
Reference in New Issue