mirror of https://github.com/scrapy/scrapy.git
Restore and deprecate the spider arg of Downloader._get_slot(). (#7178)
This commit is contained in:
parent
c49ae2115a
commit
5a7e132486
|
|
@ -151,7 +151,10 @@ class Downloader:
|
||||||
def needs_backout(self) -> bool:
|
def needs_backout(self) -> bool:
|
||||||
return len(self.active) >= self.total_concurrency
|
return len(self.active) >= self.total_concurrency
|
||||||
|
|
||||||
def _get_slot(self, request: Request) -> tuple[str, Slot]:
|
@_warn_spider_arg
|
||||||
|
def _get_slot(
|
||||||
|
self, request: Request, spider: Spider | None = None
|
||||||
|
) -> tuple[str, Slot]:
|
||||||
key = self.get_slot_key(request)
|
key = self.get_slot_key(request)
|
||||||
if key not in self.slots:
|
if key not in self.slots:
|
||||||
assert self.crawler.spider
|
assert self.crawler.spider
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ from twisted.internet.defer import inlineCallbacks
|
||||||
from scrapy import Request
|
from scrapy import Request
|
||||||
from scrapy.core.downloader import Downloader, Slot
|
from scrapy.core.downloader import Downloader, Slot
|
||||||
from scrapy.crawler import CrawlerRunner
|
from scrapy.crawler import CrawlerRunner
|
||||||
|
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||||
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
|
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
|
||||||
from scrapy.utils.spider import DefaultSpider
|
from scrapy.utils.spider import DefaultSpider
|
||||||
from scrapy.utils.test import get_crawler
|
from scrapy.utils.test import get_crawler
|
||||||
|
|
@ -108,6 +109,24 @@ def test_params():
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_slot_deprecated_spider_arg():
|
||||||
|
crawler = get_crawler(DefaultSpider)
|
||||||
|
crawler.spider = crawler._create_spider()
|
||||||
|
downloader = Downloader(crawler)
|
||||||
|
downloader._slot_gc_loop.stop() # Prevent an unclean reactor.
|
||||||
|
request = Request("https://example.com")
|
||||||
|
|
||||||
|
with pytest.warns(
|
||||||
|
ScrapyDeprecationWarning,
|
||||||
|
match=r"Passing a 'spider' argument to Downloader\._get_slot\(\) is deprecated",
|
||||||
|
):
|
||||||
|
key1, slot1 = downloader._get_slot(request, spider=crawler.spider)
|
||||||
|
key2, slot2 = downloader._get_slot(request)
|
||||||
|
|
||||||
|
assert key1 == key2
|
||||||
|
assert slot1 == slot2
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"priority_queue_class",
|
"priority_queue_class",
|
||||||
[
|
[
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue