From 6ba6b032ad6828473510e07f8423f6c1e27c043e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 11 Dec 2025 13:46:51 +0500 Subject: [PATCH] Fix handling meta["download_slot"] == None. (#7172) --- scrapy/core/downloader/__init__.py | 6 +++--- tests/test_downloaderslotssettings.py | 15 +++++++++------ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 109a6eaec..be5efba87 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -4,7 +4,7 @@ import random from collections import deque from datetime import datetime from time import time -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure @@ -173,8 +173,8 @@ class Downloader: return key, self.slots[key] def get_slot_key(self, request: Request) -> str: - if self.DOWNLOAD_SLOT in request.meta: - return cast("str", request.meta[self.DOWNLOAD_SLOT]) + if (meta_slot := request.meta.get(self.DOWNLOAD_SLOT)) is not None: + return meta_slot key = urlparse_cached(request).hostname or "" if self.ip_concurrency: diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 960fcb30a..1d76ee611 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -1,4 +1,5 @@ import time +from typing import Any from twisted.internet.defer import inlineCallbacks @@ -28,24 +29,26 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): }, } + def __init__(self, *args: Any, **kwargs: Any): + super().__init__(*args, **kwargs) + self.default_slot = self.mockserver.host + self.times: dict[str, list[float]] = {} + async def start(self): - self.times = {None: []} - slots = [*self.custom_settings.get("DOWNLOAD_SLOTS", {}), None] - for slot in slots: url = self.mockserver.url(f"/?downloader_slot={slot}") - self.times[slot] = [] + self.times[slot or self.default_slot] = [] yield Request(url, callback=self.parse, meta={"download_slot": slot}) def parse(self, response): - slot = response.meta.get("download_slot", None) + slot = response.meta.get("download_slot", self.default_slot) self.times[slot].append(time.time()) url = self.mockserver.url(f"/?downloader_slot={slot}&req=2") yield Request(url, callback=self.not_parse, meta={"download_slot": slot}) def not_parse(self, response): - slot = response.meta.get("download_slot", None) + slot = response.meta.get("download_slot", self.default_slot) self.times[slot].append(time.time())