From 388d4fcf04e3421f58cd7fc0920da28ee5449a00 Mon Sep 17 00:00:00 2001 From: Adrian Date: Tue, 4 Aug 2026 08:56:46 +0200 Subject: [PATCH] Add a _load_objects() helper for object-or-path setting lists (#7857) Co-authored-by: Claude Opus 5 (1M context) --- scrapy/downloadermiddlewares/retry.py | 7 ++----- scrapy/utils/misc.py | 5 +++++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index e1dcd90d5..f910d07c8 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING from scrapy.exceptions import NotConfigured from scrapy.utils.decorators import _warn_spider_arg -from scrapy.utils.misc import load_object +from scrapy.utils.misc import _load_objects from scrapy.utils.python import global_object_name from scrapy.utils.response import response_status_message @@ -149,10 +149,7 @@ class RetryMiddleware: self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")} self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") self.give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"] - self.exceptions_to_retry = tuple( - load_object(x) if isinstance(x, str) else x - for x in settings.getlist("RETRY_EXCEPTIONS") - ) + self.exceptions_to_retry = _load_objects(settings.getlist("RETRY_EXCEPTIONS")) @classmethod def from_crawler(cls, crawler: Crawler) -> Self: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 20e7cb381..57b526be6 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -90,6 +90,11 @@ def load_object(path: str | Callable[..., Any]) -> Any: return obj +def _load_objects(objects: Iterable[str | Callable[..., Any]]) -> tuple[Any, ...]: + """Resolve *objects* (objects or import paths) to a tuple of objects.""" + return tuple(load_object(obj) if isinstance(obj, str) else obj for obj in objects) + + def walk_modules_iter(path: str) -> Iterable[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that