From 4d74e47cfa0f86a6be48176b51d289846af1912d Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 15 Jun 2026 15:01:41 +0200 Subject: [PATCH] Address typing issues --- scrapy/_classutilities.py | 27 +++++++++++++++------------ scrapy/core/downloader/__init__.py | 11 ++++++----- scrapy/core/downloader/middleware.py | 4 ++-- scrapy/core/scraper.py | 14 +++++++------- tests/test_downloader.py | 2 +- 5 files changed, 31 insertions(+), 27 deletions(-) diff --git a/scrapy/_classutilities.py b/scrapy/_classutilities.py index 6a7120081..720c1365e 100644 --- a/scrapy/_classutilities.py +++ b/scrapy/_classutilities.py @@ -1,5 +1,5 @@ # https://github.com/david-salac/classutilities/issues/1 -# Unmodified copy of +# Modified copy of # https://github.com/david-salac/classutilities/blob/a6e4a86331936d432afaa454ed4c963528165a61/src/classutilities/classproperty.py # Allows creating a class level property @@ -22,37 +22,37 @@ class ClassPropertyContainer: self.prop_get: Any = prop_get self.prop_set: Any = prop_set - def __get__(self, obj: Any, cls: type | None = None) -> Callable: + def __get__(self, obj: Any, cls: type | None = None) -> Any: """ - Get the property getter. + Return the value of the class property. :param obj: Instance of the class. :param cls: Type of the class. - :return: Class property getter. + :return: Value of the class property. """ if cls is None: cls = type(obj) return self.prop_get.__get__(obj, cls)() - def __set__(self, obj, value) -> Callable: + def __set__(self, obj: Any, value: Any) -> None: """ - Get the property setter. + Set the value of the class property. :param obj: Instance of the class. :param value: A value to be set. - :return: Class property setter. """ if not self.prop_set: raise AttributeError("cannot set attribute") _type: type = type(obj) if _type == ClassPropertyMetaClass: _type = obj - return self.prop_set.__get__(obj, _type)(value) + self.prop_set.__get__(obj, _type)(value) def setter( - self, func: Callable | classmethod | staticmethod + self, + func: Callable[..., Any] | classmethod[Any, Any, Any] | staticmethod[Any, Any], ) -> "ClassPropertyContainer": """ Allows creating setter in a property like way. - :param func: Getter function. + :param func: Setter function. :return: Setter object for the decorator. """ if not isinstance(func, (classmethod, staticmethod)): @@ -61,7 +61,9 @@ class ClassPropertyContainer: return self -def classproperty(func): +def classproperty( + func: Callable[..., Any] | classmethod[Any, Any, Any] | staticmethod[Any, Any], +) -> "ClassPropertyContainer": """ Create a decorator for a class level property. :param func: This class method is decorated. @@ -78,8 +80,9 @@ class ClassPropertyMetaClass(type): Metaclass that allows creating a standard setter. """ - def __setattr__(cls, key, value): + def __setattr__(cls, key: str, value: Any) -> None: """Overloads setter for class""" + obj = None if key in cls.__dict__: obj = cls.__dict__.get(key) if obj and isinstance(obj, ClassPropertyContainer): diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 9c6bc27bd..a1fab7756 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -11,9 +11,9 @@ from typing import TYPE_CHECKING, Any from warnings import warn try: - from win_precise_time import time + from win_precise_time import time # type: ignore[import-not-found] except ImportError: - from time import time + from time import time # pylint: disable=ungrouped-imports from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure @@ -139,7 +139,7 @@ class Downloader: "DOWNLOAD_SLOTS" ) self._stats = crawler.stats - self._last_backout = (None, None) + self._last_backout: tuple[str | None, float | None] = (None, None) deprecated_setting_priority = self.settings.getpriority( "SCRAPER_SLOT_MAX_ACTIVE_SIZE" @@ -184,12 +184,13 @@ class Downloader: self.active.remove(request) self.middleware._discount_rough_size(rough_size) - def _record_backout(self, reason): + def _record_backout(self, reason: str | None) -> None: last_reason, last_reason_start_time = self._last_backout if last_reason == reason: return current_time = time() - if last_reason is not None: + if last_reason is not None and self._stats is not None: + assert last_reason_start_time is not None last_reason_seconds = current_time - last_reason_start_time self._stats.inc_value("request_backout_seconds/total", last_reason_seconds) self._stats.inc_value( diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 560456bde..0a126191d 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -35,10 +35,10 @@ if TYPE_CHECKING: class DownloaderMiddlewareManager(MiddlewareManager): component_name = "downloader middleware" - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self.response_active_size = 0 - self._tracked_responses = WeakSet() + self._tracked_responses: WeakSet[Response] = WeakSet() self._rough_active_size = 0 assert self.crawler is not None self._response_rough_size: int = self.crawler.settings.getint( diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 1735c67ba..ffa24b154 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -65,7 +65,7 @@ class Slot(ClassPropertiesMixin): _MIN_RESPONSE_SIZE = 1024 @classproperty - def MIN_RESPONSE_SIZE(cls): + def MIN_RESPONSE_SIZE(cls): # pylint: disable=no-self-argument warnings.warn( "scrapy.core.scraper.Slot.MIN_RESPONSE_SIZE is deprecated.", ScrapyDeprecationWarning, @@ -74,7 +74,7 @@ class Slot(ClassPropertiesMixin): return cls._MIN_RESPONSE_SIZE @MIN_RESPONSE_SIZE.setter # type: ignore[no-redef] - def MIN_RESPONSE_SIZE(cls, value): + def MIN_RESPONSE_SIZE(cls, value): # pylint: disable=no-self-argument warnings.warn( "scrapy.core.scraper.Slot.MIN_RESPONSE_SIZE is deprecated.", ScrapyDeprecationWarning, @@ -95,7 +95,7 @@ class Slot(ClassPropertiesMixin): ScrapyDeprecationWarning, stacklevel=2, ) - self._max_active_size = max_active_size + self._max_active_size: int = max_active_size self.queue: deque[QueueTuple] = deque() self.active: set[Request] = set() self.itemproc_size: int = 0 # just for scrapy.utils.engine.get_engine_status() @@ -103,7 +103,7 @@ class Slot(ClassPropertiesMixin): self._active_size: int = 0 @property - def active_size(self): + def active_size(self) -> int: warnings.warn( ( "scrapy.core.scraper.Slot.active_size is deprecated. Read " @@ -116,7 +116,7 @@ class Slot(ClassPropertiesMixin): return self._active_size @active_size.setter - def active_size(self, value): + def active_size(self, value: int) -> None: warnings.warn( ( "scrapy.core.scraper.Slot.active_size is deprecated. " @@ -134,7 +134,7 @@ class Slot(ClassPropertiesMixin): self._active_size = value @property - def max_active_size(self): + def max_active_size(self) -> int: warnings.warn( ( "scrapy.core.scraper.Slot.max_active_size is deprecated. Read " @@ -146,7 +146,7 @@ class Slot(ClassPropertiesMixin): return self._max_active_size @max_active_size.setter - def max_active_size(self, value): + def max_active_size(self, value: int) -> None: warnings.warn( ( "scrapy.core.scraper.Slot.max_active_size is deprecated. Set " diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 1469517d5..1195fd6ed 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -27,7 +27,7 @@ class OfflineSpider(Spider): class gt: - __hash__ = None + __hash__ = None # type: ignore[assignment] def __init__(self, value): self.value = value