Address typing issues

scrapy/_classutilities.py

@@ -1,5 +1,5 @@
 # https://github.com/david-salac/classutilities/issues/1
-# Unmodified copy of
+# Modified copy of
 # https://github.com/david-salac/classutilities/blob/a6e4a86331936d432afaa454ed4c963528165a61/src/classutilities/classproperty.py
 
 # Allows creating a class level property
@@ -22,37 +22,37 @@ class ClassPropertyContainer:
         self.prop_get: Any = prop_get
         self.prop_set: Any = prop_set
 
-    def __get__(self, obj: Any, cls: type | None = None) -> Callable:
+    def __get__(self, obj: Any, cls: type | None = None) -> Any:
         """
-        Get the property getter.
+        Return the value of the class property.
         :param obj: Instance of the class.
         :param cls: Type of the class.
-        :return: Class property getter.
+        :return: Value of the class property.
         """
         if cls is None:
             cls = type(obj)
         return self.prop_get.__get__(obj, cls)()
 
-    def __set__(self, obj, value) -> Callable:
+    def __set__(self, obj: Any, value: Any) -> None:
         """
-        Get the property setter.
+        Set the value of the class property.
         :param obj: Instance of the class.
         :param value: A value to be set.
-        :return: Class property setter.
         """
         if not self.prop_set:
             raise AttributeError("cannot set attribute")
         _type: type = type(obj)
         if _type == ClassPropertyMetaClass:
             _type = obj
-        return self.prop_set.__get__(obj, _type)(value)
+        self.prop_set.__get__(obj, _type)(value)
 
     def setter(
-        self, func: Callable | classmethod | staticmethod
+        self,
+        func: Callable[..., Any] | classmethod[Any, Any, Any] | staticmethod[Any, Any],
     ) -> "ClassPropertyContainer":
         """
         Allows creating setter in a property like way.
-        :param func: Getter function.
+        :param func: Setter function.
         :return: Setter object for the decorator.
         """
         if not isinstance(func, (classmethod, staticmethod)):
@@ -61,7 +61,9 @@ class ClassPropertyContainer:
         return self
 
 
-def classproperty(func):
+def classproperty(
+    func: Callable[..., Any] | classmethod[Any, Any, Any] | staticmethod[Any, Any],
+) -> "ClassPropertyContainer":
     """
     Create a decorator for a class level property.
     :param func: This class method is decorated.
@@ -78,8 +80,9 @@ class ClassPropertyMetaClass(type):
     Metaclass that allows creating a standard setter.
     """
 
-    def __setattr__(cls, key, value):
+    def __setattr__(cls, key: str, value: Any) -> None:
         """Overloads setter for class"""
+        obj = None
         if key in cls.__dict__:
             obj = cls.__dict__.get(key)
         if obj and isinstance(obj, ClassPropertyContainer):

scrapy/core/downloader/__init__.py

@@ -11,9 +11,9 @@ from typing import TYPE_CHECKING, Any
 from warnings import warn
 
 try:
-    from win_precise_time import time
+    from win_precise_time import time  # type: ignore[import-not-found]
 except ImportError:
-    from time import time
+    from time import time  # pylint: disable=ungrouped-imports
 
 from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.python.failure import Failure
@@ -139,7 +139,7 @@ class Downloader:
             "DOWNLOAD_SLOTS"
         )
         self._stats = crawler.stats
-        self._last_backout = (None, None)
+        self._last_backout: tuple[str | None, float | None] = (None, None)
 
         deprecated_setting_priority = self.settings.getpriority(
             "SCRAPER_SLOT_MAX_ACTIVE_SIZE"
@@ -184,12 +184,13 @@ class Downloader:
             self.active.remove(request)
             self.middleware._discount_rough_size(rough_size)
 
-    def _record_backout(self, reason):
+    def _record_backout(self, reason: str | None) -> None:
         last_reason, last_reason_start_time = self._last_backout
         if last_reason == reason:
             return
         current_time = time()
-        if last_reason is not None:
+        if last_reason is not None and self._stats is not None:
+            assert last_reason_start_time is not None
             last_reason_seconds = current_time - last_reason_start_time
             self._stats.inc_value("request_backout_seconds/total", last_reason_seconds)
             self._stats.inc_value(

scrapy/core/downloader/middleware.py

@@ -35,10 +35,10 @@ if TYPE_CHECKING:
 class DownloaderMiddlewareManager(MiddlewareManager):
     component_name = "downloader middleware"
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.response_active_size = 0
-        self._tracked_responses = WeakSet()
+        self._tracked_responses: WeakSet[Response] = WeakSet()
         self._rough_active_size = 0
         assert self.crawler is not None
         self._response_rough_size: int = self.crawler.settings.getint(

scrapy/core/scraper.py

@@ -65,7 +65,7 @@ class Slot(ClassPropertiesMixin):
     _MIN_RESPONSE_SIZE = 1024
 
     @classproperty
-    def MIN_RESPONSE_SIZE(cls):
+    def MIN_RESPONSE_SIZE(cls):  # pylint: disable=no-self-argument
         warnings.warn(
             "scrapy.core.scraper.Slot.MIN_RESPONSE_SIZE is deprecated.",
             ScrapyDeprecationWarning,
@@ -74,7 +74,7 @@ class Slot(ClassPropertiesMixin):
         return cls._MIN_RESPONSE_SIZE
 
     @MIN_RESPONSE_SIZE.setter  # type: ignore[no-redef]
-    def MIN_RESPONSE_SIZE(cls, value):
+    def MIN_RESPONSE_SIZE(cls, value):  # pylint: disable=no-self-argument
         warnings.warn(
             "scrapy.core.scraper.Slot.MIN_RESPONSE_SIZE is deprecated.",
             ScrapyDeprecationWarning,
@@ -95,7 +95,7 @@ class Slot(ClassPropertiesMixin):
                 ScrapyDeprecationWarning,
                 stacklevel=2,
             )
-        self._max_active_size = max_active_size
+        self._max_active_size: int = max_active_size
         self.queue: deque[QueueTuple] = deque()
         self.active: set[Request] = set()
         self.itemproc_size: int = 0  # just for scrapy.utils.engine.get_engine_status()
@@ -103,7 +103,7 @@ class Slot(ClassPropertiesMixin):
         self._active_size: int = 0
 
     @property
-    def active_size(self):
+    def active_size(self) -> int:
         warnings.warn(
             (
                 "scrapy.core.scraper.Slot.active_size is deprecated. Read "
@@ -116,7 +116,7 @@ class Slot(ClassPropertiesMixin):
         return self._active_size
 
     @active_size.setter
-    def active_size(self, value):
+    def active_size(self, value: int) -> None:
         warnings.warn(
             (
                 "scrapy.core.scraper.Slot.active_size is deprecated. "
@@ -134,7 +134,7 @@ class Slot(ClassPropertiesMixin):
         self._active_size = value
 
     @property
-    def max_active_size(self):
+    def max_active_size(self) -> int:
         warnings.warn(
             (
                 "scrapy.core.scraper.Slot.max_active_size is deprecated. Read "
@@ -146,7 +146,7 @@ class Slot(ClassPropertiesMixin):
         return self._max_active_size
 
     @max_active_size.setter
-    def max_active_size(self, value):
+    def max_active_size(self, value: int) -> None:
         warnings.warn(
             (
                 "scrapy.core.scraper.Slot.max_active_size is deprecated. Set "

tests/test_downloader.py

@@ -27,7 +27,7 @@ class OfflineSpider(Spider):
 
 
 class gt:
-    __hash__ = None
+    __hash__ = None  # type: ignore[assignment]
 
     def __init__(self, value):
         self.value = value