Full typing for scrapy/squeues.py.

scrapy/squeues.py

@@ -2,20 +2,28 @@
 Scheduler queues
 """
 
+from __future__ import annotations
+
 import marshal
 import pickle  # nosec
 from os import PathLike
 from pathlib import Path
-from typing import Union
+from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union
 
 from queuelib import queue
 
+from scrapy import Request
+from scrapy.crawler import Crawler
 from scrapy.utils.request import request_from_dict
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
-def _with_mkdir(queue_class):
+
+def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]:
     class DirectoriesCreated(queue_class):
-        def __init__(self, path: Union[str, PathLike], *args, **kwargs):
+        def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any):
             dirname = Path(path).parent
             if not dirname.exists():
                 dirname.mkdir(parents=True, exist_ok=True)
@@ -24,18 +32,23 @@ def _with_mkdir(queue_class):
     return DirectoriesCreated
 
 
-def _serializable_queue(queue_class, serialize, deserialize):
+def _serializable_queue(
+    queue_class: Type[queue.BaseQueue],
+    serialize: Callable[[Any], bytes],
+    deserialize: Callable[[bytes], Any],
+) -> Type[queue.BaseQueue]:
     class SerializableQueue(queue_class):
-        def push(self, obj):
+        def push(self, obj: Any) -> None:
             s = serialize(obj)
             super().push(s)
 
-        def pop(self):
+        def pop(self) -> Optional[Any]:
             s = super().pop()
             if s:
                 return deserialize(s)
+            return None
 
-        def peek(self):
+        def peek(self) -> Optional[Any]:
             """Returns the next object to be returned by :meth:`pop`,
             but without removing it from the queue.
 
@@ -50,31 +63,36 @@ def _serializable_queue(queue_class, serialize, deserialize):
                 ) from ex
             if s:
                 return deserialize(s)
+            return None
 
     return SerializableQueue
 
 
-def _scrapy_serialization_queue(queue_class):
+def _scrapy_serialization_queue(
+    queue_class: Type[queue.BaseQueue],
+) -> Type[queue.BaseQueue]:
     class ScrapyRequestQueue(queue_class):
-        def __init__(self, crawler, key):
+        def __init__(self, crawler: Crawler, key: str):
             self.spider = crawler.spider
             super().__init__(key)
 
         @classmethod
-        def from_crawler(cls, crawler, key, *args, **kwargs):
+        def from_crawler(
+            cls, crawler: Crawler, key: str, *args: Any, **kwargs: Any
+        ) -> Self:
             return cls(crawler, key)
 
-        def push(self, request):
-            request = request.to_dict(spider=self.spider)
-            return super().push(request)
+        def push(self, request: Request) -> None:
+            request_dict = request.to_dict(spider=self.spider)
+            super().push(request_dict)
 
-        def pop(self):
+        def pop(self) -> Optional[Request]:
             request = super().pop()
             if not request:
                 return None
             return request_from_dict(request, spider=self.spider)
 
-        def peek(self):
+        def peek(self) -> Optional[Request]:
             """Returns the next object to be returned by :meth:`pop`,
             but without removing it from the queue.
 
@@ -89,13 +107,15 @@ def _scrapy_serialization_queue(queue_class):
     return ScrapyRequestQueue
 
 
-def _scrapy_non_serialization_queue(queue_class):
+def _scrapy_non_serialization_queue(
+    queue_class: Type[queue.BaseQueue],
+) -> Type[queue.BaseQueue]:
     class ScrapyRequestQueue(queue_class):
         @classmethod
-        def from_crawler(cls, crawler, *args, **kwargs):
+        def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
             return cls()
 
-        def peek(self):
+        def peek(self) -> Optional[Any]:
             """Returns the next object to be returned by :meth:`pop`,
             but without removing it from the queue.
 
@@ -113,7 +133,7 @@ def _scrapy_non_serialization_queue(queue_class):
     return ScrapyRequestQueue
 
 
-def _pickle_serialize(obj):
+def _pickle_serialize(obj: Any) -> bytes:
     try:
         return pickle.dumps(obj, protocol=4)
     # Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s)