Disable mypy allow_any_generics. (#7532)

pyproject.toml

@@ -86,7 +86,6 @@ pattern = "^(?P<version>.+)$"
 
 [tool.mypy]
 strict = true
-allow_any_generics = true # 67 errors
 extra_checks = false # weird addErrback() errors
 untyped_calls_exclude = [
     "twisted",

scrapy/commands/genspider.py

@@ -147,7 +147,7 @@ class Command(ScrapyCommand):
         name: str,
         url: str,
         template_name: str,
-        template_file: str | os.PathLike,
+        template_file: str | os.PathLike[str],
     ) -> None:
         """Generate the spider module, based on the given template"""
         assert self.settings is not None

scrapy/contracts/__init__.py

@@ -27,7 +27,7 @@ class Contract:
     request_cls: type[Request] | None = None
     name: str
 
-    def __init__(self, method: Callable, *args: Any):
+    def __init__(self, method: Callable[..., Any], *args: Any):
         self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook")
         self.testcase_post = _create_testcase(method, f"@{self.name} post-hook")
         self.args: tuple[Any, ...] = args
@@ -105,7 +105,7 @@ class ContractsManager:
 
         return methods
 
-    def extract_contracts(self, method: Callable) -> list[Contract]:
+    def extract_contracts(self, method: Callable[..., Any]) -> list[Contract]:
         contracts: list[Contract] = []
         assert method.__doc__ is not None
         for line_ in method.__doc__.split("\n"):
@@ -134,7 +134,9 @@ class ContractsManager:
 
         return requests
 
-    def from_method(self, method: Callable, results: TestResult) -> Request | None:
+    def from_method(
+        self, method: Callable[..., Any], results: TestResult
+    ) -> Request | None:
         contracts = self.extract_contracts(method)
         if contracts:
             request_cls = Request
@@ -170,7 +172,7 @@ class ContractsManager:
         return None
 
     def _clean_req(
-        self, request: Request, method: Callable, results: TestResult
+        self, request: Request, method: Callable[..., Any], results: TestResult
     ) -> None:
         """stop the request from returning objects and records any errors"""
 
@@ -195,7 +197,7 @@ class ContractsManager:
         request.errback = eb_wrapper
 
 
-def _create_testcase(method: Callable, desc: str) -> TestCase:
+def _create_testcase(method: Callable[..., Any], desc: str) -> TestCase:
     spider = method.__self__.name  # type: ignore[attr-defined]
 
     class ContractTestCase(TestCase):

scrapy/core/downloader/handlers/s3.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, cast
 
 from scrapy.core.downloader.handlers.base import BaseDownloadHandler
 from scrapy.exceptions import NotConfigured
@@ -9,6 +9,8 @@ from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import build_from_crawler, load_object
 
 if TYPE_CHECKING:
+    from collections.abc import Mapping
+
     from scrapy import Request
     from scrapy.crawler import Crawler
     from scrapy.http import Response
@@ -59,7 +61,7 @@ class S3DownloadHandler(BaseDownloadHandler):
             awsrequest = botocore.awsrequest.AWSRequest(
                 method=request.method,
                 url=f"{scheme}://s3.amazonaws.com/{bucket}{path}",
-                headers=request.headers.to_unicode_dict(),
+                headers=cast("Mapping[str, Any]", request.headers.to_unicode_dict()),
                 data=request.body,
             )
             assert self._signer

scrapy/core/downloader/middleware.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 
 import warnings
 from functools import wraps
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, Any
 
 from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
 from scrapy.http import Request, Response
@@ -87,7 +87,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             result = await self._process_exception(ex, request)
         return await self._process_response(result, request)
 
-    def _handle_mw_method(self, method: Callable, **kwargs: Any) -> Any:
+    def _handle_mw_method(self, method: Callable[..., Any], **kwargs: Any) -> Any:
         if method in self._mw_methods_requiring_spider:
             kwargs["spider"] = self._spider
 
@@ -99,7 +99,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         download_func: Callable[[Request], Coroutine[Any, Any, Response]],
     ) -> Response | Request:
         for method in self.methods["process_request"]:
-            method = cast("Callable", method)
+            assert method is not None
             response = await ensure_awaitable(
                 self._handle_mw_method(method, request=request),
                 _warn=global_object_name(method),
@@ -122,7 +122,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             return response
 
         for method in self.methods["process_response"]:
-            method = cast("Callable", method)
+            assert method is not None
             response = await ensure_awaitable(
                 self._handle_mw_method(method, request=request, response=response),
                 _warn=global_object_name(method),
@@ -141,7 +141,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         self, exception: Exception, request: Request | Response
     ) -> Response | Request:
         for method in self.methods["process_exception"]:
-            method = cast("Callable", method)
+            assert method is not None
             response = await ensure_awaitable(
                 self._handle_mw_method(method, request=request, exception=exception),
                 _warn=global_object_name(method),

scrapy/core/spidermw.py

@@ -11,7 +11,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
 from functools import wraps
 from inspect import isasyncgenfunction
 from itertools import islice
-from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast
+from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar
 from warnings import warn
 
 from twisted.python.failure import Failure
@@ -79,7 +79,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         request: Request,
     ) -> Iterable[_T] | AsyncIterator[_T]:
         for method in self.methods["process_spider_input"]:
-            method = cast("Callable", method)
+            assert method is not None
             try:
                 if method in self._mw_methods_requiring_spider:
                     result = method(response=response, spider=self._spider)
@@ -248,9 +248,13 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
     # This method is only needed until _async compatibility methods are removed.
     @staticmethod
-    def _get_process_spider_output(mw: Any) -> Callable | None:
-        normal_method: Callable | None = getattr(mw, "process_spider_output", None)
-        async_method: Callable | None = getattr(mw, "process_spider_output_async", None)
+    def _get_process_spider_output(mw: Any) -> Callable[..., Any] | None:
+        normal_method: Callable[..., Any] | None = getattr(
+            mw, "process_spider_output", None
+        )
+        async_method: Callable[..., Any] | None = getattr(
+            mw, "process_spider_output_async", None
+        )
         if not async_method:
             if normal_method and not isasyncgenfunction(normal_method):
                 raise TypeError(

scrapy/extensions/httpcache.py

@@ -313,7 +313,9 @@ class FilesystemCacheStorage:
         self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS")
         self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP")
         # https://github.com/python/mypy/issues/10740
-        self._open: Callable[Concatenate[str | os.PathLike, str, ...], IO[bytes]] = (
+        self._open: Callable[
+            Concatenate[str | os.PathLike[str], str, ...], IO[bytes]
+        ] = (
             gzip.open if self.use_gzip else open  # type: ignore[assignment]
         )
 

scrapy/http/response/text.py

@@ -220,7 +220,7 @@ class TextResponse(Response):
 
     def follow_all(
         self,
-        urls: Iterable[str | Link] | parsel.SelectorList | None = None,
+        urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None,
         callback: CallbackT | None = None,
         method: str = "GET",
         headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,

scrapy/middleware.py

@@ -50,8 +50,8 @@ class MiddlewareManager(ABC):
             )
         self.middlewares: tuple[Any, ...] = middlewares
         # Only process_spider_output and process_spider_exception can be None.
-        self.methods: dict[str, deque[Callable | None]] = defaultdict(deque)
-        self._mw_methods_requiring_spider: set[Callable] = set()
+        self.methods: dict[str, deque[Callable[..., Any] | None]] = defaultdict(deque)
+        self._mw_methods_requiring_spider: set[Callable[..., Any]] = set()
         for mw in middlewares:
             self._add_middleware(mw)
 
@@ -116,7 +116,7 @@ class MiddlewareManager(ABC):
     def _add_middleware(self, mw: Any) -> None:  # noqa: B027
         pass
 
-    def _check_mw_method_spider_arg(self, method: Callable) -> None:
+    def _check_mw_method_spider_arg(self, method: Callable[..., Any]) -> None:
         if argument_is_required(method, "spider"):
             warnings.warn(
                 f"{method.__qualname__}() requires a spider argument,"

scrapy/spiders/crawl.py

@@ -46,7 +46,9 @@ def _identity_process_request(request: Request, response: Response) -> Request |
     return request
 
 
-def _get_method(method: Callable | str | None, spider: Spider) -> Callable | None:
+def _get_method(
+    method: Callable[..., Any] | str | None, spider: Spider
+) -> Callable[..., Any] | None:
     if callable(method):
         return method
     if isinstance(method, str):

scrapy/squeues.py

@@ -26,7 +26,7 @@ if TYPE_CHECKING:
 
 def _with_mkdir(queue_class: type[queue.BaseQueue]) -> type[queue.BaseQueue]:
     class DirectoriesCreated(queue_class):  # type: ignore[valid-type,misc]
-        def __init__(self, path: str | PathLike, *args: Any, **kwargs: Any):
+        def __init__(self, path: str | PathLike[str], *args: Any, **kwargs: Any):
             dirname = Path(path).parent
             if not dirname.exists():
                 dirname.mkdir(parents=True, exist_ok=True)

scrapy/utils/asyncio.py

@@ -148,7 +148,7 @@ class AsyncioLoopingCall:
         self._func: Callable[_P, _T] = func
         self._args: tuple[Any, ...] = args
         self._kwargs: dict[str, Any] = kwargs
-        self._task: asyncio.Task | None = None
+        self._task: asyncio.Task[None] | None = None
         self.interval: float | None = None
         self._start_time: float | None = None
 

scrapy/utils/conf.py

@@ -71,8 +71,8 @@ def arglist_to_dict(arglist: list[str]) -> dict[str, str]:
 
 
 def closest_scrapy_cfg(
-    path: str | os.PathLike = ".",
-    prevpath: str | os.PathLike | None = None,
+    path: str | os.PathLike[str] = ".",
+    prevpath: str | os.PathLike[str] | None = None,
 ) -> str:
     """Return the path to the closest scrapy.cfg file by traversing the current
     directory and its parents

scrapy/utils/datatypes.py

@@ -28,7 +28,7 @@ _KT = TypeVar("_KT")
 _VT = TypeVar("_VT")
 
 
-class CaselessDict(dict):
+class CaselessDict(dict):  # type: ignore[type-arg]
     __slots__ = ()
 
     def __new__(cls, *args: Any, **kwargs: Any) -> Self:
@@ -99,20 +99,20 @@ class CaselessDict(dict):
         return dict.pop(self, self.normkey(key), *args)
 
 
-class CaseInsensitiveDict(collections.UserDict):
+class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):
     """A dict-like structure that accepts strings or bytes
     as keys and allows case-insensitive lookups.
     """
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
-        self._keys: dict = {}
+        self._keys: dict[str | bytes, Any] = {}
         super().__init__(*args, **kwargs)
 
-    def __getitem__(self, key: AnyStr) -> Any:
+    def __getitem__(self, key: str | bytes) -> Any:
         normalized_key = self._normkey(key)
         return super().__getitem__(self._keys[normalized_key.lower()])
 
-    def __setitem__(self, key: AnyStr, value: Any) -> None:
+    def __setitem__(self, key: str | bytes, value: Any) -> None:
         normalized_key = self._normkey(key)
         try:
             lower_key = self._keys[normalized_key.lower()]
@@ -122,19 +122,19 @@ class CaseInsensitiveDict(collections.UserDict):
         super().__setitem__(normalized_key, self._normvalue(value))
         self._keys[normalized_key.lower()] = normalized_key
 
-    def __delitem__(self, key: AnyStr) -> None:
+    def __delitem__(self, key: str | bytes) -> None:
         normalized_key = self._normkey(key)
         stored_key = self._keys.pop(normalized_key.lower())
         super().__delitem__(stored_key)
 
-    def __contains__(self, key: AnyStr) -> bool:  # type: ignore[override]
+    def __contains__(self, key: str | bytes) -> bool:  # type: ignore[override]
         normalized_key = self._normkey(key)
         return normalized_key.lower() in self._keys
 
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__}: {super().__repr__()}>"
 
-    def _normkey(self, key: AnyStr) -> AnyStr:
+    def _normkey(self, key: str | bytes) -> str | bytes:
         return key
 
     def _normvalue(self, value: Any) -> Any:
@@ -158,7 +158,7 @@ class LocalCache(OrderedDict[_KT, _VT]):
         super().__setitem__(key, value)
 
 
-class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
+class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]):
     """
     A weakref.WeakKeyDictionary implementation that uses LocalCache as its
     underlying data structure, making it ordered and capable of being size-limited.
@@ -172,9 +172,9 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
 
     def __init__(self, limit: int | None = None):
         super().__init__()
-        self.data: LocalCache = LocalCache(limit=limit)
+        self.data: LocalCache[_KT, _VT] = LocalCache(limit=limit)
 
-    def __setitem__(self, key: _KT, value: _VT) -> None:
+    def __setitem__(self, key: _KT, value: _VT | None) -> None:
         # if raised, key is not weak-referenceable, skip caching
         with contextlib.suppress(TypeError):
             super().__setitem__(key, value)

scrapy/utils/defer.py

@@ -173,7 +173,7 @@ def parallel(
     return DeferredList([coop.coiterate(work) for _ in range(count)])
 
 
-class _AsyncCooperatorAdapter(Iterator, Generic[_T]):
+class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_T]):
     """A class that wraps an async iterable into a normal iterator suitable
     for using in Cooperator.coiterate(). As it's only needed for parallel_async(),
     it calls the callable directly in the callback, instead of providing a more

scrapy/utils/log.py

@@ -260,7 +260,8 @@ def logformatter_adapter(
     return (level, message, args)
 
 
-class SpiderLoggerAdapter(logging.LoggerAdapter):
+# LoggerAdapter is only parameterized since Python 3.11
+class SpiderLoggerAdapter(logging.LoggerAdapter):  # type: ignore[type-arg]
     def process(
         self, msg: str, kwargs: MutableMapping[str, Any]
     ) -> tuple[str, MutableMapping[str, Any]]:

scrapy/utils/misc.py

@@ -28,7 +28,7 @@ if TYPE_CHECKING:
 
 
 _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
-_ITER_T = TypeVar("_ITER_T", bound=dict | Item | str | bytes)
+_ITER_T = TypeVar("_ITER_T", bound=dict[Any, Any] | Item | str | bytes)
 _T = TypeVar("_T")
 _T_co = TypeVar("_T_co", covariant=True)
 _P = ParamSpec("_P")
@@ -252,7 +252,9 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]:
         yield node
 
 
-_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
+_generator_callbacks_cache: LocalWeakReferencedCache[Callable[..., Any], bool] = (
+    LocalWeakReferencedCache(limit=128)
+)
 
 
 def _returns_none(return_node: ast.Return) -> bool:

scrapy/utils/reactor.py

@@ -55,7 +55,7 @@ class CallLaterOnce(Generic[_T]):
         self._a: tuple[Any, ...] = a
         self._kw: dict[str, Any] = kw
         self._call: CallLaterResult | None = None
-        self._deferreds: list[Deferred] = []
+        self._deferreds: list[Deferred[None]] = []
 
     def schedule(self, delay: float = 0) -> None:
         # circular import

scrapy/utils/signal.py

@@ -186,7 +186,9 @@ async def _send_catch_log_asyncio(
     handlers: list[Awaitable[TypingAny]] = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
 
-        async def handler(receiver: Callable) -> tuple[Callable, TypingAny]:
+        async def handler(
+            receiver: Callable[..., Any],
+        ) -> tuple[Callable[..., Any], TypingAny]:
             result: TypingAny
             try:
                 result = await ensure_awaitable(

scrapy/utils/template.py

@@ -11,7 +11,7 @@ if TYPE_CHECKING:
     from os import PathLike
 
 
-def render_templatefile(path: str | PathLike, **kwargs: Any) -> None:
+def render_templatefile(path: str | PathLike[str], **kwargs: Any) -> None:
     path_obj = Path(path)
     raw = path_obj.read_text("utf8")
 

scrapy/utils/trackref.py

@@ -30,7 +30,9 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
 
-live_refs: defaultdict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary)
+live_refs: defaultdict[type, WeakKeyDictionary[object, float]] = defaultdict(
+    WeakKeyDictionary
+)
 
 
 class object_ref:

tests/AsyncCrawlerProcess/reactorless_custom_settings.py

@@ -23,7 +23,7 @@ class NoRequestsSpider(scrapy.Spider):
         yield
 
 
-def log_task_exception(task: Task) -> None:
+def log_task_exception(task: Task[None]) -> None:
     try:
         task.result()
     except Exception:

tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py

@@ -17,7 +17,7 @@ class AsyncioReactorSpider(scrapy.Spider):
     }
 
 
-def log_task_exception(task: Task) -> None:
+def log_task_exception(task: Task[None]) -> None:
     try:
         task.result()
     except Exception:

tests/mocks/dummydbm.py

@@ -4,7 +4,7 @@ from collections import defaultdict
 from typing import Any
 
 
-class DummyDB(dict):
+class DummyDB(dict):  # type: ignore[type-arg]
     """Provide dummy DBM-like interface."""
 
     def close(self):

tests/spiders.py

@@ -22,6 +22,7 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
 from scrapy.utils.test import get_from_asyncio_queue
 
 if TYPE_CHECKING:
+    from scrapy.settings import _SettingsKey
     from tests.mockserver.http import MockServer
 
 
@@ -94,19 +95,19 @@ class DelaySpider(MetaSpider):
 class LogSpider(MetaSpider):
     name = "log_spider"
 
-    def log_debug(self, message: str, extra: dict | None = None):
+    def log_debug(self, message: str, extra: dict[str, Any] | None = None):
         self.logger.debug(message, extra=extra)
 
-    def log_info(self, message: str, extra: dict | None = None):
+    def log_info(self, message: str, extra: dict[str, Any] | None = None):
         self.logger.info(message, extra=extra)
 
-    def log_warning(self, message: str, extra: dict | None = None):
+    def log_warning(self, message: str, extra: dict[str, Any] | None = None):
         self.logger.warning(message, extra=extra)
 
-    def log_error(self, message: str, extra: dict | None = None):
+    def log_error(self, message: str, extra: dict[str, Any] | None = None):
         self.logger.error(message, extra=extra)
 
-    def log_critical(self, message: str, extra: dict | None = None):
+    def log_critical(self, message: str, extra: dict[str, Any] | None = None):
         self.logger.critical(message, extra=extra)
 
     def parse(self, response):
@@ -417,7 +418,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
     """
 
     name = "crawl_spider_with_parse_method"
-    custom_settings: dict = {
+    custom_settings: dict[_SettingsKey, Any] = {
         "RETRY_HTTP_CODES": [],  # no need to retry
     }
     rules = (Rule(LinkExtractor(), callback="parse", follow=True),)

tests/test_command_genspider.py

@@ -9,7 +9,7 @@ from tests.test_commands import TestProjectBase
 from tests.utils.cmdline import call, proc
 
 
-def find_in_file(filename: Path, regex: str) -> re.Match | None:
+def find_in_file(filename: Path, regex: str) -> re.Match[str] | None:
     """Find first pattern occurrence in file"""
     pattern = re.compile(regex)
     with filename.open("r", encoding="utf-8") as f:

tests/test_command_shell.py

@@ -4,7 +4,7 @@ import os
 import sys
 from io import BytesIO
 from pathlib import Path
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING
 
 import pytest
 from pexpect.popen_spawn import PopenSpawn
@@ -151,8 +151,7 @@ class TestInteractiveShell:
         env = os.environ.copy()
         env["SCRAPY_PYTHON_SHELL"] = "python"
         logfile = BytesIO()
-        # https://github.com/python/typeshed/issues/14915
-        p = PopenSpawn(args, env=cast("os._Environ", env), timeout=5)
+        p = PopenSpawn(args, env=env, timeout=5)
         p.logfile_read = logfile
         p.expect_exact("Available Scrapy objects")
         p.sendline(f"fetch('{mockserver.url('/')}')")

tests/test_command_startproject.py

@@ -79,7 +79,7 @@ class TestStartprojectCommand:
 
 
 def get_permissions_dict(
-    path: str | os.PathLike, renamings=None, ignore=None
+    path: str | os.PathLike[str], renamings=None, ignore=None
 ) -> dict[str, str]:
     def get_permissions(path: Path) -> str:
         return oct(path.stat().st_mode)

tests/test_feedexport.py

@@ -41,7 +41,7 @@ from tests.spiders import ItemSpider
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
 
 if TYPE_CHECKING:
-    from collections.abc import Callable, Iterable
+    from collections.abc import Awaitable, Callable, Iterable
 
 
 def path_to_url(path: str | Path) -> str:
@@ -1312,7 +1312,9 @@ class TestFeedExporterSignals:
         self.feed_slot_closed_received = True
 
     async def run_signaled_feed_exporter(
-        self, feed_exporter_signal_handler: Callable, feed_slot_signal_handler: Callable
+        self,
+        feed_exporter_signal_handler: Callable[[], Awaitable[None] | None],
+        feed_slot_signal_handler: Callable[[Any], Awaitable[None] | None],
     ) -> None:
         crawler = get_crawler(settings_dict=self.settings)
         feed_exporter = FeedExporter.from_crawler(crawler)

tests/test_feedexport_batch.py

@@ -29,7 +29,7 @@ if TYPE_CHECKING:
     from os import PathLike
 
 
-def build_url(path: str | PathLike) -> str:
+def build_url(path: str | PathLike[str]) -> str:
     path_str = str(path)
     if path_str[0] != "/":
         path_str = "/" + path_str

tests/test_http2_client_protocol.py

@@ -61,7 +61,7 @@ def make_html_body(val: str) -> bytes:
 
 class DummySpider(Spider):
     name = "dummy"
-    start_urls: list = []
+    start_urls = []
 
     def parse(self, response):
         print(response)

tests/test_loader.py

@@ -39,7 +39,7 @@ class AttrsNameItem:
 
 @dataclasses.dataclass
 class NameDataClass:
-    name: list = dataclasses.field(default_factory=list)
+    name: list[str] = dataclasses.field(default_factory=list)
 
 
 # test item loaders

tests/test_pipeline_files.py

@@ -383,11 +383,11 @@ class TestFilesPipelineFieldsItem(TestFilesPipelineFieldsMixin):
 class FilesPipelineTestDataClass:
     name: str
     # default fields
-    file_urls: list = dataclasses.field(default_factory=list)
-    files: list = dataclasses.field(default_factory=list)
+    file_urls: list[str] = dataclasses.field(default_factory=list)
+    files: list[dict[str, str]] = dataclasses.field(default_factory=list)
     # overridden fields
-    custom_file_urls: list = dataclasses.field(default_factory=list)
-    custom_files: list = dataclasses.field(default_factory=list)
+    custom_file_urls: list[str] = dataclasses.field(default_factory=list)
+    custom_files: list[dict[str, str]] = dataclasses.field(default_factory=list)
 
 
 class TestFilesPipelineFieldsDataClass(TestFilesPipelineFieldsMixin):

tests/test_pipeline_images.py

@@ -314,11 +314,11 @@ class TestImagesPipelineFieldsItem(TestImagesPipelineFieldsMixin):
 class ImagesPipelineTestDataClass:
     name: str
     # default fields
-    image_urls: list = dataclasses.field(default_factory=list)
-    images: list = dataclasses.field(default_factory=list)
+    image_urls: list[str] = dataclasses.field(default_factory=list)
+    images: list[dict[str, str]] = dataclasses.field(default_factory=list)
     # overridden fields
-    custom_image_urls: list = dataclasses.field(default_factory=list)
-    custom_images: list = dataclasses.field(default_factory=list)
+    custom_image_urls: list[str] = dataclasses.field(default_factory=list)
+    custom_images: list[dict[str, str]] = dataclasses.field(default_factory=list)
 
 
 class TestImagesPipelineFieldsDataClass(TestImagesPipelineFieldsMixin):

tests/test_scheduler_base.py

@@ -41,10 +41,10 @@ class MinimalScheduler:
 
 
 class SimpleScheduler(MinimalScheduler):
-    def open(self, spider: Spider) -> defer.Deferred:
+    def open(self, spider: Spider) -> defer.Deferred[str]:
         return defer.succeed("open")
 
-    def close(self, reason: str) -> defer.Deferred:
+    def close(self, reason: str) -> defer.Deferred[str]:
         return defer.succeed("close")
 
     def __len__(self) -> int:

tests/test_spidermiddleware.py

@@ -110,7 +110,7 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
     Should work for process_spider_output and, when it's supported, process_start.
     """
 
-    ITEM_TYPE: type | tuple
+    ITEM_TYPE: type | tuple[type, ...]
     RESULT_COUNT = 3  # to simplify checks, let everything return 3 objects
 
     @staticmethod

tests/test_utils_datatypes.py

@@ -2,6 +2,7 @@ import copy
 import warnings
 from abc import ABC, abstractmethod
 from collections.abc import Iterator, Mapping, MutableMapping
+from typing import Any
 
 import pytest
 
@@ -20,7 +21,7 @@ from scrapy.utils.python import garbage_collect
 class TestCaseInsensitiveDictBase(ABC):
     @property
     @abstractmethod
-    def dict_class(self) -> type[MutableMapping]:
+    def dict_class(self) -> type[MutableMapping[str, Any]]:
         raise NotImplementedError
 
     def test_init_dict(self):
@@ -206,7 +207,7 @@ class TestCaseInsensitiveDictBase(ABC):
 
 
 class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
-    dict_class = CaseInsensitiveDict
+    dict_class = CaseInsensitiveDict  # type: ignore[assignment]
 
     def test_repr(self):
         d1 = self.dict_class({"foo": "bar"})

tests/test_utils_log.py

@@ -144,7 +144,9 @@ class TestStreamLogger:
     ],
 )
 def test_spider_logger_adapter_process(
-    base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict
+    base_extra: Mapping[str, Any],
+    log_extra: MutableMapping[str, Any],
+    expected_extra: dict[str, Any],
 ) -> None:
     logger = logging.getLogger("test")
     spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra)

tests/test_utils_request.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import json
 from hashlib import sha1
+from typing import Any
 from weakref import WeakKeyDictionary
 
 import pytest
@@ -56,13 +57,13 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None:
 
 
 class TestFingerprint:
-    function: staticmethod = staticmethod(fingerprint)
+    function: staticmethod[[Request], bytes] = staticmethod(fingerprint)
     cache: (
         WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]]
         | WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], str]]
     ) = _fingerprint_cache
     default_cache_key = (None, False)
-    known_hashes: tuple[tuple[Request, bytes | str, dict], ...] = (
+    known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = (
         (
             Request("http://example.org"),
             b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD",

tox.ini

@@ -44,13 +44,13 @@ commands =
 [testenv:typing]
 basepython = python3.10
 deps =
-    mypy==1.20.2
+    mypy==2.1.0
     typing-extensions==4.15.0
     Pillow==12.2.0
     Protego==0.6.0
     Twisted==26.4.0
     attrs==26.1.0
-    boto3-stubs[s3]==1.43.2
+    boto3-stubs[s3]==1.43.9
     botocore-stubs==1.42.41
     h2==4.3.0
     httpx==0.28.1
@@ -58,12 +58,12 @@ deps =
     ptpython==3.0.32
     # newer ones require newer Python
     ipython==8.39.0
-    pyOpenSSL==26.1.0
+    pyOpenSSL==26.2.0
     pytest==9.0.3
-    types-Pygments==2.20.0.20260408
+    types-Pygments==2.20.0.20260508
     types-defusedxml==0.7.0.20260504
     types-lxml==2026.2.16
-    types-pexpect==4.9.0.20260408
+    types-pexpect==4.9.0.20260508
     uvloop==0.22.1
     w3lib==2.4.1
     zstandard==0.25.0