Move to mypy --strict with exceptions. (#7300)

* Move to mypy --strict with exceptions.

* Fix typing-tests.

pyproject.toml

@@ -85,8 +85,20 @@ path = "scrapy/VERSION"
 pattern = "^(?P<version>.+)$"
 
 [tool.mypy]
-ignore_missing_imports = true
-implicit_reexport = false
+strict = true
+allow_any_generics = true # 67 errors
+allow_untyped_calls = true  # 58 errors
+extra_checks = false # weird addErrback() errors
+untyped_calls_exclude = [
+    "twisted",
+]
+warn_return_any = false # 37 errors
+
+[[tool.mypy.overrides]]
+module = "tests.*"
+allow_untyped_defs = true
+allow_incomplete_defs = true # 48 errors
+check_untyped_defs = false
 
 # Interface classes are hard to support
 [[tool.mypy.overrides]]
@@ -101,14 +113,46 @@ ignore_errors = true
 module = "twisted.internet.reactor"
 follow_imports = "skip"
 
-# FIXME: remove the following section once the issues are solved
+# just for twisted.version
+[[tool.mypy.overrides]]
+module = "twisted"
+implicit_reexport = true
+
+# TODO
 [[tool.mypy.overrides]]
 module = "scrapy.settings.default_settings"
 ignore_errors = true
 
+# deprecated modules
 [[tool.mypy.overrides]]
-module = "twisted"
-implicit_reexport = true
+module = [
+    "scrapy.core.downloader.webclient",
+    "scrapy.spiders.init",
+    "scrapy.utils.testsite",
+    "tests.test_webclient",
+]
+allow_any_generics = true
+allow_untyped_calls = true
+allow_untyped_defs = true
+check_untyped_defs = false
+warn_return_any = false
+
+# usually no type hints
+[[tool.mypy.overrides]]
+module = [
+#    "IPython.*",
+    "bpython",
+    "brotli",
+    "brotlicffi",
+    "google.*",
+    "pydispatch.*",
+    "pyftpdlib.*",
+    "pytest_twisted",
+    "robotexclusionrulesparser",
+    "testfixtures",
+    "zope.interface.*",
+]
+ignore_missing_imports = true
 
 [tool.bumpversion]
 current_version = "2.14.1"

scrapy/addons.py

@@ -55,7 +55,7 @@ class AddonManager:
         )
 
     @classmethod
-    def load_pre_crawler_settings(cls, settings: BaseSettings):
+    def load_pre_crawler_settings(cls, settings: BaseSettings) -> None:
         """Update early settings that do not require a crawler instance, such as SPIDER_MODULES.
 
         Similar to the load_settings method, this loads each add-on configured in the

scrapy/commands/bench.py

@@ -43,7 +43,7 @@ class _BenchServer:
         assert self.proc.stdout
         self.proc.stdout.readline()
 
-    def __exit__(self, exc_type, exc_value, traceback) -> None:
+    def __exit__(self, exc_type, exc_value, traceback) -> None:  # type: ignore[no-untyped-def]
         self.proc.kill()
         self.proc.wait()
         time.sleep(0.2)

scrapy/commands/check.py

@@ -1,9 +1,12 @@
 import argparse
 import time
 from collections import defaultdict
+from collections.abc import AsyncIterator
+from typing import Any
 from unittest import TextTestResult as _TextTestResult
 from unittest import TextTestRunner
 
+from scrapy import Spider
 from scrapy.commands import ScrapyCommand
 from scrapy.contracts import ContractsManager
 from scrapy.utils.conf import build_component_list
@@ -81,14 +84,14 @@ class Command(ScrapyCommand):
         assert self.crawler_process
         spider_loader = self.crawler_process.spider_loader
 
-        async def start(self):
+        async def start(self: Spider) -> AsyncIterator[Any]:
             for request in conman.from_spider(self, result):
                 yield request
 
         with set_environ(SCRAPY_CHECK="true"):
             for spidername in args or spider_loader.list():
                 spidercls = spider_loader.load(spidername)
-                spidercls.start = start  # type: ignore[assignment,method-assign,return-value]
+                spidercls.start = start  # type: ignore[method-assign]
 
                 tested_methods = conman.tested_methods_from_spidercls(spidercls)
                 if opts.list:

scrapy/commands/fetch.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import sys
 from argparse import Namespace  # noqa: TC003
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from w3lib.url import is_url
 
@@ -14,6 +14,7 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request
 
 if TYPE_CHECKING:
     from argparse import ArgumentParser
+    from collections.abc import AsyncIterator
 
     from scrapy import Spider
 
@@ -89,10 +90,10 @@ class Command(ScrapyCommand):
         else:
             spidercls = spidercls_for_request(spider_loader, request, spidercls)
 
-        async def start(self):
+        async def start(self: Spider) -> AsyncIterator[Any]:
             yield request
 
-        spidercls.start = start  # type: ignore[method-assign,attr-defined]
+        spidercls.start = start  # type: ignore[method-assign]
 
         self.crawler_process.crawl(spidercls)
         self.crawler_process.start()

scrapy/contracts/__init__.py

@@ -189,7 +189,7 @@ class ContractsManager:
         def eb_wrapper(failure: Failure) -> None:
             case = _create_testcase(method, "errback")
             exc_info = failure.type, failure.value, failure.getTracebackObject()
-            results.addError(case, exc_info)
+            results.addError(case, exc_info)  # type: ignore[arg-type]
 
         request.callback = cb_wrapper
         request.errback = eb_wrapper

scrapy/core/downloader/handlers/_httpx.py

@@ -265,5 +265,5 @@ class HttpxDownloadHandler(BaseHttpDownloadHandler):
         logger.warning(warning_msg)
         raise DownloadCancelledError(warning_msg)
 
-    async def close(self):
+    async def close(self) -> None:
         await self._client.aclose()

scrapy/core/downloader/handlers/http11.py

@@ -507,7 +507,7 @@ class ScrapyAgent:
             }
 
         # deliverBody hangs for responses without body
-        if txresponse.length == 0:
+        if cast("int", txresponse.length) == 0:
             return {
                 "txresponse": txresponse,
             }
@@ -697,7 +697,8 @@ class _ResponseReader(Protocol):
             return
 
         if reason.check(ResponseFailed) and any(
-            r.check(_DataLoss) for r in reason.value.reasons
+            r.check(_DataLoss)
+            for r in reason.value.reasons  # type: ignore[union-attr]
         ):
             if not self._fail_on_dataloss:
                 self._finish_response(flags=["dataloss"])

scrapy/core/engine.py

@@ -160,7 +160,7 @@ class ExecutionEngine:
         return scheduler_cls
 
     def start(
-        self, _start_request_processing=True
+        self, _start_request_processing: bool = True
     ) -> Deferred[None]:  # pragma: no cover
         warnings.warn(
             "ExecutionEngine.start() is deprecated, use start_async() instead",
@@ -263,12 +263,13 @@ class ExecutionEngine:
     def unpause(self) -> None:
         self.paused = False
 
-    async def _process_start_next(self):
+    async def _process_start_next(self) -> None:
         """Processes the next item or request from Spider.start().
 
         If a request, it is scheduled. If an item, it is sent to item
         pipelines.
         """
+        assert self._start is not None
         try:
             item_or_request = await self._start.__anext__()
         except StopAsyncIteration:
@@ -286,6 +287,7 @@ class ExecutionEngine:
             if isinstance(item_or_request, Request):
                 self.crawl(item_or_request)
             else:
+                assert self._slot is not None
                 _schedule_coro(
                     self.scraper.start_itemproc_async(item_or_request, response=None)
                 )

scrapy/core/http2/protocol.py

@@ -4,7 +4,7 @@ import ipaddress
 import itertools
 import logging
 from collections import deque
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 
 from h2.config import H2Configuration
 from h2.connection import H2Connection
@@ -39,6 +39,7 @@ from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute
 if TYPE_CHECKING:
     from ipaddress import IPv4Address, IPv6Address
 
+    from hpack import HeaderTuple
     from twisted.internet.defer import Deferred
     from twisted.python.failure import Failure
     from twisted.web.client import URI
@@ -408,7 +409,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         except KeyError:
             pass  # We ignore server-initiated events
         else:
-            stream.receive_headers(event.headers)
+            stream.receive_headers(cast("list[HeaderTuple]", event.headers))
 
     def settings_acknowledged(self, event: SettingsAcknowledged) -> None:
         self.metadata["settings_acknowledged"] = True

scrapy/core/spidermw.py

@@ -460,7 +460,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             start = await self._process_chain("process_start", start)
         return start
 
-    def _check_deprecated_start_requests_use(self):
+    def _check_deprecated_start_requests_use(self) -> None:
         start_requests_cls = None
         start_cls = None
         spidercls = self._spider.__class__

scrapy/crawler.py

@@ -135,7 +135,7 @@ class Crawler:
             "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
         )
 
-    def _apply_reactorless_default_settings(self):
+    def _apply_reactorless_default_settings(self) -> None:
         """Change some setting defaults when not using a Twisted reactor.
 
         Some settings need different defaults when using and not using a

scrapy/extensions/debug.py

@@ -34,8 +34,8 @@ class StackTraceDump:
     def __init__(self, crawler: Crawler):
         self.crawler: Crawler = crawler
         try:
-            signal.signal(signal.SIGUSR2, self.dump_stacktrace)  # type: ignore[attr-defined]
-            signal.signal(signal.SIGQUIT, self.dump_stacktrace)  # type: ignore[attr-defined]
+            signal.signal(signal.SIGUSR2, self.dump_stacktrace)  # type: ignore[attr-defined,unused-ignore]
+            signal.signal(signal.SIGQUIT, self.dump_stacktrace)  # type: ignore[attr-defined,unused-ignore]
         except AttributeError:
             # win32 platforms don't support SIGUSR signals
             pass
@@ -72,7 +72,7 @@ class Debugger:
     def __init__(self) -> None:
         # win32 platforms don't support SIGUSR signals
         with contextlib.suppress(AttributeError):
-            signal.signal(signal.SIGUSR2, self._enter_debugger)  # type: ignore[attr-defined]
+            signal.signal(signal.SIGUSR2, self._enter_debugger)  # type: ignore[attr-defined,unused-ignore]
 
     def _enter_debugger(self, signum: int, frame: FrameType | None) -> None:
         assert frame

scrapy/extensions/feedexport.py

@@ -89,20 +89,20 @@ class ItemFilter:
         return True  # accept all items by default
 
 
-class IFeedStorage(Interface):
+class IFeedStorage(Interface):  # type: ignore[misc]
     """Interface that all Feed Storages must implement"""
 
     # pylint: disable=no-self-argument
 
-    def __init__(uri, *, feed_options=None):  # pylint: disable=super-init-not-called
+    def __init__(uri, *, feed_options=None):  # type: ignore[no-untyped-def]  # pylint: disable=super-init-not-called
         """Initialize the storage with the parameters given in the URI and the
         feed-specific options (see :setting:`FEEDS`)"""
 
-    def open(spider):
+    def open(spider):  # type: ignore[no-untyped-def]
         """Open the storage for the given spider. It must return a file-like
         object that will be used for the exporters"""
 
-    def store(file):
+    def store(file):  # type: ignore[no-untyped-def]
         """Store the given file stream"""
 
 

scrapy/extensions/telnet.py

@@ -86,7 +86,7 @@ class TelnetConsole(protocol.ServerFactory):
         class Portal:
             """An implementation of IPortal"""
 
-            def login(self_, credentials, mind, *interfaces):  # pylint: disable=no-self-argument
+            def login(self_, credentials, mind, *interfaces):  # type: ignore[no-untyped-def]  # pylint: disable=no-self-argument
                 if not (
                     credentials.username == self.username.encode("utf8")
                     and credentials.checkPassword(self.password.encode("utf8"))

scrapy/http/cookies.py

@@ -74,7 +74,7 @@ class CookieJar:
 
     @property
     def _cookies(self) -> dict[str, dict[str, dict[str, Cookie]]]:
-        return self.jar._cookies  # type: ignore[attr-defined,no-any-return]
+        return self.jar._cookies  # type: ignore[attr-defined]
 
     def clear_session_cookies(self) -> None:
         return self.jar.clear_session_cookies()

scrapy/http/request/form.py

@@ -108,7 +108,7 @@ def _urlencode(seq: Iterable[FormdataKVType], enc: str) -> str:
     values = [
         (to_bytes(k, enc), to_bytes(v, enc))
         for k, vs in seq
-        for v in (cast("Iterable[str]", vs) if is_listlike(vs) else [cast("str", vs)])
+        for v in (vs if is_listlike(vs) else [cast("str", vs)])
     ]
     return urlencode(values, doseq=True)
 

scrapy/pipelines/__init__.py

@@ -10,7 +10,7 @@ import asyncio
 import warnings
 from typing import TYPE_CHECKING, Any, cast
 
-from twisted.internet.defer import Deferred, DeferredList
+from twisted.internet.defer import Deferred, DeferredList, FirstError
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.middleware import MiddlewareManager
@@ -80,6 +80,7 @@ class ItemPipelineManager(MiddlewareManager):
         d2: Deferred[list[None]] = d.addCallback(lambda r: [x[1] for x in r])
 
         def eb(failure: Failure) -> Failure:
+            assert isinstance(failure.value, FirstError)
             return failure.value.subFailure
 
         d2.addErrback(eb)

scrapy/pipelines/files.py

@@ -305,7 +305,7 @@ class GCSFilesStore:
     def stat_file(
         self, path: str, info: MediaPipeline.SpiderInfo
     ) -> Deferred[StatInfo]:
-        def _onsuccess(blob) -> StatInfo:
+        def _onsuccess(blob: Any) -> StatInfo:
             if blob:
                 checksum = base64.b64decode(blob.md5_hash).hex()
                 last_modified = time.mktime(blob.updated.timetuple())

scrapy/pipelines/media.py

@@ -250,6 +250,7 @@ class MediaPipeline(ABC):
             # the encapsulated exception when it is a StopIteration instance
             context = getattr(result.value, "__context__", None)
             if isinstance(context, StopIteration):
+                assert result.value is not None
                 result.value.__context__ = None
 
         info.downloading.remove(fp)

scrapy/shell.py

@@ -104,7 +104,7 @@ class Shell:
             event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"]
             set_asyncio_event_loop(event_loop_path)
 
-        def crawl_request(_):
+        def crawl_request(_: None) -> None:
             assert self.crawler.engine is not None
             self.crawler.engine.crawl(request)
 

scrapy/signalmanager.py

@@ -101,14 +101,14 @@ class SignalManager:
         kwargs.setdefault("sender", self.sender)
         _signal.disconnect_all(signal, **kwargs)
 
-    async def wait_for(self, signal):
+    async def wait_for(self, signal: Any) -> None:
         """Await the next *signal*.
 
         See :ref:`start-requests-lazy` for an example.
         """
-        d = Deferred()
+        d: Deferred[None] = Deferred()
 
-        def handle():
+        def handle() -> None:
             self.disconnect(handle, signal)
             d.callback(None)
 

scrapy/statscollectors.py

@@ -27,7 +27,7 @@ class StatsCollector:
         self._stats: StatsT = {}
         self._crawler: Crawler = crawler
 
-    def __getattribute__(self, name):
+    def __getattribute__(self, name: str) -> Any:
         cached_name = f"_cached_{name}"
         try:
             return super().__getattribute__(cached_name)

scrapy/utils/benchserver.py

@@ -27,8 +27,10 @@ class Root(Resource):
         return b""
 
 
-def _getarg(request, name: bytes, default: Any = None, type_=str):
-    return type_(request.args[name][0]) if name in request.args else default
+def _getarg(
+    request: Request, name: bytes, default: Any = None, type_: type = str
+) -> Any:
+    return type_(request.args[name][0]) if name in request.args else default  # type: ignore[index,operator]
 
 
 if __name__ == "__main__":

scrapy/utils/console.py

@@ -20,10 +20,10 @@ def _embed_ipython_shell(
         from IPython.terminal.embed import InteractiveShellEmbed  # noqa: T100,PLC0415
         from IPython.terminal.ipapp import load_default_config  # noqa: PLC0415
     except ImportError:
-        from IPython.frontend.terminal.embed import (  # type: ignore[no-redef]  # noqa: T100,PLC0415
+        from IPython.frontend.terminal.embed import (  # type: ignore[import-not-found,no-redef]  # noqa: T100,PLC0415
             InteractiveShellEmbed,
         )
-        from IPython.frontend.terminal.ipapp import (  # type: ignore[no-redef]  # noqa: PLC0415
+        from IPython.frontend.terminal.ipapp import (  # type: ignore[import-not-found,no-redef]  # noqa: PLC0415
             load_default_config,
         )
 
@@ -81,7 +81,7 @@ def _embed_standard_shell(
     else:
         import rlcompleter  # noqa: F401,PLC0415
 
-        readline.parse_and_bind("tab:complete")  # type: ignore[attr-defined]
+        readline.parse_and_bind("tab:complete")  # type: ignore[attr-defined,unused-ignore]
 
     @wraps(_embed_standard_shell)
     def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:

scrapy/utils/datatypes.py

@@ -83,7 +83,7 @@ class CaselessDict(dict):
         return dict.get(self, self.normkey(key), self.normvalue(def_val))
 
     def setdefault(self, key: AnyStr, def_val: Any = None) -> Any:
-        return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))  # type: ignore[arg-type]
+        return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
 
     # doesn't fully implement MutableMapping.update()
     def update(self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]) -> None:  # type: ignore[override]
@@ -179,7 +179,7 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
         with contextlib.suppress(TypeError):
             super().__setitem__(key, value)
 
-    def __getitem__(self, key: _KT) -> _VT | None:  # type: ignore[override]
+    def __getitem__(self, key: _KT) -> _VT | None:
         try:
             return super().__getitem__(key)
         except (TypeError, KeyError):

scrapy/utils/defer.py

@@ -21,7 +21,7 @@ from typing import (
     overload,
 )
 
-from twisted.internet.defer import Deferred, DeferredList, fail, succeed
+from twisted.internet.defer import Deferred, DeferredList, FirstError, fail, succeed
 from twisted.internet.task import Cooperator
 from twisted.python import failure
 
@@ -334,6 +334,7 @@ def process_parallel(
     d2: Deferred[list[_T2]] = d.addCallback(lambda r: [x[1] for x in r])
 
     def eb(failure: Failure) -> Failure:
+        assert isinstance(failure.value, FirstError)
         return failure.value.subFailure
 
     d2.addErrback(eb)

scrapy/utils/deprecate.py

@@ -60,7 +60,7 @@ def create_deprecated_class(
     """
 
     # https://github.com/python/mypy/issues/4177
-    class DeprecatedClass(new_class.__class__):  # type: ignore[misc, name-defined]
+    class DeprecatedClass(new_class.__class__):  # type: ignore[misc,name-defined]
         # pylint: disable=no-self-argument
         deprecated_class: type | None = None
         warned_on_subclass: bool = False

scrapy/utils/log.py

@@ -189,7 +189,7 @@ def log_scrapy_info(settings: Settings) -> None:
         "Scrapy %(version)s started (bot: %(bot)s)",
         {"version": scrapy.__version__, "bot": settings["BOT_NAME"]},
     )
-    software = settings.getlist("LOG_VERSIONS")
+    software: list[str] = settings.getlist("LOG_VERSIONS")
     if not software:
         return
     versions = pprint.pformat(dict(get_versions(software)), sort_dicts=False)

scrapy/utils/reactor.py

@@ -80,11 +80,11 @@ class CallLaterOnce(Generic[_T]):
 
         return result
 
-    async def wait(self):
+    async def wait(self) -> None:
         # circular import
         from scrapy.utils.defer import maybe_deferred_to_future  # noqa: PLC0415
 
-        d = Deferred()
+        d: Deferred[None] = Deferred()
         self._deferreds.append(d)
         await maybe_deferred_to_future(d)
 

scrapy/utils/url.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 import re
 import warnings
 from importlib import import_module
-from typing import TYPE_CHECKING, TypeAlias
+from typing import TYPE_CHECKING, Any, TypeAlias
 from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
 from warnings import warn
 
@@ -20,7 +20,7 @@ from w3lib.url import parse_url as _parse_url
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
-def __getattr__(name: str):
+def __getattr__(name: str) -> Any:
     if name in ("_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects):
         obj_type = "attribute" if name == "_safe_chars" else "function"
         warnings.warn(

scrapy/utils/versions.py

@@ -11,10 +11,10 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings.default_settings import LOG_VERSIONS
 from scrapy.utils.ssl import get_openssl_version
 
-_DEFAULT_SOFTWARE = ["Scrapy", *LOG_VERSIONS]
+_DEFAULT_SOFTWARE: list[str] = ["Scrapy", *LOG_VERSIONS]
 
 
-def _version(item):
+def _version(item: str) -> str:
     lowercase_item = item.lower()
     if lowercase_item == "libxml2":
         return ".".join(map(str, lxml.etree.LIBXML_VERSION))
@@ -28,7 +28,7 @@ def _version(item):
 
 
 def get_versions(
-    software: list | None = None,
+    software: list[str] | None = None,
 ) -> list[tuple[str, str]]:
     software = software or _DEFAULT_SOFTWARE
     return [(item, _version(item)) for item in software]

tests/test_downloader_handler_twisted_ftp.py

@@ -47,7 +47,7 @@ class TestFTPBase(ABC):
     def _get_factory(self, tmp_path: Path) -> FTPFactory:
         raise NotImplementedError
 
-    @async_yield_fixture
+    @async_yield_fixture  # type: ignore[untyped-decorator]
     async def server_url(self, tmp_path: Path) -> AsyncGenerator[str]:
         from twisted.internet import reactor
 

tests/test_downloadermiddleware_httpproxy.py

@@ -10,8 +10,6 @@ from scrapy.utils.test import get_crawler
 
 
 class TestHttpProxyMiddleware:
-    failureException = AssertionError  # type: ignore[assignment]
-
     def setup_method(self):
         self._oldenv = os.environ.copy()
 

tests/test_http2_client_protocol.py

@@ -213,7 +213,7 @@ class TestHttps2ClientProtocol:
         r.putChild(b"request-headers", RequestHeaders())
         return Site(r, timeout=None)
 
-    @async_yield_fixture
+    @async_yield_fixture  # type: ignore[untyped-decorator]
     async def server_port(self, site: Site) -> AsyncGenerator[int]:
         from twisted.internet import reactor
 
@@ -236,7 +236,7 @@ class TestHttps2ClientProtocol:
         ) + self.certificate_file.read_text(encoding="utf-8")
         return PrivateCertificate.loadPEM(pem)
 
-    @async_yield_fixture
+    @async_yield_fixture  # type: ignore[untyped-decorator]
     async def client(
         self, server_port: int, client_certificate: PrivateCertificate
     ) -> AsyncGenerator[H2ClientProtocol]:

tests/test_utils_iterators.py

@@ -527,7 +527,7 @@ class TestBodyOrStr:
         r3 = _body_or_str(obj, unicode=False)
         self._assert_type_and_value(r3, self.bbody, obj)
         assert type(r1) is type(r2)
-        assert type(r1) is not type(r3)
+        assert type(r1) is not type(r3)  # type: ignore[comparison-overlap]
 
     @staticmethod
     def _assert_type_and_value(

tests_typing/test_http_request.mypy-testing

@@ -15,7 +15,7 @@ class MyRequest2(Request):
 
 
 @pytest.mark.mypy_testing
-def mypy_test_headers():
+def mypy_test_headers() -> None:
     Request("data:,", headers=1)  # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
     Request("data:,", headers=None)
     Request("data:,", headers={})
@@ -29,7 +29,7 @@ def mypy_test_headers():
 
 
 @pytest.mark.mypy_testing
-def mypy_test_copy():
+def mypy_test_copy() -> None:
     req = Request("data:,")
     reveal_type(req)  # R: scrapy.http.request.Request
     req_copy = req.copy()
@@ -37,7 +37,7 @@ def mypy_test_copy():
 
 
 @pytest.mark.mypy_testing
-def mypy_test_copy_subclass():
+def mypy_test_copy_subclass() -> None:
     req = MyRequest("data:,")
     reveal_type(req)  # R: __main__.MyRequest
     req_copy = req.copy()
@@ -45,7 +45,7 @@ def mypy_test_copy_subclass():
 
 
 @pytest.mark.mypy_testing
-def mypy_test_replace():
+def mypy_test_replace() -> None:
     req = Request("data:,")
     reveal_type(req)  # R: scrapy.http.request.Request
     req_copy = req.replace(body=b"a")
@@ -56,7 +56,7 @@ def mypy_test_replace():
 
 
 @pytest.mark.mypy_testing
-def mypy_test_replace_subclass():
+def mypy_test_replace_subclass() -> None:
     req = MyRequest("data:,")
     reveal_type(req)  # R: __main__.MyRequest
     req_copy = req.replace(body=b"a")
@@ -69,7 +69,7 @@ def mypy_test_replace_subclass():
 
 
 @pytest.mark.mypy_testing
-def mypy_test_jsonrequest_copy_replace():
+def mypy_test_jsonrequest_copy_replace() -> None:
     req = JsonRequest("data:,")
     reveal_type(req)  # R: scrapy.http.request.json_request.JsonRequest
     req_copy = req.copy()

tests_typing/test_http_response.mypy-testing

@@ -6,7 +6,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse
 
 
 @pytest.mark.mypy_testing
-def mypy_test_headers():
+def mypy_test_headers() -> None:
     Response("data:,", headers=1)  # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
     Response("data:,", headers=None)
     Response("data:,", headers={})
@@ -20,7 +20,7 @@ def mypy_test_headers():
 
 
 @pytest.mark.mypy_testing
-def mypy_test_copy():
+def mypy_test_copy() -> None:
     resp = Response("data:,")
     reveal_type(resp)  # R: scrapy.http.response.Response
     resp_copy = resp.copy()
@@ -28,7 +28,7 @@ def mypy_test_copy():
 
 
 @pytest.mark.mypy_testing
-def mypy_test_copy_subclass():
+def mypy_test_copy_subclass() -> None:
     resp = HtmlResponse("data:,")
     reveal_type(resp)  # R: scrapy.http.response.html.HtmlResponse
     resp_copy = resp.copy()
@@ -36,7 +36,7 @@ def mypy_test_copy_subclass():
 
 
 @pytest.mark.mypy_testing
-def mypy_test_replace():
+def mypy_test_replace() -> None:
     resp = Response("data:,")
     reveal_type(resp)  # R: scrapy.http.response.Response
     resp_copy = resp.replace(body=b"a")
@@ -47,7 +47,7 @@ def mypy_test_replace():
 
 
 @pytest.mark.mypy_testing
-def mypy_test_replace_subclass():
+def mypy_test_replace_subclass() -> None:
     resp = HtmlResponse("data:,")
     reveal_type(resp)  # R: scrapy.http.response.html.HtmlResponse
     resp_copy = resp.replace(body=b"a")

tox.ini

@@ -41,22 +41,27 @@ commands =
 [testenv:typing]
 basepython = python3.10
 deps =
-    mypy==1.18.2
+    mypy==1.19.1
     typing-extensions==4.15.0
-    types-defusedxml==0.7.0.20250822
-    types-lxml==2025.8.25
-    types-pexpect==4.9.0.20250916
-    types-Pygments==2.19.0.20250809
-    botocore-stubs==1.40.59
-    boto3-stubs[s3]==1.40.59
+    Pillow==12.1.1
+    Protego==0.6.0
+    attrs==25.4.0
+    boto3-stubs[s3]==1.42.59
+    botocore-stubs==1.42.41
+    h2==4.3.0
     httpx==0.28.1
-    itemadapter==0.12.2
-    Protego==0.5.0
-    w3lib==2.3.1
-    attrs >= 18.2.0
-    Pillow >= 10.3.0
-    pyOpenSSL >= 24.2.1
-    pytest >= 8.2.0
+    itemadapter==0.13.1
+    ptpython==3.0.32
+    ipython
+    pyOpenSSL==25.3.0
+    pytest==9.0.2
+    types-Pygments==2.19.0.20251121
+    types-defusedxml==0.7.0.20250822
+    types-lxml==2026.2.16
+    types-pexpect==4.9.0.20260127
+    uvloop==0.22.1
+    w3lib==2.4.0
+    zstandard==0.25.0
 commands =
     mypy {posargs:scrapy tests}
 
@@ -65,7 +70,7 @@ basepython = python3.10
 deps =
     {[test-requirements]deps}
     {[testenv:typing]deps}
-    pytest-mypy-testing==0.1.3
+    pytest-mypy-testing==0.2.0
 commands =
     pytest {posargs:tests_typing}