Enable mypy warn_return_any. (#7492)

pyproject.toml

@@ -91,7 +91,6 @@ extra_checks = false # weird addErrback() errors
 untyped_calls_exclude = [
     "twisted",
 ]
-warn_return_any = false # 37 errors
 
 [[tool.mypy.overrides]]
 module = "tests.*"

scrapy/commands/parse.py

@@ -144,17 +144,16 @@ class Command(BaseRunSpiderCommand):
     def iterate_spider_output(self, result: _T) -> Iterable[Any]: ...
 
     def iterate_spider_output(self, result: Any) -> Iterable[Any] | Deferred[Any]:
+        d: Deferred[Any]
         if inspect.isasyncgen(result):
             d = deferred_from_coro(
                 collect_asyncgen(aiter_errback(result, self.handle_exception))
             )
-            d.addCallback(self.iterate_spider_output)
-            return d
-        if inspect.iscoroutine(result):
+            return d.addCallback(self.iterate_spider_output)
         d = deferred_from_coro(result)
-            d.addCallback(self.iterate_spider_output)
-            return d
-        return arg_to_iter(deferred_from_coro(result))
+        if inspect.iscoroutine(result):
+            return d.addCallback(self.iterate_spider_output)
+        return arg_to_iter(d)
 
     def add_items(self, lvl: int, new_items: list[Any]) -> None:
         old_items = self.items.get(lvl, [])

scrapy/core/downloader/__init__.py

@@ -128,11 +128,12 @@ class Downloader:
     ) -> Generator[Deferred[Any], Any, Response | Request]:
         self.active.add(request)
         try:
-            return (
-                yield deferred_from_coro(
+            result: Response | Request = yield (
+                deferred_from_coro(
                     self.middleware.download_async(self._enqueue_request, request)
                 )
             )
+            return result
         finally:
             self.active.remove(request)
 
@@ -163,7 +164,8 @@ class Downloader:
         return key, self.slots[key]
 
     def get_slot_key(self, request: Request) -> str:
-        if (meta_slot := request.meta.get(self.DOWNLOAD_SLOT)) is not None:
+        meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT)
+        if meta_slot is not None:
             return meta_slot
 
         key = urlparse_cached(request).hostname or ""

scrapy/core/downloader/contextfactory.py

@@ -108,7 +108,7 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
 
     def _get_cert_options(self) -> CertificateOptions:
         with _filter_method_warning():
-            return CertificateOptions(
+            return CertificateOptions(  # type: ignore[no-any-return]
                 method=self._ssl_method,
                 fixBrokenPeers=True,
                 acceptableCiphers=self.tls_ciphers,
@@ -122,7 +122,7 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
 
     def _get_context(self) -> SSL.Context:
         cert_options = self._get_cert_options()
-        ctx = cert_options.getContext()
+        ctx: SSL.Context = cert_options.getContext()
         ctx.set_options(0x4)  # OP_LEGACY_SERVER_CONNECT
         return ctx
 
@@ -135,7 +135,7 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         # Otherwise use the normal Twisted function.
         # Note that this doesn't use self._get_context().
         with _filter_method_warning():
-            return optionsForClientTLS(
+            return optionsForClientTLS(  # type: ignore[no-any-return]
                 hostname=hostname.decode("ascii"),
                 extraCertificateOptions={
                     "method": self._ssl_method,
@@ -186,7 +186,7 @@ class BrowserLikeContextFactory(_ScrapyClientContextFactory):
 
     def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
         with _filter_method_warning():
-            return optionsForClientTLS(
+            return optionsForClientTLS(  # type: ignore[no-any-return]
                 hostname=hostname.decode("ascii"),
                 extraCertificateOptions={"method": self._ssl_method},
             )

scrapy/core/downloader/handlers/s3.py

@@ -39,7 +39,7 @@ class S3DownloadHandler(BaseDownloadHandler):
                 )
             )
 
-        _http_handler = build_from_crawler(
+        _http_handler: BaseDownloadHandler = build_from_crawler(
             load_object(crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"]),
             crawler,
         )

scrapy/core/http2/agent.py

@@ -143,7 +143,7 @@ class H2Agent:
         )
 
     def get_endpoint(self, uri: URI) -> HostnameEndpoint:
-        return self.endpoint_factory.endpointForURI(uri)
+        return self.endpoint_factory.endpointForURI(uri)  # type: ignore[no-any-return]
 
     def get_key(self, uri: URI) -> ConnectionKeyT:
         """
@@ -187,7 +187,7 @@ class ScrapyProxyH2Agent(H2Agent):
         self._proxy_uri = proxy_uri
 
     def get_endpoint(self, uri: URI) -> HostnameEndpoint:
-        return self.endpoint_factory.endpointForURI(self._proxy_uri)
+        return self.endpoint_factory.endpointForURI(self._proxy_uri)  # type: ignore[no-any-return]
 
     def get_key(self, uri: URI) -> ConnectionKeyT:
         """We use the proxy uri instead of uri obtained from request url"""

scrapy/core/scheduler.py

@@ -4,7 +4,7 @@ import json
 import logging
 from abc import abstractmethod
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 
 # working around https://github.com/sphinx-doc/sphinx/issues/10400
 from twisted.internet.defer import Deferred  # noqa: TC002
@@ -334,7 +334,7 @@ class Scheduler(BaseScheduler):
         cls = crawler.settings[f"SCHEDULER_START_{queue}_QUEUE"]
         if not cls:
             return None
-        return load_object(cls)
+        return cast("type[BaseQueue]", load_object(cls))
 
     def has_pending_requests(self) -> bool:
         return len(self) > 0

scrapy/http/cookies.py

@@ -74,7 +74,7 @@ class CookieJar:
 
     @property
     def _cookies(self) -> dict[str, dict[str, dict[str, Cookie]]]:
-        return self.jar._cookies  # type: ignore[attr-defined]
+        return self.jar._cookies  # type: ignore[attr-defined,no-any-return]
 
     def clear_session_cookies(self) -> None:
         return self.jar.clear_session_cookies()

scrapy/pipelines/files.py

@@ -323,9 +323,10 @@ class GCSFilesStore:
     ) -> Deferred[StatInfo]:
 
         blob_path = self._get_blob_path(path)
-        return deferred_from_coro(
+        d: Deferred[Any] = deferred_from_coro(
             run_in_thread(self.bucket.get_blob, blob_path)
-        ).addCallback(self._onsuccess)
+        )
+        return d.addCallback(self._onsuccess)
 
     def _get_content_type(self, headers: dict[str, str] | None) -> str:
         if headers and "Content-Type" in headers:

scrapy/pipelines/media.py

@@ -29,9 +29,9 @@ from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import global_object_name
 
 if TYPE_CHECKING:
-    # typing.Self requires Python 3.11
-    from collections.abc import Awaitable
+    from collections.abc import Awaitable, Callable
 
+    # typing.Self requires Python 3.11
     from typing_extensions import Self
 
     from scrapy import Spider
@@ -153,7 +153,7 @@ class MediaPipeline(ABC):
     ) -> FileInfo:
         fp = self._fingerprinter.fingerprint(request)
 
-        eb = request.errback
+        eb: Callable[[Failure], FileInfo] | None = request.errback
         request.callback = NO_CALLBACK
         request.errback = None
 

scrapy/robotstxt.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import logging
 import sys
 from abc import ABCMeta, abstractmethod
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, cast
 from urllib.robotparser import RobotFileParser
 
 from protego import Protego
@@ -103,7 +103,7 @@ class RerpRobotParser(RobotParser):
     def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
         user_agent = to_unicode(user_agent)
         url = to_unicode(url)
-        return self.rp.is_allowed(user_agent, url)
+        return cast("bool", self.rp.is_allowed(user_agent, url))
 
 
 class ProtegoRobotParser(RobotParser):

scrapy/utils/benchserver.py

@@ -30,7 +30,7 @@ class Root(Resource):
 def _getarg(
     request: Request, name: bytes, default: Any = None, type_: type = str
 ) -> Any:
-    return type_(request.args[name][0]) if name in request.args else default  # type: ignore[index,operator]
+    return type_(request.args[name][0]) if name in request.args else default
 
 
 if __name__ == "__main__":

scrapy/utils/datatypes.py

@@ -13,7 +13,7 @@ import warnings
 import weakref
 from collections import OrderedDict
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, AnyStr, TypeVar
+from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, cast
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
@@ -181,7 +181,7 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
 
     def __getitem__(self, key: _KT) -> _VT | None:
         try:
-            return super().__getitem__(key)
+            return cast("_VT", super().__getitem__(key))
         except (TypeError, KeyError):
             return None  # key is either not weak-referenceable or not cached
 

scrapy/utils/decorators.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import inspect
 import warnings
 from functools import wraps
-from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, overload
+from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, cast, overload
 
 from twisted.internet.defer import Deferred, maybeDeferred
 
@@ -115,7 +115,7 @@ def _warn_spider_arg(
         @wraps(func)
         async def async_inner(*args: _P.args, **kwargs: _P.kwargs) -> _T:
             check_args(*args, **kwargs)
-            return await func(*args, **kwargs)
+            return cast("_T", await func(*args, **kwargs))
 
         return async_inner
 

scrapy/utils/deprecate.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import inspect
 import warnings
-from typing import TYPE_CHECKING, Any, overload
+from typing import TYPE_CHECKING, Any, cast, overload
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.python import get_func_args_dict
@@ -68,7 +68,7 @@ def create_deprecated_class(
         def __new__(  # pylint: disable=bad-classmethod-argument
             metacls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any]
         ) -> type:
-            cls = super().__new__(metacls, name, bases, clsdict_)
+            cls: type = super().__new__(metacls, name, bases, clsdict_)
             if metacls.deprecated_class is None:
                 metacls.deprecated_class = cls
             return cls
@@ -100,7 +100,7 @@ def create_deprecated_class(
                 # is the deprecated class itself - subclasses of the
                 # deprecated class should not use custom `__subclasscheck__`
                 # method.
-                return super().__subclasscheck__(sub)
+                return cast("bool", super().__subclasscheck__(sub))
 
             if not inspect.isclass(sub):
                 raise TypeError("issubclass() arg 1 must be a class")

scrapy/utils/misc.py

@@ -13,7 +13,7 @@ from contextlib import contextmanager
 from functools import partial
 from importlib import import_module
 from pkgutil import iter_modules
-from typing import IO, TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, overload
+from typing import IO, TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, cast, overload
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.item import Item
@@ -51,7 +51,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]:
     if arg is None:
         return ()
     if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"):
-        return arg
+        return cast("Iterable[Any]", arg)
     return [arg]
 
 

scrapy/utils/reactor.py

@@ -33,12 +33,12 @@ def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:
     if len(portrange) > 2:
         raise ValueError(f"invalid portrange: {portrange}")
     if not portrange:
-        return reactor.listenTCP(0, factory, interface=host)
+        return reactor.listenTCP(0, factory, interface=host)  # type: ignore[no-any-return]
     if len(portrange) == 1:
-        return reactor.listenTCP(portrange[0], factory, interface=host)
+        return reactor.listenTCP(portrange[0], factory, interface=host)  # type: ignore[no-any-return]
     for x in range(portrange[0], portrange[1] + 1):
         try:
-            return reactor.listenTCP(x, factory, interface=host)
+            return reactor.listenTCP(x, factory, interface=host)  # type: ignore[no-any-return]
         except error.CannotListenError:
             if x == portrange[1]:
                 raise

scrapy/utils/signal.py

@@ -7,6 +7,7 @@ import logging
 import warnings
 from collections.abc import Awaitable, Callable, Generator, Sequence
 from typing import Any as TypingAny
+from typing import cast
 
 from pydispatch.dispatcher import (
     Anonymous,
@@ -185,7 +186,7 @@ async def _send_catch_log_asyncio(
     handlers: list[Awaitable[TypingAny]] = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
 
-        async def handler(receiver: Callable) -> TypingAny:
+        async def handler(receiver: Callable) -> tuple[Callable, TypingAny]:
             result: TypingAny
             try:
                 result = await ensure_awaitable(
@@ -208,7 +209,10 @@ async def _send_catch_log_asyncio(
 
         handlers.append(handler(receiver))
 
-    return await asyncio.gather(*handlers, return_exceptions=True)
+    return cast(
+        "list[tuple[TypingAny, TypingAny]]",
+        await asyncio.gather(*handlers, return_exceptions=True),
+    )
 
 
 def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None:

scrapy/utils/spider.py

@@ -41,11 +41,10 @@ def iterate_spider_output(
 ) -> Iterable[Any] | AsyncGenerator[_T] | Deferred[_T]:
     if inspect.isasyncgen(result):
         return result
+    d: Deferred[_T] = deferred_from_coro(result)
     if inspect.iscoroutine(result):
-        d = deferred_from_coro(result)
-        d.addCallback(iterate_spider_output)
-        return d
-    return arg_to_iter(deferred_from_coro(result))
+        return d.addCallback(iterate_spider_output)
+    return arg_to_iter(d)
 
 
 def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]:

tests/mockserver/http_resources.py

@@ -239,7 +239,7 @@ class ArbitraryLengthPayloadResource(LeafResource):
 
 class NoMetaRefreshRedirect(Redirect):
     def render(self, request: server.Request) -> bytes:
-        content = Redirect.render(self, request)
+        content: bytes = Redirect.render(self, request)
         return content.replace(
             b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"'
         )

tests/mockserver/utils.py

@@ -1,22 +1,26 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from cryptography.hazmat.primitives.serialization import load_pem_private_key
 from cryptography.x509 import load_pem_x509_certificate
 from OpenSSL import SSL
 from OpenSSL.crypto import FILETYPE_PEM, load_certificate, load_privatekey
-from twisted.internet.ssl import CertificateOptions, ContextFactory
+from twisted.internet.ssl import CertificateOptions
 
 from scrapy.utils._deps_compat import PYOPENSSL_WANTS_X509_PKEY
 from scrapy.utils.python import to_bytes
 
+if TYPE_CHECKING:
+    from twisted.internet.interfaces import IOpenSSLContextFactory
+
 
 def ssl_context_factory(
     keyfile: str = "keys/localhost.key",
     certfile: str = "keys/localhost.crt",
     cipher_string: str | None = None,
-) -> ContextFactory:
+) -> IOpenSSLContextFactory:
     keyfile_path = Path(__file__).parent.parent / keyfile
     certfile_path = Path(__file__).parent.parent / certfile
 
@@ -27,7 +31,7 @@ def ssl_context_factory(
         cert = load_certificate(FILETYPE_PEM, certfile_path.read_bytes())  # type: ignore[assignment]
         key = load_privatekey(FILETYPE_PEM, keyfile_path.read_bytes())  # type: ignore[assignment]
 
-    factory = CertificateOptions(
+    factory: CertificateOptions = CertificateOptions(
         privateKey=key,
         certificate=cert,
     )

tests/test_downloadermiddleware_robotstxt.py

@@ -23,8 +23,8 @@ if TYPE_CHECKING:
 
 
 class TestRobotsTxtMiddleware:
-    def setup_method(self):
-        self.crawler = mock.MagicMock()
+    def setup_method(self) -> None:
+        self.crawler: mock.MagicMock = mock.MagicMock()
         self.crawler.settings = Settings()
         self.crawler.engine.download_async = mock.AsyncMock()
 

tests/test_http2_client_protocol.py

@@ -234,7 +234,7 @@ class TestHttps2ClientProtocol:
         pem = self.key_file.read_text(
             encoding="utf-8"
         ) + self.certificate_file.read_text(encoding="utf-8")
-        return PrivateCertificate.loadPEM(pem)
+        return PrivateCertificate.loadPEM(pem)  # type: ignore[no-any-return]
 
     @async_yield_fixture  # type: ignore[untyped-decorator]
     async def client(

tests/test_scheduler.py

@@ -4,7 +4,7 @@ import warnings
 from abc import ABC, abstractmethod
 from collections import deque
 from contextlib import AbstractAsyncContextManager, asynccontextmanager
-from typing import TYPE_CHECKING, Any, NamedTuple
+from typing import TYPE_CHECKING, Any, NamedTuple, cast
 from unittest.mock import Mock
 
 import pytest
@@ -29,9 +29,9 @@ if TYPE_CHECKING:
 class MemoryScheduler(BaseScheduler):
     paused = False
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any):
         super().__init__(*args, **kwargs)
-        self.queue = deque(
+        self.queue: deque[Request] = deque(
             Request(value) if isinstance(value, str) else value
             for value in getattr(self, "queue", [])
         )
@@ -68,7 +68,7 @@ class MockDownloader:
 
     def get_slot_key(self, request: Request) -> str:
         if Downloader.DOWNLOAD_SLOT in request.meta:
-            return request.meta[Downloader.DOWNLOAD_SLOT]
+            return cast("str", request.meta[Downloader.DOWNLOAD_SLOT])
 
         return urlparse_cached(request).hostname or ""
 

tests/test_spidermiddleware.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from collections.abc import AsyncIterator, Iterable
 from inspect import isasyncgen
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 from unittest import mock
 
 import pytest
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
 
 
 class TestSpiderMiddleware:
-    def setup_method(self):
+    def setup_method(self) -> None:
         self.request = Request("http://example.com/index.html")
         self.response = Response(self.request.url, request=self.request)
         self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES_BASE": {}})
@@ -39,11 +39,10 @@ class TestSpiderMiddleware:
         Raise exception in case of failure.
         """
 
-        def scrape_func(
+        async def scrape_func(
             response: Response | Failure, request: Request
-        ) -> defer.Deferred[Iterable[Any]]:
-            it = mock.MagicMock()
-            return defer.succeed(it)
+        ) -> Iterable[Any]:
+            return mock.MagicMock()
 
         return await self.mwman.scrape_response_async(
             scrape_func, self.response, self.request
@@ -141,7 +140,7 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
     async def _scrape_func(
         self, response: Response | Failure, request: Request
     ) -> Iterable[Any] | AsyncIterator[Any]:
-        return self._callback()
+        return cast("Iterable[Any] | AsyncIterator[Any]", self._callback())
 
     async def _get_middleware_result(
         self, *mw_classes: type[Any], start_index: int | None = None

tests/utils/decorators.py

@@ -35,7 +35,8 @@ def inline_callbacks_test(
         async def wrapper_coro(*args: _P.args, **kwargs: _P.kwargs) -> None:
             await deferred_to_future(inlineCallbacks(f)(*args, **kwargs))
 
-        return wrapper_coro
+        # Likely https://github.com/python/mypy/issues/17171
+        return wrapper_coro  # type: ignore[no-any-return]
 
     @wraps(f)
     @inlineCallbacks

tests_typing/test_spiders.mypy-testing

@@ -59,7 +59,7 @@ def test_spider_parse_override_no_kwargs() -> None:
 @pytest.mark.mypy_testing
 def test_spider_parse_override_specific_kwargs() -> None:
     spider = SpecificKwargsSpider()
-    reveal_type(spider.parse)  # R: def (response: scrapy.http.response.Response, page: builtins.int) -> Any
+    reveal_type(spider.parse)  # R: def (response: scrapy.http.response.Response, page: int) -> Any
 
 
 @pytest.mark.mypy_testing

tox.ini

@@ -44,26 +44,28 @@ commands =
 [testenv:typing]
 basepython = python3.10
 deps =
-    mypy==1.19.1
+    mypy==1.20.2
     typing-extensions==4.15.0
-    Pillow==12.1.1
+    Pillow==12.2.0
     Protego==0.6.0
-    attrs==25.4.0
-    boto3-stubs[s3]==1.42.59
+    Twisted==25.5.0
+    attrs==26.1.0
+    boto3-stubs[s3]==1.43.2
     botocore-stubs==1.42.41
     h2==4.3.0
     httpx==0.28.1
     itemadapter==0.13.1
     ptpython==3.0.32
-    ipython
-    pyOpenSSL==25.3.0
-    pytest==9.0.2
-    types-Pygments==2.19.0.20251121
-    types-defusedxml==0.7.0.20250822
+    # newer ones require newer Python
+    ipython==8.39.0
+    pyOpenSSL==26.1.0
+    pytest==9.0.3
+    types-Pygments==2.20.0.20260408
+    types-defusedxml==0.7.0.20260504
     types-lxml==2026.2.16
-    types-pexpect==4.9.0.20260127
+    types-pexpect==4.9.0.20260408
     uvloop==0.22.1
-    w3lib==2.4.0
+    w3lib==2.4.1
     zstandard==0.25.0
 commands =
     mypy {posargs:scrapy tests}