From fc14a0ce5989e32794636a287c6914c0af396ed9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 5 May 2026 01:24:47 +0500 Subject: [PATCH] Enable mypy warn_return_any. (#7492) --- pyproject.toml | 1 - scrapy/commands/parse.py | 11 ++++----- scrapy/core/downloader/__init__.py | 8 ++++--- scrapy/core/downloader/contextfactory.py | 8 +++---- scrapy/core/downloader/handlers/s3.py | 2 +- scrapy/core/http2/agent.py | 4 ++-- scrapy/core/scheduler.py | 4 ++-- scrapy/http/cookies.py | 2 +- scrapy/pipelines/files.py | 5 ++-- scrapy/pipelines/media.py | 6 ++--- scrapy/robotstxt.py | 4 ++-- scrapy/utils/benchserver.py | 2 +- scrapy/utils/datatypes.py | 4 ++-- scrapy/utils/decorators.py | 4 ++-- scrapy/utils/deprecate.py | 6 ++--- scrapy/utils/misc.py | 4 ++-- scrapy/utils/reactor.py | 6 ++--- scrapy/utils/signal.py | 8 +++++-- scrapy/utils/spider.py | 7 +++--- tests/mockserver/http_resources.py | 2 +- tests/mockserver/utils.py | 10 +++++--- tests/test_downloadermiddleware_robotstxt.py | 4 ++-- tests/test_http2_client_protocol.py | 2 +- tests/test_scheduler.py | 8 +++---- tests/test_spidermiddleware.py | 13 +++++------ tests/utils/decorators.py | 3 ++- tests_typing/test_spiders.mypy-testing | 2 +- tox.ini | 24 +++++++++++--------- 28 files changed, 87 insertions(+), 77 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a68222b36..11d36d0ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,7 +91,6 @@ extra_checks = false # weird addErrback() errors untyped_calls_exclude = [ "twisted", ] -warn_return_any = false # 37 errors [[tool.mypy.overrides]] module = "tests.*" diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 632186a7d..2ac65bf3f 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -144,17 +144,16 @@ class Command(BaseRunSpiderCommand): def iterate_spider_output(self, result: _T) -> Iterable[Any]: ... def iterate_spider_output(self, result: Any) -> Iterable[Any] | Deferred[Any]: + d: Deferred[Any] if inspect.isasyncgen(result): d = deferred_from_coro( collect_asyncgen(aiter_errback(result, self.handle_exception)) ) - d.addCallback(self.iterate_spider_output) - return d + return d.addCallback(self.iterate_spider_output) + d = deferred_from_coro(result) if inspect.iscoroutine(result): - d = deferred_from_coro(result) - d.addCallback(self.iterate_spider_output) - return d - return arg_to_iter(deferred_from_coro(result)) + return d.addCallback(self.iterate_spider_output) + return arg_to_iter(d) def add_items(self, lvl: int, new_items: list[Any]) -> None: old_items = self.items.get(lvl, []) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 01d5d42b0..7c0ee0eec 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -128,11 +128,12 @@ class Downloader: ) -> Generator[Deferred[Any], Any, Response | Request]: self.active.add(request) try: - return ( - yield deferred_from_coro( + result: Response | Request = yield ( + deferred_from_coro( self.middleware.download_async(self._enqueue_request, request) ) ) + return result finally: self.active.remove(request) @@ -163,7 +164,8 @@ class Downloader: return key, self.slots[key] def get_slot_key(self, request: Request) -> str: - if (meta_slot := request.meta.get(self.DOWNLOAD_SLOT)) is not None: + meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT) + if meta_slot is not None: return meta_slot key = urlparse_cached(request).hostname or "" diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index bbda7efcd..6575d59c1 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -108,7 +108,7 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): def _get_cert_options(self) -> CertificateOptions: with _filter_method_warning(): - return CertificateOptions( + return CertificateOptions( # type: ignore[no-any-return] method=self._ssl_method, fixBrokenPeers=True, acceptableCiphers=self.tls_ciphers, @@ -122,7 +122,7 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): def _get_context(self) -> SSL.Context: cert_options = self._get_cert_options() - ctx = cert_options.getContext() + ctx: SSL.Context = cert_options.getContext() ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT return ctx @@ -135,7 +135,7 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # Otherwise use the normal Twisted function. # Note that this doesn't use self._get_context(). with _filter_method_warning(): - return optionsForClientTLS( + return optionsForClientTLS( # type: ignore[no-any-return] hostname=hostname.decode("ascii"), extraCertificateOptions={ "method": self._ssl_method, @@ -186,7 +186,7 @@ class BrowserLikeContextFactory(_ScrapyClientContextFactory): def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: with _filter_method_warning(): - return optionsForClientTLS( + return optionsForClientTLS( # type: ignore[no-any-return] hostname=hostname.decode("ascii"), extraCertificateOptions={"method": self._ssl_method}, ) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index a609cedb3..19a1e8503 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -39,7 +39,7 @@ class S3DownloadHandler(BaseDownloadHandler): ) ) - _http_handler = build_from_crawler( + _http_handler: BaseDownloadHandler = build_from_crawler( load_object(crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"]), crawler, ) diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 540b9aa74..822dffc4b 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -143,7 +143,7 @@ class H2Agent: ) def get_endpoint(self, uri: URI) -> HostnameEndpoint: - return self.endpoint_factory.endpointForURI(uri) + return self.endpoint_factory.endpointForURI(uri) # type: ignore[no-any-return] def get_key(self, uri: URI) -> ConnectionKeyT: """ @@ -187,7 +187,7 @@ class ScrapyProxyH2Agent(H2Agent): self._proxy_uri = proxy_uri def get_endpoint(self, uri: URI) -> HostnameEndpoint: - return self.endpoint_factory.endpointForURI(self._proxy_uri) + return self.endpoint_factory.endpointForURI(self._proxy_uri) # type: ignore[no-any-return] def get_key(self, uri: URI) -> ConnectionKeyT: """We use the proxy uri instead of uri obtained from request url""" diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 78329460e..7217da942 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,7 +4,7 @@ import json import logging from abc import abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast # working around https://github.com/sphinx-doc/sphinx/issues/10400 from twisted.internet.defer import Deferred # noqa: TC002 @@ -334,7 +334,7 @@ class Scheduler(BaseScheduler): cls = crawler.settings[f"SCHEDULER_START_{queue}_QUEUE"] if not cls: return None - return load_object(cls) + return cast("type[BaseQueue]", load_object(cls)) def has_pending_requests(self) -> bool: return len(self) > 0 diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 09286606d..599f20947 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -74,7 +74,7 @@ class CookieJar: @property def _cookies(self) -> dict[str, dict[str, dict[str, Cookie]]]: - return self.jar._cookies # type: ignore[attr-defined] + return self.jar._cookies # type: ignore[attr-defined,no-any-return] def clear_session_cookies(self) -> None: return self.jar.clear_session_cookies() diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index ffed96446..0066fd38f 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -323,9 +323,10 @@ class GCSFilesStore: ) -> Deferred[StatInfo]: blob_path = self._get_blob_path(path) - return deferred_from_coro( + d: Deferred[Any] = deferred_from_coro( run_in_thread(self.bucket.get_blob, blob_path) - ).addCallback(self._onsuccess) + ) + return d.addCallback(self._onsuccess) def _get_content_type(self, headers: dict[str, str] | None) -> str: if headers and "Content-Type" in headers: diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 1043da332..5b5d2dcb2 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -29,9 +29,9 @@ from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import global_object_name if TYPE_CHECKING: - # typing.Self requires Python 3.11 - from collections.abc import Awaitable + from collections.abc import Awaitable, Callable + # typing.Self requires Python 3.11 from typing_extensions import Self from scrapy import Spider @@ -153,7 +153,7 @@ class MediaPipeline(ABC): ) -> FileInfo: fp = self._fingerprinter.fingerprint(request) - eb = request.errback + eb: Callable[[Failure], FileInfo] | None = request.errback request.callback = NO_CALLBACK request.errback = None diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 18b622546..0c64ea5a5 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging import sys from abc import ABCMeta, abstractmethod -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast from urllib.robotparser import RobotFileParser from protego import Protego @@ -103,7 +103,7 @@ class RerpRobotParser(RobotParser): def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: user_agent = to_unicode(user_agent) url = to_unicode(url) - return self.rp.is_allowed(user_agent, url) + return cast("bool", self.rp.is_allowed(user_agent, url)) class ProtegoRobotParser(RobotParser): diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index c4c96ef00..403cd54a8 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -30,7 +30,7 @@ class Root(Resource): def _getarg( request: Request, name: bytes, default: Any = None, type_: type = str ) -> Any: - return type_(request.args[name][0]) if name in request.args else default # type: ignore[index,operator] + return type_(request.args[name][0]) if name in request.args else default if __name__ == "__main__": diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 64d2cde1b..146e3ae56 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -13,7 +13,7 @@ import warnings import weakref from collections import OrderedDict from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, AnyStr, TypeVar +from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, cast from scrapy.exceptions import ScrapyDeprecationWarning @@ -181,7 +181,7 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): def __getitem__(self, key: _KT) -> _VT | None: try: - return super().__getitem__(key) + return cast("_VT", super().__getitem__(key)) except (TypeError, KeyError): return None # key is either not weak-referenceable or not cached diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index aea2557d2..a5bb6fa24 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -3,7 +3,7 @@ from __future__ import annotations import inspect import warnings from functools import wraps -from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, overload +from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, cast, overload from twisted.internet.defer import Deferred, maybeDeferred @@ -115,7 +115,7 @@ def _warn_spider_arg( @wraps(func) async def async_inner(*args: _P.args, **kwargs: _P.kwargs) -> _T: check_args(*args, **kwargs) - return await func(*args, **kwargs) + return cast("_T", await func(*args, **kwargs)) return async_inner diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index da1838030..359f819d7 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -4,7 +4,7 @@ from __future__ import annotations import inspect import warnings -from typing import TYPE_CHECKING, Any, overload +from typing import TYPE_CHECKING, Any, cast, overload from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.python import get_func_args_dict @@ -68,7 +68,7 @@ def create_deprecated_class( def __new__( # pylint: disable=bad-classmethod-argument metacls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any] ) -> type: - cls = super().__new__(metacls, name, bases, clsdict_) + cls: type = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls @@ -100,7 +100,7 @@ def create_deprecated_class( # is the deprecated class itself - subclasses of the # deprecated class should not use custom `__subclasscheck__` # method. - return super().__subclasscheck__(sub) + return cast("bool", super().__subclasscheck__(sub)) if not inspect.isclass(sub): raise TypeError("issubclass() arg 1 must be a class") diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 757756e65..3baa02e2b 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -13,7 +13,7 @@ from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from typing import IO, TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, overload +from typing import IO, TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, cast, overload from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item @@ -51,7 +51,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]: if arg is None: return () if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"): - return arg + return cast("Iterable[Any]", arg) return [arg] diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index b41b8af38..ddebd8ba2 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -33,12 +33,12 @@ def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: if len(portrange) > 2: raise ValueError(f"invalid portrange: {portrange}") if not portrange: - return reactor.listenTCP(0, factory, interface=host) + return reactor.listenTCP(0, factory, interface=host) # type: ignore[no-any-return] if len(portrange) == 1: - return reactor.listenTCP(portrange[0], factory, interface=host) + return reactor.listenTCP(portrange[0], factory, interface=host) # type: ignore[no-any-return] for x in range(portrange[0], portrange[1] + 1): try: - return reactor.listenTCP(x, factory, interface=host) + return reactor.listenTCP(x, factory, interface=host) # type: ignore[no-any-return] except error.CannotListenError: if x == portrange[1]: raise diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 997e8eba8..d9a72273a 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -7,6 +7,7 @@ import logging import warnings from collections.abc import Awaitable, Callable, Generator, Sequence from typing import Any as TypingAny +from typing import cast from pydispatch.dispatcher import ( Anonymous, @@ -185,7 +186,7 @@ async def _send_catch_log_asyncio( handlers: list[Awaitable[TypingAny]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): - async def handler(receiver: Callable) -> TypingAny: + async def handler(receiver: Callable) -> tuple[Callable, TypingAny]: result: TypingAny try: result = await ensure_awaitable( @@ -208,7 +209,10 @@ async def _send_catch_log_asyncio( handlers.append(handler(receiver)) - return await asyncio.gather(*handlers, return_exceptions=True) + return cast( + "list[tuple[TypingAny, TypingAny]]", + await asyncio.gather(*handlers, return_exceptions=True), + ) def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None: diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 75d6c9bb0..9f43df1ce 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -41,11 +41,10 @@ def iterate_spider_output( ) -> Iterable[Any] | AsyncGenerator[_T] | Deferred[_T]: if inspect.isasyncgen(result): return result + d: Deferred[_T] = deferred_from_coro(result) if inspect.iscoroutine(result): - d = deferred_from_coro(result) - d.addCallback(iterate_spider_output) - return d - return arg_to_iter(deferred_from_coro(result)) + return d.addCallback(iterate_spider_output) + return arg_to_iter(d) def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]: diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.py index 7f52f0092..98ac6cf6a 100644 --- a/tests/mockserver/http_resources.py +++ b/tests/mockserver/http_resources.py @@ -239,7 +239,7 @@ class ArbitraryLengthPayloadResource(LeafResource): class NoMetaRefreshRedirect(Redirect): def render(self, request: server.Request) -> bytes: - content = Redirect.render(self, request) + content: bytes = Redirect.render(self, request) return content.replace( b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"' ) diff --git a/tests/mockserver/utils.py b/tests/mockserver/utils.py index f900ee693..188eef61d 100644 --- a/tests/mockserver/utils.py +++ b/tests/mockserver/utils.py @@ -1,22 +1,26 @@ from __future__ import annotations from pathlib import Path +from typing import TYPE_CHECKING from cryptography.hazmat.primitives.serialization import load_pem_private_key from cryptography.x509 import load_pem_x509_certificate from OpenSSL import SSL from OpenSSL.crypto import FILETYPE_PEM, load_certificate, load_privatekey -from twisted.internet.ssl import CertificateOptions, ContextFactory +from twisted.internet.ssl import CertificateOptions from scrapy.utils._deps_compat import PYOPENSSL_WANTS_X509_PKEY from scrapy.utils.python import to_bytes +if TYPE_CHECKING: + from twisted.internet.interfaces import IOpenSSLContextFactory + def ssl_context_factory( keyfile: str = "keys/localhost.key", certfile: str = "keys/localhost.crt", cipher_string: str | None = None, -) -> ContextFactory: +) -> IOpenSSLContextFactory: keyfile_path = Path(__file__).parent.parent / keyfile certfile_path = Path(__file__).parent.parent / certfile @@ -27,7 +31,7 @@ def ssl_context_factory( cert = load_certificate(FILETYPE_PEM, certfile_path.read_bytes()) # type: ignore[assignment] key = load_privatekey(FILETYPE_PEM, keyfile_path.read_bytes()) # type: ignore[assignment] - factory = CertificateOptions( + factory: CertificateOptions = CertificateOptions( privateKey=key, certificate=cert, ) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 15c68779d..082fc743e 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -23,8 +23,8 @@ if TYPE_CHECKING: class TestRobotsTxtMiddleware: - def setup_method(self): - self.crawler = mock.MagicMock() + def setup_method(self) -> None: + self.crawler: mock.MagicMock = mock.MagicMock() self.crawler.settings = Settings() self.crawler.engine.download_async = mock.AsyncMock() diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 65da89472..6072e2f6d 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -234,7 +234,7 @@ class TestHttps2ClientProtocol: pem = self.key_file.read_text( encoding="utf-8" ) + self.certificate_file.read_text(encoding="utf-8") - return PrivateCertificate.loadPEM(pem) + return PrivateCertificate.loadPEM(pem) # type: ignore[no-any-return] @async_yield_fixture # type: ignore[untyped-decorator] async def client( diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 7fec479f3..eb88baf01 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -4,7 +4,7 @@ import warnings from abc import ABC, abstractmethod from collections import deque from contextlib import AbstractAsyncContextManager, asynccontextmanager -from typing import TYPE_CHECKING, Any, NamedTuple +from typing import TYPE_CHECKING, Any, NamedTuple, cast from unittest.mock import Mock import pytest @@ -29,9 +29,9 @@ if TYPE_CHECKING: class MemoryScheduler(BaseScheduler): paused = False - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) - self.queue = deque( + self.queue: deque[Request] = deque( Request(value) if isinstance(value, str) else value for value in getattr(self, "queue", []) ) @@ -68,7 +68,7 @@ class MockDownloader: def get_slot_key(self, request: Request) -> str: if Downloader.DOWNLOAD_SLOT in request.meta: - return request.meta[Downloader.DOWNLOAD_SLOT] + return cast("str", request.meta[Downloader.DOWNLOAD_SLOT]) return urlparse_cached(request).hostname or "" diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 9651e3f8b..bfa60ac9c 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -2,7 +2,7 @@ from __future__ import annotations from collections.abc import AsyncIterator, Iterable from inspect import isasyncgen -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from unittest import mock import pytest @@ -27,7 +27,7 @@ if TYPE_CHECKING: class TestSpiderMiddleware: - def setup_method(self): + def setup_method(self) -> None: self.request = Request("http://example.com/index.html") self.response = Response(self.request.url, request=self.request) self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES_BASE": {}}) @@ -39,11 +39,10 @@ class TestSpiderMiddleware: Raise exception in case of failure. """ - def scrape_func( + async def scrape_func( response: Response | Failure, request: Request - ) -> defer.Deferred[Iterable[Any]]: - it = mock.MagicMock() - return defer.succeed(it) + ) -> Iterable[Any]: + return mock.MagicMock() return await self.mwman.scrape_response_async( scrape_func, self.response, self.request @@ -141,7 +140,7 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): async def _scrape_func( self, response: Response | Failure, request: Request ) -> Iterable[Any] | AsyncIterator[Any]: - return self._callback() + return cast("Iterable[Any] | AsyncIterator[Any]", self._callback()) async def _get_middleware_result( self, *mw_classes: type[Any], start_index: int | None = None diff --git a/tests/utils/decorators.py b/tests/utils/decorators.py index 4750158e5..0e8988a0a 100644 --- a/tests/utils/decorators.py +++ b/tests/utils/decorators.py @@ -35,7 +35,8 @@ def inline_callbacks_test( async def wrapper_coro(*args: _P.args, **kwargs: _P.kwargs) -> None: await deferred_to_future(inlineCallbacks(f)(*args, **kwargs)) - return wrapper_coro + # Likely https://github.com/python/mypy/issues/17171 + return wrapper_coro # type: ignore[no-any-return] @wraps(f) @inlineCallbacks diff --git a/tests_typing/test_spiders.mypy-testing b/tests_typing/test_spiders.mypy-testing index 162e31d0c..dfdd2ba4d 100644 --- a/tests_typing/test_spiders.mypy-testing +++ b/tests_typing/test_spiders.mypy-testing @@ -59,7 +59,7 @@ def test_spider_parse_override_no_kwargs() -> None: @pytest.mark.mypy_testing def test_spider_parse_override_specific_kwargs() -> None: spider = SpecificKwargsSpider() - reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response, page: builtins.int) -> Any + reveal_type(spider.parse) # R: def (response: scrapy.http.response.Response, page: int) -> Any @pytest.mark.mypy_testing diff --git a/tox.ini b/tox.ini index 07583a11e..a3b0cd91d 100644 --- a/tox.ini +++ b/tox.ini @@ -44,26 +44,28 @@ commands = [testenv:typing] basepython = python3.10 deps = - mypy==1.19.1 + mypy==1.20.2 typing-extensions==4.15.0 - Pillow==12.1.1 + Pillow==12.2.0 Protego==0.6.0 - attrs==25.4.0 - boto3-stubs[s3]==1.42.59 + Twisted==25.5.0 + attrs==26.1.0 + boto3-stubs[s3]==1.43.2 botocore-stubs==1.42.41 h2==4.3.0 httpx==0.28.1 itemadapter==0.13.1 ptpython==3.0.32 - ipython - pyOpenSSL==25.3.0 - pytest==9.0.2 - types-Pygments==2.19.0.20251121 - types-defusedxml==0.7.0.20250822 + # newer ones require newer Python + ipython==8.39.0 + pyOpenSSL==26.1.0 + pytest==9.0.3 + types-Pygments==2.20.0.20260408 + types-defusedxml==0.7.0.20260504 types-lxml==2026.2.16 - types-pexpect==4.9.0.20260127 + types-pexpect==4.9.0.20260408 uvloop==0.22.1 - w3lib==2.4.0 + w3lib==2.4.1 zstandard==0.25.0 commands = mypy {posargs:scrapy tests}