From e02ad08672a5946f659acf4874c4a315e7886346 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 2 Mar 2026 15:47:23 +0500 Subject: [PATCH] Move to mypy --strict with exceptions. (#7300) * Move to mypy --strict with exceptions. * Fix typing-tests. --- pyproject.toml | 54 ++++++++++++++++++-- scrapy/addons.py | 2 +- scrapy/commands/bench.py | 2 +- scrapy/commands/check.py | 7 ++- scrapy/commands/fetch.py | 7 +-- scrapy/contracts/__init__.py | 2 +- scrapy/core/downloader/handlers/_httpx.py | 2 +- scrapy/core/downloader/handlers/http11.py | 5 +- scrapy/core/engine.py | 6 ++- scrapy/core/http2/protocol.py | 5 +- scrapy/core/spidermw.py | 2 +- scrapy/crawler.py | 2 +- scrapy/extensions/debug.py | 6 +-- scrapy/extensions/feedexport.py | 8 +-- scrapy/extensions/telnet.py | 2 +- scrapy/http/cookies.py | 2 +- scrapy/http/request/form.py | 2 +- scrapy/pipelines/__init__.py | 3 +- scrapy/pipelines/files.py | 2 +- scrapy/pipelines/media.py | 1 + scrapy/shell.py | 2 +- scrapy/signalmanager.py | 6 +-- scrapy/statscollectors.py | 2 +- scrapy/utils/benchserver.py | 6 ++- scrapy/utils/console.py | 6 +-- scrapy/utils/datatypes.py | 4 +- scrapy/utils/defer.py | 3 +- scrapy/utils/deprecate.py | 2 +- scrapy/utils/log.py | 2 +- scrapy/utils/reactor.py | 4 +- scrapy/utils/url.py | 4 +- scrapy/utils/versions.py | 6 +-- tests/test_downloader_handler_twisted_ftp.py | 2 +- tests/test_downloadermiddleware_httpproxy.py | 2 - tests/test_http2_client_protocol.py | 4 +- tests/test_utils_iterators.py | 2 +- tests_typing/test_http_request.mypy-testing | 12 ++--- tests_typing/test_http_response.mypy-testing | 10 ++-- tox.ini | 35 +++++++------ 39 files changed, 148 insertions(+), 88 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e4fae8bf4..5138fda88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,8 +85,20 @@ path = "scrapy/VERSION" pattern = "^(?P.+)$" [tool.mypy] -ignore_missing_imports = true -implicit_reexport = false +strict = true +allow_any_generics = true # 67 errors +allow_untyped_calls = true # 58 errors +extra_checks = false # weird addErrback() errors +untyped_calls_exclude = [ + "twisted", +] +warn_return_any = false # 37 errors + +[[tool.mypy.overrides]] +module = "tests.*" +allow_untyped_defs = true +allow_incomplete_defs = true # 48 errors +check_untyped_defs = false # Interface classes are hard to support [[tool.mypy.overrides]] @@ -101,14 +113,46 @@ ignore_errors = true module = "twisted.internet.reactor" follow_imports = "skip" -# FIXME: remove the following section once the issues are solved +# just for twisted.version +[[tool.mypy.overrides]] +module = "twisted" +implicit_reexport = true + +# TODO [[tool.mypy.overrides]] module = "scrapy.settings.default_settings" ignore_errors = true +# deprecated modules [[tool.mypy.overrides]] -module = "twisted" -implicit_reexport = true +module = [ + "scrapy.core.downloader.webclient", + "scrapy.spiders.init", + "scrapy.utils.testsite", + "tests.test_webclient", +] +allow_any_generics = true +allow_untyped_calls = true +allow_untyped_defs = true +check_untyped_defs = false +warn_return_any = false + +# usually no type hints +[[tool.mypy.overrides]] +module = [ +# "IPython.*", + "bpython", + "brotli", + "brotlicffi", + "google.*", + "pydispatch.*", + "pyftpdlib.*", + "pytest_twisted", + "robotexclusionrulesparser", + "testfixtures", + "zope.interface.*", +] +ignore_missing_imports = true [tool.bumpversion] current_version = "2.14.1" diff --git a/scrapy/addons.py b/scrapy/addons.py index 1024d2dcd..2e12f8c8a 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -55,7 +55,7 @@ class AddonManager: ) @classmethod - def load_pre_crawler_settings(cls, settings: BaseSettings): + def load_pre_crawler_settings(cls, settings: BaseSettings) -> None: """Update early settings that do not require a crawler instance, such as SPIDER_MODULES. Similar to the load_settings method, this loads each add-on configured in the diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index c4e277a60..cc39d344a 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -43,7 +43,7 @@ class _BenchServer: assert self.proc.stdout self.proc.stdout.readline() - def __exit__(self, exc_type, exc_value, traceback) -> None: + def __exit__(self, exc_type, exc_value, traceback) -> None: # type: ignore[no-untyped-def] self.proc.kill() self.proc.wait() time.sleep(0.2) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index e9ada0fb6..17e66e20c 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -1,9 +1,12 @@ import argparse import time from collections import defaultdict +from collections.abc import AsyncIterator +from typing import Any from unittest import TextTestResult as _TextTestResult from unittest import TextTestRunner +from scrapy import Spider from scrapy.commands import ScrapyCommand from scrapy.contracts import ContractsManager from scrapy.utils.conf import build_component_list @@ -81,14 +84,14 @@ class Command(ScrapyCommand): assert self.crawler_process spider_loader = self.crawler_process.spider_loader - async def start(self): + async def start(self: Spider) -> AsyncIterator[Any]: for request in conman.from_spider(self, result): yield request with set_environ(SCRAPY_CHECK="true"): for spidername in args or spider_loader.list(): spidercls = spider_loader.load(spidername) - spidercls.start = start # type: ignore[assignment,method-assign,return-value] + spidercls.start = start # type: ignore[method-assign] tested_methods = conman.tested_methods_from_spidercls(spidercls) if opts.list: diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 1b1d2442f..0b8311efb 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -2,7 +2,7 @@ from __future__ import annotations import sys from argparse import Namespace # noqa: TC003 -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from w3lib.url import is_url @@ -14,6 +14,7 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request if TYPE_CHECKING: from argparse import ArgumentParser + from collections.abc import AsyncIterator from scrapy import Spider @@ -89,10 +90,10 @@ class Command(ScrapyCommand): else: spidercls = spidercls_for_request(spider_loader, request, spidercls) - async def start(self): + async def start(self: Spider) -> AsyncIterator[Any]: yield request - spidercls.start = start # type: ignore[method-assign,attr-defined] + spidercls.start = start # type: ignore[method-assign] self.crawler_process.crawl(spidercls) self.crawler_process.start() diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index f5ba5ba13..cbdb36d2f 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -189,7 +189,7 @@ class ContractsManager: def eb_wrapper(failure: Failure) -> None: case = _create_testcase(method, "errback") exc_info = failure.type, failure.value, failure.getTracebackObject() - results.addError(case, exc_info) + results.addError(case, exc_info) # type: ignore[arg-type] request.callback = cb_wrapper request.errback = eb_wrapper diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index 1f5cf007e..4a9617506 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -265,5 +265,5 @@ class HttpxDownloadHandler(BaseHttpDownloadHandler): logger.warning(warning_msg) raise DownloadCancelledError(warning_msg) - async def close(self): + async def close(self) -> None: await self._client.aclose() diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 992c318c5..2fd3bd988 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -507,7 +507,7 @@ class ScrapyAgent: } # deliverBody hangs for responses without body - if txresponse.length == 0: + if cast("int", txresponse.length) == 0: return { "txresponse": txresponse, } @@ -697,7 +697,8 @@ class _ResponseReader(Protocol): return if reason.check(ResponseFailed) and any( - r.check(_DataLoss) for r in reason.value.reasons + r.check(_DataLoss) + for r in reason.value.reasons # type: ignore[union-attr] ): if not self._fail_on_dataloss: self._finish_response(flags=["dataloss"]) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 1555fc0d6..dc4106963 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -160,7 +160,7 @@ class ExecutionEngine: return scheduler_cls def start( - self, _start_request_processing=True + self, _start_request_processing: bool = True ) -> Deferred[None]: # pragma: no cover warnings.warn( "ExecutionEngine.start() is deprecated, use start_async() instead", @@ -263,12 +263,13 @@ class ExecutionEngine: def unpause(self) -> None: self.paused = False - async def _process_start_next(self): + async def _process_start_next(self) -> None: """Processes the next item or request from Spider.start(). If a request, it is scheduled. If an item, it is sent to item pipelines. """ + assert self._start is not None try: item_or_request = await self._start.__anext__() except StopAsyncIteration: @@ -286,6 +287,7 @@ class ExecutionEngine: if isinstance(item_or_request, Request): self.crawl(item_or_request) else: + assert self._slot is not None _schedule_coro( self.scraper.start_itemproc_async(item_or_request, response=None) ) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 9f6216157..fe4128a68 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -4,7 +4,7 @@ import ipaddress import itertools import logging from collections import deque -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from h2.config import H2Configuration from h2.connection import H2Connection @@ -39,6 +39,7 @@ from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute if TYPE_CHECKING: from ipaddress import IPv4Address, IPv6Address + from hpack import HeaderTuple from twisted.internet.defer import Deferred from twisted.python.failure import Failure from twisted.web.client import URI @@ -408,7 +409,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): except KeyError: pass # We ignore server-initiated events else: - stream.receive_headers(event.headers) + stream.receive_headers(cast("list[HeaderTuple]", event.headers)) def settings_acknowledged(self, event: SettingsAcknowledged) -> None: self.metadata["settings_acknowledged"] = True diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 476d2e504..007d03a04 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -460,7 +460,7 @@ class SpiderMiddlewareManager(MiddlewareManager): start = await self._process_chain("process_start", start) return start - def _check_deprecated_start_requests_use(self): + def _check_deprecated_start_requests_use(self) -> None: start_requests_cls = None start_cls = None spidercls = self._spider.__class__ diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 95cd04b51..b2f08450b 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -135,7 +135,7 @@ class Crawler: "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} ) - def _apply_reactorless_default_settings(self): + def _apply_reactorless_default_settings(self) -> None: """Change some setting defaults when not using a Twisted reactor. Some settings need different defaults when using and not using a diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index afaf81928..5def7509e 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -34,8 +34,8 @@ class StackTraceDump: def __init__(self, crawler: Crawler): self.crawler: Crawler = crawler try: - signal.signal(signal.SIGUSR2, self.dump_stacktrace) # type: ignore[attr-defined] - signal.signal(signal.SIGQUIT, self.dump_stacktrace) # type: ignore[attr-defined] + signal.signal(signal.SIGUSR2, self.dump_stacktrace) # type: ignore[attr-defined,unused-ignore] + signal.signal(signal.SIGQUIT, self.dump_stacktrace) # type: ignore[attr-defined,unused-ignore] except AttributeError: # win32 platforms don't support SIGUSR signals pass @@ -72,7 +72,7 @@ class Debugger: def __init__(self) -> None: # win32 platforms don't support SIGUSR signals with contextlib.suppress(AttributeError): - signal.signal(signal.SIGUSR2, self._enter_debugger) # type: ignore[attr-defined] + signal.signal(signal.SIGUSR2, self._enter_debugger) # type: ignore[attr-defined,unused-ignore] def _enter_debugger(self, signum: int, frame: FrameType | None) -> None: assert frame diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index cb56d1c92..86593aba8 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -89,20 +89,20 @@ class ItemFilter: return True # accept all items by default -class IFeedStorage(Interface): +class IFeedStorage(Interface): # type: ignore[misc] """Interface that all Feed Storages must implement""" # pylint: disable=no-self-argument - def __init__(uri, *, feed_options=None): # pylint: disable=super-init-not-called + def __init__(uri, *, feed_options=None): # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called """Initialize the storage with the parameters given in the URI and the feed-specific options (see :setting:`FEEDS`)""" - def open(spider): + def open(spider): # type: ignore[no-untyped-def] """Open the storage for the given spider. It must return a file-like object that will be used for the exporters""" - def store(file): + def store(file): # type: ignore[no-untyped-def] """Store the given file stream""" diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index f95cdb040..fec6b2d56 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -86,7 +86,7 @@ class TelnetConsole(protocol.ServerFactory): class Portal: """An implementation of IPortal""" - def login(self_, credentials, mind, *interfaces): # pylint: disable=no-self-argument + def login(self_, credentials, mind, *interfaces): # type: ignore[no-untyped-def] # pylint: disable=no-self-argument if not ( credentials.username == self.username.encode("utf8") and credentials.checkPassword(self.password.encode("utf8")) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index e5b4b28a7..13d1c85d0 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -74,7 +74,7 @@ class CookieJar: @property def _cookies(self) -> dict[str, dict[str, dict[str, Cookie]]]: - return self.jar._cookies # type: ignore[attr-defined,no-any-return] + return self.jar._cookies # type: ignore[attr-defined] def clear_session_cookies(self) -> None: return self.jar.clear_session_cookies() diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 7cb8bfd5d..d2d13b8df 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -108,7 +108,7 @@ def _urlencode(seq: Iterable[FormdataKVType], enc: str) -> str: values = [ (to_bytes(k, enc), to_bytes(v, enc)) for k, vs in seq - for v in (cast("Iterable[str]", vs) if is_listlike(vs) else [cast("str", vs)]) + for v in (vs if is_listlike(vs) else [cast("str", vs)]) ] return urlencode(values, doseq=True) diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 14d53f1c5..f58864471 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -10,7 +10,7 @@ import asyncio import warnings from typing import TYPE_CHECKING, Any, cast -from twisted.internet.defer import Deferred, DeferredList +from twisted.internet.defer import Deferred, DeferredList, FirstError from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.middleware import MiddlewareManager @@ -80,6 +80,7 @@ class ItemPipelineManager(MiddlewareManager): d2: Deferred[list[None]] = d.addCallback(lambda r: [x[1] for x in r]) def eb(failure: Failure) -> Failure: + assert isinstance(failure.value, FirstError) return failure.value.subFailure d2.addErrback(eb) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 795a0143f..55e01c4ab 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -305,7 +305,7 @@ class GCSFilesStore: def stat_file( self, path: str, info: MediaPipeline.SpiderInfo ) -> Deferred[StatInfo]: - def _onsuccess(blob) -> StatInfo: + def _onsuccess(blob: Any) -> StatInfo: if blob: checksum = base64.b64decode(blob.md5_hash).hex() last_modified = time.mktime(blob.updated.timetuple()) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index cea6b9c35..7bc29f28f 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -250,6 +250,7 @@ class MediaPipeline(ABC): # the encapsulated exception when it is a StopIteration instance context = getattr(result.value, "__context__", None) if isinstance(context, StopIteration): + assert result.value is not None result.value.__context__ = None info.downloading.remove(fp) diff --git a/scrapy/shell.py b/scrapy/shell.py index 00097d224..6f33f3f14 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -104,7 +104,7 @@ class Shell: event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] set_asyncio_event_loop(event_loop_path) - def crawl_request(_): + def crawl_request(_: None) -> None: assert self.crawler.engine is not None self.crawler.engine.crawl(request) diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 3ca9718d7..ca3b5b7dd 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -101,14 +101,14 @@ class SignalManager: kwargs.setdefault("sender", self.sender) _signal.disconnect_all(signal, **kwargs) - async def wait_for(self, signal): + async def wait_for(self, signal: Any) -> None: """Await the next *signal*. See :ref:`start-requests-lazy` for an example. """ - d = Deferred() + d: Deferred[None] = Deferred() - def handle(): + def handle() -> None: self.disconnect(handle, signal) d.callback(None) diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index f74842ac9..1f9894519 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -27,7 +27,7 @@ class StatsCollector: self._stats: StatsT = {} self._crawler: Crawler = crawler - def __getattribute__(self, name): + def __getattribute__(self, name: str) -> Any: cached_name = f"_cached_{name}" try: return super().__getattribute__(cached_name) diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index e34b7190f..6ac5e66b2 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -27,8 +27,10 @@ class Root(Resource): return b"" -def _getarg(request, name: bytes, default: Any = None, type_=str): - return type_(request.args[name][0]) if name in request.args else default +def _getarg( + request: Request, name: bytes, default: Any = None, type_: type = str +) -> Any: + return type_(request.args[name][0]) if name in request.args else default # type: ignore[index,operator] if __name__ == "__main__": diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 644965cb5..87cfd3c28 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -20,10 +20,10 @@ def _embed_ipython_shell( from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415 from IPython.terminal.ipapp import load_default_config # noqa: PLC0415 except ImportError: - from IPython.frontend.terminal.embed import ( # type: ignore[no-redef] # noqa: T100,PLC0415 + from IPython.frontend.terminal.embed import ( # type: ignore[import-not-found,no-redef] # noqa: T100,PLC0415 InteractiveShellEmbed, ) - from IPython.frontend.terminal.ipapp import ( # type: ignore[no-redef] # noqa: PLC0415 + from IPython.frontend.terminal.ipapp import ( # type: ignore[import-not-found,no-redef] # noqa: PLC0415 load_default_config, ) @@ -81,7 +81,7 @@ def _embed_standard_shell( else: import rlcompleter # noqa: F401,PLC0415 - readline.parse_and_bind("tab:complete") # type: ignore[attr-defined] + readline.parse_and_bind("tab:complete") # type: ignore[attr-defined,unused-ignore] @wraps(_embed_standard_shell) def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index ae7f627e1..64d2cde1b 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -83,7 +83,7 @@ class CaselessDict(dict): return dict.get(self, self.normkey(key), self.normvalue(def_val)) def setdefault(self, key: AnyStr, def_val: Any = None) -> Any: - return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) # type: ignore[arg-type] + return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) # doesn't fully implement MutableMapping.update() def update(self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]) -> None: # type: ignore[override] @@ -179,7 +179,7 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): with contextlib.suppress(TypeError): super().__setitem__(key, value) - def __getitem__(self, key: _KT) -> _VT | None: # type: ignore[override] + def __getitem__(self, key: _KT) -> _VT | None: try: return super().__getitem__(key) except (TypeError, KeyError): diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 0d937fea5..1acb576b4 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -21,7 +21,7 @@ from typing import ( overload, ) -from twisted.internet.defer import Deferred, DeferredList, fail, succeed +from twisted.internet.defer import Deferred, DeferredList, FirstError, fail, succeed from twisted.internet.task import Cooperator from twisted.python import failure @@ -334,6 +334,7 @@ def process_parallel( d2: Deferred[list[_T2]] = d.addCallback(lambda r: [x[1] for x in r]) def eb(failure: Failure) -> Failure: + assert isinstance(failure.value, FirstError) return failure.value.subFailure d2.addErrback(eb) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index e5b2ae23d..3bf6639c5 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -60,7 +60,7 @@ def create_deprecated_class( """ # https://github.com/python/mypy/issues/4177 - class DeprecatedClass(new_class.__class__): # type: ignore[misc, name-defined] + class DeprecatedClass(new_class.__class__): # type: ignore[misc,name-defined] # pylint: disable=no-self-argument deprecated_class: type | None = None warned_on_subclass: bool = False diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 5f0d19913..102362506 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -189,7 +189,7 @@ def log_scrapy_info(settings: Settings) -> None: "Scrapy %(version)s started (bot: %(bot)s)", {"version": scrapy.__version__, "bot": settings["BOT_NAME"]}, ) - software = settings.getlist("LOG_VERSIONS") + software: list[str] = settings.getlist("LOG_VERSIONS") if not software: return versions = pprint.pformat(dict(get_versions(software)), sort_dicts=False) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 6aa72064b..419c552bf 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -80,11 +80,11 @@ class CallLaterOnce(Generic[_T]): return result - async def wait(self): + async def wait(self) -> None: # circular import from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415 - d = Deferred() + d: Deferred[None] = Deferred() self._deferreds.append(d) await maybe_deferred_to_future(d) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 8cfa5cce4..097809cac 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -8,7 +8,7 @@ from __future__ import annotations import re import warnings from importlib import import_module -from typing import TYPE_CHECKING, TypeAlias +from typing import TYPE_CHECKING, Any, TypeAlias from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse from warnings import warn @@ -20,7 +20,7 @@ from w3lib.url import parse_url as _parse_url from scrapy.exceptions import ScrapyDeprecationWarning -def __getattr__(name: str): +def __getattr__(name: str) -> Any: if name in ("_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects): obj_type = "attribute" if name == "_safe_chars" else "function" warnings.warn( diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index b15063900..6d3572cb1 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -11,10 +11,10 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings.default_settings import LOG_VERSIONS from scrapy.utils.ssl import get_openssl_version -_DEFAULT_SOFTWARE = ["Scrapy", *LOG_VERSIONS] +_DEFAULT_SOFTWARE: list[str] = ["Scrapy", *LOG_VERSIONS] -def _version(item): +def _version(item: str) -> str: lowercase_item = item.lower() if lowercase_item == "libxml2": return ".".join(map(str, lxml.etree.LIBXML_VERSION)) @@ -28,7 +28,7 @@ def _version(item): def get_versions( - software: list | None = None, + software: list[str] | None = None, ) -> list[tuple[str, str]]: software = software or _DEFAULT_SOFTWARE return [(item, _version(item)) for item in software] diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py index 60d2dfd12..957ad35c9 100644 --- a/tests/test_downloader_handler_twisted_ftp.py +++ b/tests/test_downloader_handler_twisted_ftp.py @@ -47,7 +47,7 @@ class TestFTPBase(ABC): def _get_factory(self, tmp_path: Path) -> FTPFactory: raise NotImplementedError - @async_yield_fixture + @async_yield_fixture # type: ignore[untyped-decorator] async def server_url(self, tmp_path: Path) -> AsyncGenerator[str]: from twisted.internet import reactor diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index b9d368653..a2d421e39 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -10,8 +10,6 @@ from scrapy.utils.test import get_crawler class TestHttpProxyMiddleware: - failureException = AssertionError # type: ignore[assignment] - def setup_method(self): self._oldenv = os.environ.copy() diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index b5dab8861..af9941f3b 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -213,7 +213,7 @@ class TestHttps2ClientProtocol: r.putChild(b"request-headers", RequestHeaders()) return Site(r, timeout=None) - @async_yield_fixture + @async_yield_fixture # type: ignore[untyped-decorator] async def server_port(self, site: Site) -> AsyncGenerator[int]: from twisted.internet import reactor @@ -236,7 +236,7 @@ class TestHttps2ClientProtocol: ) + self.certificate_file.read_text(encoding="utf-8") return PrivateCertificate.loadPEM(pem) - @async_yield_fixture + @async_yield_fixture # type: ignore[untyped-decorator] async def client( self, server_port: int, client_certificate: PrivateCertificate ) -> AsyncGenerator[H2ClientProtocol]: diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 73e55b736..1276c1392 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -527,7 +527,7 @@ class TestBodyOrStr: r3 = _body_or_str(obj, unicode=False) self._assert_type_and_value(r3, self.bbody, obj) assert type(r1) is type(r2) - assert type(r1) is not type(r3) + assert type(r1) is not type(r3) # type: ignore[comparison-overlap] @staticmethod def _assert_type_and_value( diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing index 95f03e17c..a431091d5 100644 --- a/tests_typing/test_http_request.mypy-testing +++ b/tests_typing/test_http_request.mypy-testing @@ -15,7 +15,7 @@ class MyRequest2(Request): @pytest.mark.mypy_testing -def mypy_test_headers(): +def mypy_test_headers() -> None: Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" Request("data:,", headers=None) Request("data:,", headers={}) @@ -29,7 +29,7 @@ def mypy_test_headers(): @pytest.mark.mypy_testing -def mypy_test_copy(): +def mypy_test_copy() -> None: req = Request("data:,") reveal_type(req) # R: scrapy.http.request.Request req_copy = req.copy() @@ -37,7 +37,7 @@ def mypy_test_copy(): @pytest.mark.mypy_testing -def mypy_test_copy_subclass(): +def mypy_test_copy_subclass() -> None: req = MyRequest("data:,") reveal_type(req) # R: __main__.MyRequest req_copy = req.copy() @@ -45,7 +45,7 @@ def mypy_test_copy_subclass(): @pytest.mark.mypy_testing -def mypy_test_replace(): +def mypy_test_replace() -> None: req = Request("data:,") reveal_type(req) # R: scrapy.http.request.Request req_copy = req.replace(body=b"a") @@ -56,7 +56,7 @@ def mypy_test_replace(): @pytest.mark.mypy_testing -def mypy_test_replace_subclass(): +def mypy_test_replace_subclass() -> None: req = MyRequest("data:,") reveal_type(req) # R: __main__.MyRequest req_copy = req.replace(body=b"a") @@ -69,7 +69,7 @@ def mypy_test_replace_subclass(): @pytest.mark.mypy_testing -def mypy_test_jsonrequest_copy_replace(): +def mypy_test_jsonrequest_copy_replace() -> None: req = JsonRequest("data:,") reveal_type(req) # R: scrapy.http.request.json_request.JsonRequest req_copy = req.copy() diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing index 630754c1e..d497c2470 100644 --- a/tests_typing/test_http_response.mypy-testing +++ b/tests_typing/test_http_response.mypy-testing @@ -6,7 +6,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse @pytest.mark.mypy_testing -def mypy_test_headers(): +def mypy_test_headers() -> None: Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" Response("data:,", headers=None) Response("data:,", headers={}) @@ -20,7 +20,7 @@ def mypy_test_headers(): @pytest.mark.mypy_testing -def mypy_test_copy(): +def mypy_test_copy() -> None: resp = Response("data:,") reveal_type(resp) # R: scrapy.http.response.Response resp_copy = resp.copy() @@ -28,7 +28,7 @@ def mypy_test_copy(): @pytest.mark.mypy_testing -def mypy_test_copy_subclass(): +def mypy_test_copy_subclass() -> None: resp = HtmlResponse("data:,") reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse resp_copy = resp.copy() @@ -36,7 +36,7 @@ def mypy_test_copy_subclass(): @pytest.mark.mypy_testing -def mypy_test_replace(): +def mypy_test_replace() -> None: resp = Response("data:,") reveal_type(resp) # R: scrapy.http.response.Response resp_copy = resp.replace(body=b"a") @@ -47,7 +47,7 @@ def mypy_test_replace(): @pytest.mark.mypy_testing -def mypy_test_replace_subclass(): +def mypy_test_replace_subclass() -> None: resp = HtmlResponse("data:,") reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse resp_copy = resp.replace(body=b"a") diff --git a/tox.ini b/tox.ini index 7656cb017..1bebaf2aa 100644 --- a/tox.ini +++ b/tox.ini @@ -41,22 +41,27 @@ commands = [testenv:typing] basepython = python3.10 deps = - mypy==1.18.2 + mypy==1.19.1 typing-extensions==4.15.0 - types-defusedxml==0.7.0.20250822 - types-lxml==2025.8.25 - types-pexpect==4.9.0.20250916 - types-Pygments==2.19.0.20250809 - botocore-stubs==1.40.59 - boto3-stubs[s3]==1.40.59 + Pillow==12.1.1 + Protego==0.6.0 + attrs==25.4.0 + boto3-stubs[s3]==1.42.59 + botocore-stubs==1.42.41 + h2==4.3.0 httpx==0.28.1 - itemadapter==0.12.2 - Protego==0.5.0 - w3lib==2.3.1 - attrs >= 18.2.0 - Pillow >= 10.3.0 - pyOpenSSL >= 24.2.1 - pytest >= 8.2.0 + itemadapter==0.13.1 + ptpython==3.0.32 + ipython + pyOpenSSL==25.3.0 + pytest==9.0.2 + types-Pygments==2.19.0.20251121 + types-defusedxml==0.7.0.20250822 + types-lxml==2026.2.16 + types-pexpect==4.9.0.20260127 + uvloop==0.22.1 + w3lib==2.4.0 + zstandard==0.25.0 commands = mypy {posargs:scrapy tests} @@ -65,7 +70,7 @@ basepython = python3.10 deps = {[test-requirements]deps} {[testenv:typing]deps} - pytest-mypy-testing==0.1.3 + pytest-mypy-testing==0.2.0 commands = pytest {posargs:tests_typing}