From 55c17a89858e72c2e85391b1db3bbb2ba6c77360 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 18 May 2026 18:54:06 +0500 Subject: [PATCH] Disable mypy allow_any_generics. (#7532) --- pyproject.toml | 1 - scrapy/commands/genspider.py | 2 +- scrapy/contracts/__init__.py | 12 +++++----- scrapy/core/downloader/handlers/s3.py | 6 +++-- scrapy/core/downloader/middleware.py | 10 ++++----- scrapy/core/spidermw.py | 14 +++++++----- scrapy/extensions/httpcache.py | 4 +++- scrapy/http/response/text.py | 2 +- scrapy/middleware.py | 6 ++--- scrapy/spiders/crawl.py | 4 +++- scrapy/squeues.py | 2 +- scrapy/utils/asyncio.py | 2 +- scrapy/utils/conf.py | 4 ++-- scrapy/utils/datatypes.py | 22 +++++++++---------- scrapy/utils/defer.py | 2 +- scrapy/utils/log.py | 3 ++- scrapy/utils/misc.py | 6 +++-- scrapy/utils/reactor.py | 2 +- scrapy/utils/signal.py | 4 +++- scrapy/utils/template.py | 2 +- scrapy/utils/trackref.py | 4 +++- .../reactorless_custom_settings.py | 2 +- .../twisted_reactor_custom_settings_select.py | 2 +- tests/mocks/dummydbm.py | 2 +- tests/spiders.py | 13 ++++++----- tests/test_command_genspider.py | 2 +- tests/test_command_shell.py | 5 ++--- tests/test_command_startproject.py | 2 +- tests/test_feedexport.py | 6 +++-- tests/test_feedexport_batch.py | 2 +- tests/test_http2_client_protocol.py | 2 +- tests/test_loader.py | 2 +- tests/test_pipeline_files.py | 8 +++---- tests/test_pipeline_images.py | 8 +++---- tests/test_scheduler_base.py | 4 ++-- tests/test_spidermiddleware.py | 2 +- tests/test_utils_datatypes.py | 5 +++-- tests/test_utils_log.py | 4 +++- tests/test_utils_request.py | 5 +++-- tox.ini | 10 ++++----- 40 files changed, 112 insertions(+), 88 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 76b8bbead..2f07dcd80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,6 @@ pattern = "^(?P.+)$" [tool.mypy] strict = true -allow_any_generics = true # 67 errors extra_checks = false # weird addErrback() errors untyped_calls_exclude = [ "twisted", diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 17bcf19b0..cc8624fa1 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -147,7 +147,7 @@ class Command(ScrapyCommand): name: str, url: str, template_name: str, - template_file: str | os.PathLike, + template_file: str | os.PathLike[str], ) -> None: """Generate the spider module, based on the given template""" assert self.settings is not None diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 24f56b7a6..c0da7dfa4 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -27,7 +27,7 @@ class Contract: request_cls: type[Request] | None = None name: str - def __init__(self, method: Callable, *args: Any): + def __init__(self, method: Callable[..., Any], *args: Any): self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook") self.testcase_post = _create_testcase(method, f"@{self.name} post-hook") self.args: tuple[Any, ...] = args @@ -105,7 +105,7 @@ class ContractsManager: return methods - def extract_contracts(self, method: Callable) -> list[Contract]: + def extract_contracts(self, method: Callable[..., Any]) -> list[Contract]: contracts: list[Contract] = [] assert method.__doc__ is not None for line_ in method.__doc__.split("\n"): @@ -134,7 +134,9 @@ class ContractsManager: return requests - def from_method(self, method: Callable, results: TestResult) -> Request | None: + def from_method( + self, method: Callable[..., Any], results: TestResult + ) -> Request | None: contracts = self.extract_contracts(method) if contracts: request_cls = Request @@ -170,7 +172,7 @@ class ContractsManager: return None def _clean_req( - self, request: Request, method: Callable, results: TestResult + self, request: Request, method: Callable[..., Any], results: TestResult ) -> None: """stop the request from returning objects and records any errors""" @@ -195,7 +197,7 @@ class ContractsManager: request.errback = eb_wrapper -def _create_testcase(method: Callable, desc: str) -> TestCase: +def _create_testcase(method: Callable[..., Any], desc: str) -> TestCase: spider = method.__self__.name # type: ignore[attr-defined] class ContractTestCase(TestCase): diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 19a1e8503..c6660ff48 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, cast from scrapy.core.downloader.handlers.base import BaseDownloadHandler from scrapy.exceptions import NotConfigured @@ -9,6 +9,8 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: + from collections.abc import Mapping + from scrapy import Request from scrapy.crawler import Crawler from scrapy.http import Response @@ -59,7 +61,7 @@ class S3DownloadHandler(BaseDownloadHandler): awsrequest = botocore.awsrequest.AWSRequest( method=request.method, url=f"{scheme}://s3.amazonaws.com/{bucket}{path}", - headers=request.headers.to_unicode_dict(), + headers=cast("Mapping[str, Any]", request.headers.to_unicode_dict()), data=request.body, ) assert self._signer diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 6685d90af..ab74e22a4 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -8,7 +8,7 @@ from __future__ import annotations import warnings from functools import wraps -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Request, Response @@ -87,7 +87,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): result = await self._process_exception(ex, request) return await self._process_response(result, request) - def _handle_mw_method(self, method: Callable, **kwargs: Any) -> Any: + def _handle_mw_method(self, method: Callable[..., Any], **kwargs: Any) -> Any: if method in self._mw_methods_requiring_spider: kwargs["spider"] = self._spider @@ -99,7 +99,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): download_func: Callable[[Request], Coroutine[Any, Any, Response]], ) -> Response | Request: for method in self.methods["process_request"]: - method = cast("Callable", method) + assert method is not None response = await ensure_awaitable( self._handle_mw_method(method, request=request), _warn=global_object_name(method), @@ -122,7 +122,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response for method in self.methods["process_response"]: - method = cast("Callable", method) + assert method is not None response = await ensure_awaitable( self._handle_mw_method(method, request=request, response=response), _warn=global_object_name(method), @@ -141,7 +141,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): self, exception: Exception, request: Request | Response ) -> Response | Request: for method in self.methods["process_exception"]: - method = cast("Callable", method) + assert method is not None response = await ensure_awaitable( self._handle_mw_method(method, request=request, exception=exception), _warn=global_object_name(method), diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index bcf38bb75..fbc6f2530 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -11,7 +11,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable from functools import wraps from inspect import isasyncgenfunction from itertools import islice -from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast +from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar from warnings import warn from twisted.python.failure import Failure @@ -79,7 +79,7 @@ class SpiderMiddlewareManager(MiddlewareManager): request: Request, ) -> Iterable[_T] | AsyncIterator[_T]: for method in self.methods["process_spider_input"]: - method = cast("Callable", method) + assert method is not None try: if method in self._mw_methods_requiring_spider: result = method(response=response, spider=self._spider) @@ -248,9 +248,13 @@ class SpiderMiddlewareManager(MiddlewareManager): # This method is only needed until _async compatibility methods are removed. @staticmethod - def _get_process_spider_output(mw: Any) -> Callable | None: - normal_method: Callable | None = getattr(mw, "process_spider_output", None) - async_method: Callable | None = getattr(mw, "process_spider_output_async", None) + def _get_process_spider_output(mw: Any) -> Callable[..., Any] | None: + normal_method: Callable[..., Any] | None = getattr( + mw, "process_spider_output", None + ) + async_method: Callable[..., Any] | None = getattr( + mw, "process_spider_output_async", None + ) if not async_method: if normal_method and not isasyncgenfunction(normal_method): raise TypeError( diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 86b066a38..dbb79b02d 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -313,7 +313,9 @@ class FilesystemCacheStorage: self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP") # https://github.com/python/mypy/issues/10740 - self._open: Callable[Concatenate[str | os.PathLike, str, ...], IO[bytes]] = ( + self._open: Callable[ + Concatenate[str | os.PathLike[str], str, ...], IO[bytes] + ] = ( gzip.open if self.use_gzip else open # type: ignore[assignment] ) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 13853f64d..6876e35e8 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -220,7 +220,7 @@ class TextResponse(Response): def follow_all( self, - urls: Iterable[str | Link] | parsel.SelectorList | None = None, + urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None, callback: CallbackT | None = None, method: str = "GET", headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 14b591a5f..37ec0f583 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -50,8 +50,8 @@ class MiddlewareManager(ABC): ) self.middlewares: tuple[Any, ...] = middlewares # Only process_spider_output and process_spider_exception can be None. - self.methods: dict[str, deque[Callable | None]] = defaultdict(deque) - self._mw_methods_requiring_spider: set[Callable] = set() + self.methods: dict[str, deque[Callable[..., Any] | None]] = defaultdict(deque) + self._mw_methods_requiring_spider: set[Callable[..., Any]] = set() for mw in middlewares: self._add_middleware(mw) @@ -116,7 +116,7 @@ class MiddlewareManager(ABC): def _add_middleware(self, mw: Any) -> None: # noqa: B027 pass - def _check_mw_method_spider_arg(self, method: Callable) -> None: + def _check_mw_method_spider_arg(self, method: Callable[..., Any]) -> None: if argument_is_required(method, "spider"): warnings.warn( f"{method.__qualname__}() requires a spider argument," diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index f0d093c6e..e14279b64 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -46,7 +46,9 @@ def _identity_process_request(request: Request, response: Response) -> Request | return request -def _get_method(method: Callable | str | None, spider: Spider) -> Callable | None: +def _get_method( + method: Callable[..., Any] | str | None, spider: Spider +) -> Callable[..., Any] | None: if callable(method): return method if isinstance(method, str): diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 7007cd4b8..e06e38e23 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -26,7 +26,7 @@ if TYPE_CHECKING: def _with_mkdir(queue_class: type[queue.BaseQueue]) -> type[queue.BaseQueue]: class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] - def __init__(self, path: str | PathLike, *args: Any, **kwargs: Any): + def __init__(self, path: str | PathLike[str], *args: Any, **kwargs: Any): dirname = Path(path).parent if not dirname.exists(): dirname.mkdir(parents=True, exist_ok=True) diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index ecb4c1492..44604c0fe 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -148,7 +148,7 @@ class AsyncioLoopingCall: self._func: Callable[_P, _T] = func self._args: tuple[Any, ...] = args self._kwargs: dict[str, Any] = kwargs - self._task: asyncio.Task | None = None + self._task: asyncio.Task[None] | None = None self.interval: float | None = None self._start_time: float | None = None diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 5869cf52e..4850b370b 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -71,8 +71,8 @@ def arglist_to_dict(arglist: list[str]) -> dict[str, str]: def closest_scrapy_cfg( - path: str | os.PathLike = ".", - prevpath: str | os.PathLike | None = None, + path: str | os.PathLike[str] = ".", + prevpath: str | os.PathLike[str] | None = None, ) -> str: """Return the path to the closest scrapy.cfg file by traversing the current directory and its parents diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 146e3ae56..4e65c062e 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -28,7 +28,7 @@ _KT = TypeVar("_KT") _VT = TypeVar("_VT") -class CaselessDict(dict): +class CaselessDict(dict): # type: ignore[type-arg] __slots__ = () def __new__(cls, *args: Any, **kwargs: Any) -> Self: @@ -99,20 +99,20 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) -class CaseInsensitiveDict(collections.UserDict): +class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]): """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups. """ def __init__(self, *args: Any, **kwargs: Any) -> None: - self._keys: dict = {} + self._keys: dict[str | bytes, Any] = {} super().__init__(*args, **kwargs) - def __getitem__(self, key: AnyStr) -> Any: + def __getitem__(self, key: str | bytes) -> Any: normalized_key = self._normkey(key) return super().__getitem__(self._keys[normalized_key.lower()]) - def __setitem__(self, key: AnyStr, value: Any) -> None: + def __setitem__(self, key: str | bytes, value: Any) -> None: normalized_key = self._normkey(key) try: lower_key = self._keys[normalized_key.lower()] @@ -122,19 +122,19 @@ class CaseInsensitiveDict(collections.UserDict): super().__setitem__(normalized_key, self._normvalue(value)) self._keys[normalized_key.lower()] = normalized_key - def __delitem__(self, key: AnyStr) -> None: + def __delitem__(self, key: str | bytes) -> None: normalized_key = self._normkey(key) stored_key = self._keys.pop(normalized_key.lower()) super().__delitem__(stored_key) - def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] + def __contains__(self, key: str | bytes) -> bool: # type: ignore[override] normalized_key = self._normkey(key) return normalized_key.lower() in self._keys def __repr__(self) -> str: return f"<{self.__class__.__name__}: {super().__repr__()}>" - def _normkey(self, key: AnyStr) -> AnyStr: + def _normkey(self, key: str | bytes) -> str | bytes: return key def _normvalue(self, value: Any) -> Any: @@ -158,7 +158,7 @@ class LocalCache(OrderedDict[_KT, _VT]): super().__setitem__(key, value) -class LocalWeakReferencedCache(weakref.WeakKeyDictionary): +class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]): """ A weakref.WeakKeyDictionary implementation that uses LocalCache as its underlying data structure, making it ordered and capable of being size-limited. @@ -172,9 +172,9 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): def __init__(self, limit: int | None = None): super().__init__() - self.data: LocalCache = LocalCache(limit=limit) + self.data: LocalCache[_KT, _VT] = LocalCache(limit=limit) - def __setitem__(self, key: _KT, value: _VT) -> None: + def __setitem__(self, key: _KT, value: _VT | None) -> None: # if raised, key is not weak-referenceable, skip caching with contextlib.suppress(TypeError): super().__setitem__(key, value) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 90fd04777..29a34d4ef 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -173,7 +173,7 @@ def parallel( return DeferredList([coop.coiterate(work) for _ in range(count)]) -class _AsyncCooperatorAdapter(Iterator, Generic[_T]): +class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_T]): """A class that wraps an async iterable into a normal iterator suitable for using in Cooperator.coiterate(). As it's only needed for parallel_async(), it calls the callable directly in the callback, instead of providing a more diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 102362506..ee65d4155 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -260,7 +260,8 @@ def logformatter_adapter( return (level, message, args) -class SpiderLoggerAdapter(logging.LoggerAdapter): +# LoggerAdapter is only parameterized since Python 3.11 +class SpiderLoggerAdapter(logging.LoggerAdapter): # type: ignore[type-arg] def process( self, msg: str, kwargs: MutableMapping[str, Any] ) -> tuple[str, MutableMapping[str, Any]]: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 3baa02e2b..0b67eaa34 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -28,7 +28,7 @@ if TYPE_CHECKING: _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes -_ITER_T = TypeVar("_ITER_T", bound=dict | Item | str | bytes) +_ITER_T = TypeVar("_ITER_T", bound=dict[Any, Any] | Item | str | bytes) _T = TypeVar("_T") _T_co = TypeVar("_T_co", covariant=True) _P = ParamSpec("_P") @@ -252,7 +252,9 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]: yield node -_generator_callbacks_cache = LocalWeakReferencedCache(limit=128) +_generator_callbacks_cache: LocalWeakReferencedCache[Callable[..., Any], bool] = ( + LocalWeakReferencedCache(limit=128) +) def _returns_none(return_node: ast.Return) -> bool: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 37db6c25f..7ab58093a 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -55,7 +55,7 @@ class CallLaterOnce(Generic[_T]): self._a: tuple[Any, ...] = a self._kw: dict[str, Any] = kw self._call: CallLaterResult | None = None - self._deferreds: list[Deferred] = [] + self._deferreds: list[Deferred[None]] = [] def schedule(self, delay: float = 0) -> None: # circular import diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index d9a72273a..ee391a49e 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -186,7 +186,9 @@ async def _send_catch_log_asyncio( handlers: list[Awaitable[TypingAny]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): - async def handler(receiver: Callable) -> tuple[Callable, TypingAny]: + async def handler( + receiver: Callable[..., Any], + ) -> tuple[Callable[..., Any], TypingAny]: result: TypingAny try: result = await ensure_awaitable( diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 3e4dae5c8..977d5a42a 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -11,7 +11,7 @@ if TYPE_CHECKING: from os import PathLike -def render_templatefile(path: str | PathLike, **kwargs: Any) -> None: +def render_templatefile(path: str | PathLike[str], **kwargs: Any) -> None: path_obj = Path(path) raw = path_obj.read_text("utf8") diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 082aca4b1..87df10a02 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -30,7 +30,9 @@ if TYPE_CHECKING: from typing_extensions import Self -live_refs: defaultdict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) +live_refs: defaultdict[type, WeakKeyDictionary[object, float]] = defaultdict( + WeakKeyDictionary +) class object_ref: diff --git a/tests/AsyncCrawlerProcess/reactorless_custom_settings.py b/tests/AsyncCrawlerProcess/reactorless_custom_settings.py index 93259f696..f49741441 100644 --- a/tests/AsyncCrawlerProcess/reactorless_custom_settings.py +++ b/tests/AsyncCrawlerProcess/reactorless_custom_settings.py @@ -23,7 +23,7 @@ class NoRequestsSpider(scrapy.Spider): yield -def log_task_exception(task: Task) -> None: +def log_task_exception(task: Task[None]) -> None: try: task.result() except Exception: diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py index f6fead718..b85f60f87 100644 --- a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py +++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py @@ -17,7 +17,7 @@ class AsyncioReactorSpider(scrapy.Spider): } -def log_task_exception(task: Task) -> None: +def log_task_exception(task: Task[None]) -> None: try: task.result() except Exception: diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index e358eaca4..98fe58860 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -4,7 +4,7 @@ from collections import defaultdict from typing import Any -class DummyDB(dict): +class DummyDB(dict): # type: ignore[type-arg] """Provide dummy DBM-like interface.""" def close(self): diff --git a/tests/spiders.py b/tests/spiders.py index 373dc00bd..065a0e6d7 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -22,6 +22,7 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future from scrapy.utils.test import get_from_asyncio_queue if TYPE_CHECKING: + from scrapy.settings import _SettingsKey from tests.mockserver.http import MockServer @@ -94,19 +95,19 @@ class DelaySpider(MetaSpider): class LogSpider(MetaSpider): name = "log_spider" - def log_debug(self, message: str, extra: dict | None = None): + def log_debug(self, message: str, extra: dict[str, Any] | None = None): self.logger.debug(message, extra=extra) - def log_info(self, message: str, extra: dict | None = None): + def log_info(self, message: str, extra: dict[str, Any] | None = None): self.logger.info(message, extra=extra) - def log_warning(self, message: str, extra: dict | None = None): + def log_warning(self, message: str, extra: dict[str, Any] | None = None): self.logger.warning(message, extra=extra) - def log_error(self, message: str, extra: dict | None = None): + def log_error(self, message: str, extra: dict[str, Any] | None = None): self.logger.error(message, extra=extra) - def log_critical(self, message: str, extra: dict | None = None): + def log_critical(self, message: str, extra: dict[str, Any] | None = None): self.logger.critical(message, extra=extra) def parse(self, response): @@ -417,7 +418,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): """ name = "crawl_spider_with_parse_method" - custom_settings: dict = { + custom_settings: dict[_SettingsKey, Any] = { "RETRY_HTTP_CODES": [], # no need to retry } rules = (Rule(LinkExtractor(), callback="parse", follow=True),) diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py index fd9505060..67e3eb50a 100644 --- a/tests/test_command_genspider.py +++ b/tests/test_command_genspider.py @@ -9,7 +9,7 @@ from tests.test_commands import TestProjectBase from tests.utils.cmdline import call, proc -def find_in_file(filename: Path, regex: str) -> re.Match | None: +def find_in_file(filename: Path, regex: str) -> re.Match[str] | None: """Find first pattern occurrence in file""" pattern = re.compile(regex) with filename.open("r", encoding="utf-8") as f: diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index b407211dd..1585835cc 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -4,7 +4,7 @@ import os import sys from io import BytesIO from pathlib import Path -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING import pytest from pexpect.popen_spawn import PopenSpawn @@ -151,8 +151,7 @@ class TestInteractiveShell: env = os.environ.copy() env["SCRAPY_PYTHON_SHELL"] = "python" logfile = BytesIO() - # https://github.com/python/typeshed/issues/14915 - p = PopenSpawn(args, env=cast("os._Environ", env), timeout=5) + p = PopenSpawn(args, env=env, timeout=5) p.logfile_read = logfile p.expect_exact("Available Scrapy objects") p.sendline(f"fetch('{mockserver.url('/')}')") diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py index 4eed09fcd..2a9d0ed57 100644 --- a/tests/test_command_startproject.py +++ b/tests/test_command_startproject.py @@ -79,7 +79,7 @@ class TestStartprojectCommand: def get_permissions_dict( - path: str | os.PathLike, renamings=None, ignore=None + path: str | os.PathLike[str], renamings=None, ignore=None ) -> dict[str, str]: def get_permissions(path: Path) -> str: return oct(path.stat().st_mode) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index cbf568524..27e0c6445 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -41,7 +41,7 @@ from tests.spiders import ItemSpider from tests.utils.decorators import coroutine_test, inline_callbacks_test if TYPE_CHECKING: - from collections.abc import Callable, Iterable + from collections.abc import Awaitable, Callable, Iterable def path_to_url(path: str | Path) -> str: @@ -1312,7 +1312,9 @@ class TestFeedExporterSignals: self.feed_slot_closed_received = True async def run_signaled_feed_exporter( - self, feed_exporter_signal_handler: Callable, feed_slot_signal_handler: Callable + self, + feed_exporter_signal_handler: Callable[[], Awaitable[None] | None], + feed_slot_signal_handler: Callable[[Any], Awaitable[None] | None], ) -> None: crawler = get_crawler(settings_dict=self.settings) feed_exporter = FeedExporter.from_crawler(crawler) diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py index 3b70e896c..d855d0f74 100644 --- a/tests/test_feedexport_batch.py +++ b/tests/test_feedexport_batch.py @@ -29,7 +29,7 @@ if TYPE_CHECKING: from os import PathLike -def build_url(path: str | PathLike) -> str: +def build_url(path: str | PathLike[str]) -> str: path_str = str(path) if path_str[0] != "/": path_str = "/" + path_str diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 6072e2f6d..cec5d728b 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -61,7 +61,7 @@ def make_html_body(val: str) -> bytes: class DummySpider(Spider): name = "dummy" - start_urls: list = [] + start_urls = [] def parse(self, response): print(response) diff --git a/tests/test_loader.py b/tests/test_loader.py index 7cfc7ed26..c094d25d8 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -39,7 +39,7 @@ class AttrsNameItem: @dataclasses.dataclass class NameDataClass: - name: list = dataclasses.field(default_factory=list) + name: list[str] = dataclasses.field(default_factory=list) # test item loaders diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index dbe8d85a5..c6a41fa6e 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -383,11 +383,11 @@ class TestFilesPipelineFieldsItem(TestFilesPipelineFieldsMixin): class FilesPipelineTestDataClass: name: str # default fields - file_urls: list = dataclasses.field(default_factory=list) - files: list = dataclasses.field(default_factory=list) + file_urls: list[str] = dataclasses.field(default_factory=list) + files: list[dict[str, str]] = dataclasses.field(default_factory=list) # overridden fields - custom_file_urls: list = dataclasses.field(default_factory=list) - custom_files: list = dataclasses.field(default_factory=list) + custom_file_urls: list[str] = dataclasses.field(default_factory=list) + custom_files: list[dict[str, str]] = dataclasses.field(default_factory=list) class TestFilesPipelineFieldsDataClass(TestFilesPipelineFieldsMixin): diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 199ec5afa..38662348f 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -314,11 +314,11 @@ class TestImagesPipelineFieldsItem(TestImagesPipelineFieldsMixin): class ImagesPipelineTestDataClass: name: str # default fields - image_urls: list = dataclasses.field(default_factory=list) - images: list = dataclasses.field(default_factory=list) + image_urls: list[str] = dataclasses.field(default_factory=list) + images: list[dict[str, str]] = dataclasses.field(default_factory=list) # overridden fields - custom_image_urls: list = dataclasses.field(default_factory=list) - custom_images: list = dataclasses.field(default_factory=list) + custom_image_urls: list[str] = dataclasses.field(default_factory=list) + custom_images: list[dict[str, str]] = dataclasses.field(default_factory=list) class TestImagesPipelineFieldsDataClass(TestImagesPipelineFieldsMixin): diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 176fefbbf..db023e1f8 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -41,10 +41,10 @@ class MinimalScheduler: class SimpleScheduler(MinimalScheduler): - def open(self, spider: Spider) -> defer.Deferred: + def open(self, spider: Spider) -> defer.Deferred[str]: return defer.succeed("open") - def close(self, reason: str) -> defer.Deferred: + def close(self, reason: str) -> defer.Deferred[str]: return defer.succeed("close") def __len__(self) -> int: diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 2e0436ae1..a0d296553 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -110,7 +110,7 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): Should work for process_spider_output and, when it's supported, process_start. """ - ITEM_TYPE: type | tuple + ITEM_TYPE: type | tuple[type, ...] RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects @staticmethod diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index c58b02ca5..f573993a1 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -2,6 +2,7 @@ import copy import warnings from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, MutableMapping +from typing import Any import pytest @@ -20,7 +21,7 @@ from scrapy.utils.python import garbage_collect class TestCaseInsensitiveDictBase(ABC): @property @abstractmethod - def dict_class(self) -> type[MutableMapping]: + def dict_class(self) -> type[MutableMapping[str, Any]]: raise NotImplementedError def test_init_dict(self): @@ -206,7 +207,7 @@ class TestCaseInsensitiveDictBase(ABC): class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): - dict_class = CaseInsensitiveDict + dict_class = CaseInsensitiveDict # type: ignore[assignment] def test_repr(self): d1 = self.dict_class({"foo": "bar"}) diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index f40e424ff..318370d40 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -144,7 +144,9 @@ class TestStreamLogger: ], ) def test_spider_logger_adapter_process( - base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict + base_extra: Mapping[str, Any], + log_extra: MutableMapping[str, Any], + expected_extra: dict[str, Any], ) -> None: logger = logging.getLogger("test") spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index b25129c87..70800dcec 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -2,6 +2,7 @@ from __future__ import annotations import json from hashlib import sha1 +from typing import Any from weakref import WeakKeyDictionary import pytest @@ -56,13 +57,13 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None: class TestFingerprint: - function: staticmethod = staticmethod(fingerprint) + function: staticmethod[[Request], bytes] = staticmethod(fingerprint) cache: ( WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]] | WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], str]] ) = _fingerprint_cache default_cache_key = (None, False) - known_hashes: tuple[tuple[Request, bytes | str, dict], ...] = ( + known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = ( ( Request("http://example.org"), b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD", diff --git a/tox.ini b/tox.ini index da5a8a9a6..5fef37e5b 100644 --- a/tox.ini +++ b/tox.ini @@ -44,13 +44,13 @@ commands = [testenv:typing] basepython = python3.10 deps = - mypy==1.20.2 + mypy==2.1.0 typing-extensions==4.15.0 Pillow==12.2.0 Protego==0.6.0 Twisted==26.4.0 attrs==26.1.0 - boto3-stubs[s3]==1.43.2 + boto3-stubs[s3]==1.43.9 botocore-stubs==1.42.41 h2==4.3.0 httpx==0.28.1 @@ -58,12 +58,12 @@ deps = ptpython==3.0.32 # newer ones require newer Python ipython==8.39.0 - pyOpenSSL==26.1.0 + pyOpenSSL==26.2.0 pytest==9.0.3 - types-Pygments==2.20.0.20260408 + types-Pygments==2.20.0.20260508 types-defusedxml==0.7.0.20260504 types-lxml==2026.2.16 - types-pexpect==4.9.0.20260408 + types-pexpect==4.9.0.20260508 uvloop==0.22.1 w3lib==2.4.1 zstandard==0.25.0