From d414d393d4f7bff5a7d059346ea7b6c45e0ba256 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 27 Oct 2025 16:37:49 +0500 Subject: [PATCH] Remove Python 3.9 support (#7121) --- .github/workflows/checks.yml | 4 +- .github/workflows/tests-macos.yml | 2 +- .github/workflows/tests-ubuntu.yml | 11 ++- .github/workflows/tests-windows.yml | 7 +- README.rst | 2 +- docs/intro/install.rst | 2 +- docs/topics/media-pipeline.rst | 4 +- pyproject.toml | 7 +- scrapy/cmdline.py | 13 +--- scrapy/contracts/default.py | 5 +- scrapy/core/scraper.py | 4 +- scrapy/core/spidermw.py | 16 ++--- scrapy/extensions/feedexport.py | 6 +- scrapy/extensions/httpcache.py | 5 +- scrapy/http/headers.py | 16 ++--- scrapy/http/request/__init__.py | 11 +-- scrapy/http/request/form.py | 8 +-- scrapy/linkextractors/lxmlhtml.py | 14 ++-- scrapy/middleware.py | 8 +-- scrapy/pipelines/__init__.py | 4 +- scrapy/pipelines/images.py | 2 +- scrapy/pipelines/media.py | 6 +- scrapy/settings/__init__.py | 72 ++++++++++---------- scrapy/spiders/__init__.py | 4 +- scrapy/spiders/crawl.py | 6 +- scrapy/utils/asyncio.py | 7 +- scrapy/utils/conf.py | 4 +- scrapy/utils/decorators.py | 8 +-- scrapy/utils/defer.py | 19 ++++-- scrapy/utils/iterators.py | 2 +- scrapy/utils/log.py | 8 +-- scrapy/utils/ossignal.py | 8 +-- scrapy/utils/python.py | 9 ++- scrapy/utils/reactor.py | 7 +- scrapy/utils/url.py | 4 +- tests/test_crawl.py | 2 +- tests/test_extension_periodic_log.py | 6 +- tests/test_feedexport.py | 4 +- tests/test_http2_client_protocol.py | 4 +- tests/test_http_response.py | 6 +- tests_typing/test_http_request.mypy-testing | 2 +- tests_typing/test_http_response.mypy-testing | 2 +- tox.ini | 20 +++--- 43 files changed, 175 insertions(+), 186 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 312af3b2e..ee5f4b1f9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -20,10 +20,10 @@ jobs: - python-version: "3.13" env: TOXENV: pylint - - python-version: "3.9" + - python-version: "3.10" env: TOXENV: typing - - python-version: "3.9" + - python-version: "3.10" env: TOXENV: typing-tests - python-version: "3.13" # Keep in sync with .readthedocs.yml diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index d740808cc..0e9e108cc 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 23e60df9f..0a5852b6c 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -17,9 +17,6 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.9" - env: - TOXENV: py - python-version: "3.10" env: TOXENV: py @@ -40,19 +37,19 @@ jobs: TOXENV: pypy3 # pinned deps - - python-version: "3.9.21" + - python-version: "3.10.19" env: TOXENV: pinned - - python-version: "3.9.21" + - python-version: "3.10.19" env: TOXENV: default-reactor-pinned - python-version: pypy3.11 env: TOXENV: pypy3-pinned - - python-version: "3.9.21" + - python-version: "3.10.19" env: TOXENV: extra-deps-pinned - - python-version: "3.9.21" + - python-version: "3.10.19" env: TOXENV: botocore-pinned diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index bbbb704e5..4da82fd0b 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -17,9 +17,6 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.9" - env: - TOXENV: py - python-version: "3.10" env: TOXENV: py @@ -37,10 +34,10 @@ jobs: TOXENV: default-reactor # pinned deps - - python-version: "3.9.13" + - python-version: "3.10.11" env: TOXENV: pinned - - python-version: "3.9.13" + - python-version: "3.10.11" env: TOXENV: extra-deps-pinned diff --git a/README.rst b/README.rst index 536dec7f0..6235cb20c 100644 --- a/README.rst +++ b/README.rst @@ -40,7 +40,7 @@ :alt: Ask DeepWiki Scrapy_ is a web scraping framework to extract structured data from websites. -It is cross-platform, and requires Python 3.9+. It is maintained by Zyte_ +It is cross-platform, and requires Python 3.10+. It is maintained by Zyte_ (formerly Scrapinghub) and `many other contributors`_. .. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 488a66f36..0810e7274 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,7 +9,7 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.9+, either the CPython implementation (default) or +Scrapy requires Python 3.10+, either the CPython implementation (default) or the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 01da53342..fa17dfda6 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you can configure some extra functions like generating thumbnails and filtering the images based on their size. -The Images Pipeline requires Pillow_ 8.0.0 or greater. It is used for +The Images Pipeline requires Pillow_ 8.3.2 or greater. It is used for thumbnailing and normalizing images to JPEG/RGB format. .. _Pillow: https://github.com/python-pillow/Pillow @@ -238,7 +238,7 @@ Amazon S3 storage .. setting:: FILES_STORE_S3_ACL .. setting:: IMAGES_STORE_S3_ACL -If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and +If botocore_ >= 1.13.45 is installed, :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will automatically upload the files to the bucket. diff --git a/pyproject.toml b/pyproject.toml index 47bdd3a6b..055faa493 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "defusedxml>=0.7.1", "itemadapter>=0.1.0", "itemloaders>=1.0.1", - "lxml>=4.6.0", + "lxml>=4.6.4", "packaging", "parsel>=1.5.0", "protego>=0.1.15", @@ -35,7 +35,6 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -49,7 +48,7 @@ classifiers = [ license = "BSD-3-Clause" license-files = ["LICENSE", "AUTHORS"] readme = "README.rst" -requires-python = ">=3.9" +requires-python = ">=3.10" authors = [{ name = "Scrapy developers", email = "pablo@pablohoffman.com" }] maintainers = [{ name = "Pablo Hoffman", email = "pablo@pablohoffman.com" }] @@ -375,6 +374,8 @@ ignore = [ "SIM115", # Yoda condition detected "SIM300", + # removed in recent ruff + "UP038", # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 2b0204071..487980abd 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -6,7 +6,7 @@ import inspect import os import sys from importlib.metadata import entry_points -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, ParamSpec import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter @@ -20,12 +20,9 @@ from scrapy.utils.reactor import _asyncio_reactor_path if TYPE_CHECKING: from collections.abc import Callable, Iterable - # typing.ParamSpec requires Python 3.10 - from typing_extensions import ParamSpec - from scrapy.settings import BaseSettings, Settings - _P = ParamSpec("_P") +_P = ParamSpec("_P") class ScrapyArgumentParser(argparse.ArgumentParser): @@ -67,11 +64,7 @@ def _get_commands_from_entry_points( inproject: bool, group: str = "scrapy.commands" ) -> dict[str, ScrapyCommand]: cmds: dict[str, ScrapyCommand] = {} - if sys.version_info >= (3, 10): - eps = entry_points(group=group) - else: - eps = entry_points().get(group, ()) - for entry_point in eps: + for entry_point in entry_points(group=group): obj = entry_point.load() if inspect.isclass(obj): cmds[entry_point.name] = obj() diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 6f357ba20..90f054a87 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,7 +1,7 @@ from __future__ import annotations import json -from typing import Any, Callable +from typing import TYPE_CHECKING, Any from itemadapter import ItemAdapter, is_item @@ -9,6 +9,9 @@ from scrapy.contracts import Contract from scrapy.exceptions import ContractFail from scrapy.http import Request +if TYPE_CHECKING: + from collections.abc import Callable + # contracts class UrlContract(Contract): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 84cc6a0ee..d29dc2c06 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -7,7 +7,7 @@ import logging import warnings from collections import deque from collections.abc import AsyncIterator -from typing import TYPE_CHECKING, Any, TypeVar, Union +from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure @@ -53,7 +53,7 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -QueueTuple = tuple[Union[Response, Failure], Request, Deferred[None]] +QueueTuple: TypeAlias = tuple[Response | Failure, Request, Deferred[None]] class Slot: diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 75246710d..6f694753a 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -11,7 +11,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable from functools import wraps from inspect import isasyncgenfunction, iscoroutine from itertools import islice -from typing import TYPE_CHECKING, Any, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast from warnings import warn from twisted.internet.defer import Deferred, inlineCallbacks @@ -41,9 +41,9 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -ScrapeFunc = Callable[ - [Union[Response, Failure], Request], - Coroutine[Any, Any, Union[Iterable[_T], AsyncIterator[_T]]], +ScrapeFunc: TypeAlias = Callable[ + [Response | Failure, Request], + Coroutine[Any, Any, Iterable[_T] | AsyncIterator[_T]], ] @@ -179,7 +179,7 @@ class SpiderMiddlewareManager(MiddlewareManager): yield from iterable except Exception as ex: exception_result = cast( - "Union[Failure, MutableChain[_T]]", + "Failure | MutableChain[_T]", self._process_spider_exception( response, ex, exception_processor_index ), @@ -195,7 +195,7 @@ class SpiderMiddlewareManager(MiddlewareManager): yield r except Exception as ex: exception_result = cast( - "Union[Failure, MutableAsyncChain[_T]]", + "Failure | MutableAsyncChain[_T]", self._process_spider_exception( response, ex, exception_processor_index ), @@ -241,9 +241,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # simplified when downgrading is removed. if dfd.called: # the result is available immediately if _process_spider_output didn't do downgrading - return cast( - "Union[MutableChain[_T], MutableAsyncChain[_T]]", dfd.result - ) + return cast("MutableChain[_T] | MutableAsyncChain[_T]", dfd.result) # we forbid waiting here because otherwise we would need to return a deferred from # _process_spider_exception too, which complicates the architecture msg = f"Async iterable returned from {global_object_name(method)} cannot be downgraded" diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 983bbcfb9..a850b330e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -16,7 +16,7 @@ from collections.abc import Callable from datetime import datetime, timezone from pathlib import Path, PureWindowsPath from tempfile import NamedTemporaryFile -from typing import IO, TYPE_CHECKING, Any, Optional, Protocol, TypeVar, cast +from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, TypeVar, cast from urllib.parse import unquote, urlparse from twisted.internet.defer import Deferred, DeferredList, maybeDeferred @@ -50,7 +50,9 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -UriParamsCallableT = Callable[[dict[str, Any], Spider], Optional[dict[str, Any]]] +UriParamsCallableT: TypeAlias = Callable[ + [dict[str, Any], Spider], dict[str, Any] | None +] _StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol") diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index f704ae014..efda50a82 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -7,7 +7,7 @@ from email.utils import mktime_tz, parsedate_tz from importlib import import_module from pathlib import Path from time import time -from typing import IO, TYPE_CHECKING, Any, cast +from typing import IO, TYPE_CHECKING, Any, Concatenate, cast from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict @@ -23,9 +23,6 @@ if TYPE_CHECKING: from collections.abc import Callable from types import ModuleType - # typing.Concatenate requires Python 3.10 - from typing_extensions import Concatenate - from scrapy.http.request import Request from scrapy.settings import BaseSettings from scrapy.spiders import Spider diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 5498e1138..f3f60bffb 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, AnyStr, Union, cast +from typing import TYPE_CHECKING, Any, AnyStr, TypeAlias, cast from w3lib.http import headers_dict_to_raw @@ -15,7 +15,7 @@ if TYPE_CHECKING: from typing_extensions import Self -_RawValueT = Union[bytes, str, int] +_RawValue: TypeAlias = bytes | str | int # isn't fully compatible typing-wise with either dict or CaselessDict, @@ -44,9 +44,9 @@ class Headers(CaselessDict): """Normalize key to bytes""" return self._tobytes(key.title()) - def normvalue(self, value: _RawValueT | Iterable[_RawValueT]) -> list[bytes]: + def normvalue(self, value: _RawValue | Iterable[_RawValue]) -> list[bytes]: """Normalize values to bytes""" - _value: Iterable[_RawValueT] + _value: Iterable[_RawValue] if value is None: _value = [] elif isinstance(value, (str, bytes)): @@ -58,7 +58,7 @@ class Headers(CaselessDict): return [self._tobytes(x) for x in _value] - def _tobytes(self, x: _RawValueT) -> bytes: + def _tobytes(self, x: _RawValue) -> bytes: if isinstance(x, bytes): return x if isinstance(x, str): @@ -87,15 +87,15 @@ class Headers(CaselessDict): return self.normvalue(def_val) return [] - def setlist(self, key: AnyStr, list_: Iterable[_RawValueT]) -> None: + def setlist(self, key: AnyStr, list_: Iterable[_RawValue]) -> None: self[key] = list_ def setlistdefault( - self, key: AnyStr, default_list: Iterable[_RawValueT] = () + self, key: AnyStr, default_list: Iterable[_RawValue] = () ) -> Any: return self.setdefault(key, default_list) - def appendlist(self, key: AnyStr, value: Iterable[_RawValueT]) -> None: + def appendlist(self, key: AnyStr, value: Iterable[_RawValue]) -> None: lst = self.getlist(key) lst.extend(self.normvalue(value)) self[key] = lst diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 2b8d0ab84..f0d20843f 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -12,10 +12,11 @@ from typing import ( TYPE_CHECKING, Any, AnyStr, + Concatenate, NoReturn, + TypeAlias, TypedDict, TypeVar, - Union, overload, ) @@ -33,13 +34,13 @@ if TYPE_CHECKING: from twisted.python.failure import Failure - # typing.Concatenate requires Python 3.10 # typing.NotRequired and typing.Self require Python 3.11 - from typing_extensions import Concatenate, NotRequired, Self + from typing_extensions import NotRequired, Self + # circular import from scrapy.http import Response - CallbackT = Callable[Concatenate[Response, ...], Any] + CallbackT: TypeAlias = Callable[Concatenate[Response, ...], Any] class VerboseCookie(TypedDict): @@ -50,7 +51,7 @@ class VerboseCookie(TypedDict): secure: NotRequired[bool] -CookiesT = Union[dict[str, str], list[VerboseCookie]] +CookiesT: TypeAlias = dict[str, str] | list[VerboseCookie] RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index aa15a0222..7b2e480c4 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations from collections.abc import Iterable -from typing import TYPE_CHECKING, Any, Optional, Union, cast +from typing import TYPE_CHECKING, Any, TypeAlias, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from parsel.csstranslator import HTMLTranslator @@ -31,9 +31,9 @@ if TYPE_CHECKING: from scrapy.http.response.text import TextResponse -FormdataVType = Union[str, Iterable[str]] -FormdataKVType = tuple[str, FormdataVType] -FormdataType = Optional[Union[dict[str, FormdataVType], list[FormdataKVType]]] +FormdataVType: TypeAlias = str | Iterable[str] +FormdataKVType: TypeAlias = tuple[str, FormdataVType] +FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None class FormRequest(Request): diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 7c9b4d3e3..0ea78e35a 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -9,7 +9,7 @@ import operator import re from collections.abc import Callable, Iterable from functools import partial -from typing import TYPE_CHECKING, Any, Union, cast +from typing import TYPE_CHECKING, Any, TypeAlias, cast from urllib.parse import urljoin, urlparse from lxml import etree @@ -157,8 +157,8 @@ class LxmlParserLinkExtractor: return links -_RegexT = Union[str, re.Pattern[str]] -_RegexOrSeveralT = Union[_RegexT, Iterable[_RegexT]] +_Regex: TypeAlias = str | re.Pattern[str] +_RegexOrSeveral: TypeAlias = _Regex | Iterable[_Regex] class LxmlLinkExtractor: @@ -166,8 +166,8 @@ class LxmlLinkExtractor: def __init__( self, - allow: _RegexOrSeveralT = (), - deny: _RegexOrSeveralT = (), + allow: _RegexOrSeveral = (), + deny: _RegexOrSeveral = (), allow_domains: str | Iterable[str] = (), deny_domains: str | Iterable[str] = (), restrict_xpaths: str | Iterable[str] = (), @@ -179,7 +179,7 @@ class LxmlLinkExtractor: deny_extensions: str | Iterable[str] | None = None, restrict_css: str | Iterable[str] = (), strip: bool = True, - restrict_text: _RegexOrSeveralT | None = None, + restrict_text: _RegexOrSeveral | None = None, ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) self.link_extractor = LxmlParserLinkExtractor( @@ -208,7 +208,7 @@ class LxmlLinkExtractor: self.restrict_text: list[re.Pattern[str]] = self._compile_regexes(restrict_text) @staticmethod - def _compile_regexes(value: _RegexOrSeveralT | None) -> list[re.Pattern[str]]: + def _compile_regexes(value: _RegexOrSeveral | None) -> list[re.Pattern[str]]: return [ x if isinstance(x, re.Pattern) else re.compile(x) for x in arg_to_iter(value) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index d441be3ba..32fae7cc0 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -5,7 +5,7 @@ import pprint import warnings from abc import ABC, abstractmethod from collections import defaultdict, deque -from typing import TYPE_CHECKING, Any, TypeVar, cast +from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, cast from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.utils.defer import ensure_awaitable @@ -18,20 +18,18 @@ if TYPE_CHECKING: from twisted.internet.defer import Deferred - # typing.Concatenate and typing.ParamSpec require Python 3.10 # typing.Self requires Python 3.11 - from typing_extensions import Concatenate, ParamSpec, Self + from typing_extensions import Self from scrapy import Spider from scrapy.crawler import Crawler from scrapy.settings import BaseSettings, Settings - _P = ParamSpec("_P") - logger = logging.getLogger(__name__) _T = TypeVar("_T") +_P = ParamSpec("_P") class MiddlewareManager(ABC): diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 6d2494719..398e895b8 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -7,7 +7,7 @@ See documentation in docs/item-pipeline.rst from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any, Callable, cast +from typing import TYPE_CHECKING, Any, cast from twisted.internet.defer import Deferred, DeferredList @@ -22,7 +22,7 @@ from scrapy.utils.defer import ( from scrapy.utils.python import global_object_name if TYPE_CHECKING: - from collections.abc import Iterable + from collections.abc import Callable, Iterable from twisted.python.failure import Failure diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 124a7f706..5ac211dfb 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -69,7 +69,7 @@ class ImagesPipeline(FilesPipeline): self._ImageOps = ImageOps except ImportError: raise NotConfigured( - "ImagesPipeline requires installing Pillow 8.0.0 or later" + "ImagesPipeline requires installing Pillow 8.3.2 or later" ) super().__init__( diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index a7cd51166..f8fc3687e 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -5,7 +5,7 @@ import logging import warnings from abc import ABC, abstractmethod from collections import defaultdict -from typing import TYPE_CHECKING, Any, Literal, TypedDict, Union, cast +from typing import TYPE_CHECKING, Any, Literal, TypeAlias, TypedDict, cast from twisted import version as twisted_version from twisted.internet.defer import ( @@ -47,7 +47,9 @@ class FileInfo(TypedDict): status: str -FileInfoOrError = Union[tuple[Literal[True], FileInfo], tuple[Literal[False], Failure]] +FileInfoOrError: TypeAlias = ( + tuple[Literal[True], FileInfo] | tuple[Literal[False], Failure] +) logger = logging.getLogger(__name__) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 625231fe6..c330c4c35 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -6,7 +6,7 @@ import warnings from collections.abc import Iterable, Iterator, Mapping, MutableMapping from importlib import import_module from pprint import pformat -from typing import TYPE_CHECKING, Any, Union, cast +from typing import TYPE_CHECKING, Any, TypeAlias, cast from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import default_settings @@ -14,7 +14,7 @@ from scrapy.utils.misc import load_object # The key types are restricted in BaseSettings._get_key() to ones supported by JSON, # see https://github.com/scrapy/scrapy/issues/5383. -_SettingsKeyT = Union[bool, float, int, str, None] +_SettingsKey: TypeAlias = bool | float | int | str | None if TYPE_CHECKING: from types import ModuleType @@ -25,7 +25,7 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self - _SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None] + _SettingsInput: TypeAlias = SupportsItems[_SettingsKey, Any] | str | None SETTINGS_PRIORITIES: dict[str, int] = { @@ -76,7 +76,7 @@ class SettingsAttribute: return f"" -class BaseSettings(MutableMapping[_SettingsKeyT, Any]): +class BaseSettings(MutableMapping[_SettingsKey, Any]): """ Instances of this class behave like dictionaries, but store priorities along with their ``(key, value)`` pairs, and can be frozen (i.e. marked @@ -100,13 +100,13 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): __default = object() - def __init__(self, values: _SettingsInputT = None, priority: int | str = "project"): + def __init__(self, values: _SettingsInput = None, priority: int | str = "project"): self.frozen: bool = False - self.attributes: dict[_SettingsKeyT, SettingsAttribute] = {} + self.attributes: dict[_SettingsKey, SettingsAttribute] = {} if values: self.update(values, priority) - def __getitem__(self, opt_name: _SettingsKeyT) -> Any: + def __getitem__(self, opt_name: _SettingsKey) -> Any: if opt_name not in self: return None return self.attributes[opt_name].value @@ -114,7 +114,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def __contains__(self, name: Any) -> bool: return name in self.attributes - def add_to_list(self, name: _SettingsKeyT, item: Any) -> None: + def add_to_list(self, name: _SettingsKey, item: Any) -> None: """Append *item* to the :class:`list` setting with the specified *name* if *item* is not already in that list. @@ -125,7 +125,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): if item not in value: self.set(name, [*value, item], self.getpriority(name) or 0) - def remove_from_list(self, name: _SettingsKeyT, item: Any) -> None: + def remove_from_list(self, name: _SettingsKey, item: Any) -> None: """Remove *item* from the :class:`list` setting with the specified *name*. @@ -139,7 +139,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).") self.set(name, [v for v in value if v != item], self.getpriority(name) or 0) - def get(self, name: _SettingsKeyT, default: Any = None) -> Any: + def get(self, name: _SettingsKey, default: Any = None) -> Any: """ Get a setting value without affecting its original type. @@ -160,7 +160,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return self[name] if self[name] is not None else default - def getbool(self, name: _SettingsKeyT, default: bool = False) -> bool: + def getbool(self, name: _SettingsKey, default: bool = False) -> bool: """ Get a setting value as a boolean. @@ -190,7 +190,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): "'True'/'False' and 'true'/'false'" ) - def getint(self, name: _SettingsKeyT, default: int = 0) -> int: + def getint(self, name: _SettingsKey, default: int = 0) -> int: """ Get a setting value as an int. @@ -202,7 +202,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ return int(self.get(name, default)) - def getfloat(self, name: _SettingsKeyT, default: float = 0.0) -> float: + def getfloat(self, name: _SettingsKey, default: float = 0.0) -> float: """ Get a setting value as a float. @@ -215,7 +215,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return float(self.get(name, default)) def getlist( - self, name: _SettingsKeyT, default: list[Any] | None = None + self, name: _SettingsKey, default: list[Any] | None = None ) -> list[Any]: """ Get a setting value as a list. If the setting original type is a list, @@ -239,7 +239,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return list(value) def getdict( - self, name: _SettingsKeyT, default: dict[Any, Any] | None = None + self, name: _SettingsKey, default: dict[Any, Any] | None = None ) -> dict[Any, Any]: """ Get a setting value as a dictionary. If the setting original type is a @@ -263,7 +263,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def getdictorlist( self, - name: _SettingsKeyT, + name: _SettingsKey, default: dict[Any, Any] | list[Any] | tuple[Any] | None = None, ) -> dict[Any, Any] | list[Any]: """Get a setting value as either a :class:`dict` or a :class:`list`. @@ -310,7 +310,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): ) return copy.deepcopy(value) - def getwithbase(self, name: _SettingsKeyT) -> BaseSettings: + def getwithbase(self, name: _SettingsKey) -> BaseSettings: """Get a composition of a dictionary-like setting and its `_BASE` counterpart. @@ -324,7 +324,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): compbs.update(self[name]) return compbs - def getpriority(self, name: _SettingsKeyT) -> int | None: + def getpriority(self, name: _SettingsKey) -> int | None: """ Return the current numerical priority value of a setting, or ``None`` if the given ``name`` does not exist. @@ -349,7 +349,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def replace_in_component_priority_dict( self, - name: _SettingsKeyT, + name: _SettingsKey, old_cls: type, new_cls: type, priority: int | None = None, @@ -388,11 +388,11 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): ) self.set(name, component_priority_dict, priority=self.getpriority(name) or 0) - def __setitem__(self, name: _SettingsKeyT, value: Any) -> None: + def __setitem__(self, name: _SettingsKey, value: Any) -> None: self.set(name, value) def set( - self, name: _SettingsKeyT, value: Any, priority: int | str = "project" + self, name: _SettingsKey, value: Any, priority: int | str = "project" ) -> None: """ Store a key/value attribute with a given priority. @@ -422,7 +422,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): self.attributes[name].set(value, priority) def set_in_component_priority_dict( - self, name: _SettingsKeyT, cls: type, priority: int | None + self, name: _SettingsKey, cls: type, priority: int | None ) -> None: """Set the *cls* component in the *name* :ref:`component priority dictionary ` setting with *priority*. @@ -447,7 +447,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): def setdefault( self, - name: _SettingsKeyT, + name: _SettingsKey, default: Any = None, priority: int | str = "project", ) -> Any: @@ -458,7 +458,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): return self.attributes[name].value def setdefault_in_component_priority_dict( - self, name: _SettingsKeyT, cls: type, priority: int | None + self, name: _SettingsKey, cls: type, priority: int | None ) -> None: """Set the *cls* component in the *name* :ref:`component priority dictionary ` setting with *priority* @@ -475,7 +475,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): component_priority_dict[cls] = priority self.set(name, component_priority_dict, self.getpriority(name) or 0) - def setdict(self, values: _SettingsInputT, priority: int | str = "project") -> None: + def setdict(self, values: _SettingsInput, priority: int | str = "project") -> None: self.update(values, priority) def setmodule( @@ -503,7 +503,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): self.set(key, getattr(module, key), priority) # BaseSettings.update() doesn't support all inputs that MutableMapping.update() supports - def update(self, values: _SettingsInputT, priority: int | str = "project") -> None: # type: ignore[override] + def update(self, values: _SettingsInput, priority: int | str = "project") -> None: # type: ignore[override] """ Store key/value pairs with a given priority. @@ -527,7 +527,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): """ self._assert_mutability() if isinstance(values, str): - values = cast("dict[_SettingsKeyT, Any]", json.loads(values)) + values = cast("dict[_SettingsKey, Any]", json.loads(values)) if values is not None: if isinstance(values, BaseSettings): for name, value in values.items(): @@ -536,7 +536,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): for name, value in values.items(): self.set(name, value, priority) - def delete(self, name: _SettingsKeyT, priority: int | str = "project") -> None: + def delete(self, name: _SettingsKey, priority: int | str = "project") -> None: if name not in self: raise KeyError(name) self._assert_mutability() @@ -544,7 +544,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): if priority >= cast("int", self.getpriority(name)): del self.attributes[name] - def __delitem__(self, name: _SettingsKeyT) -> None: + def __delitem__(self, name: _SettingsKey) -> None: self._assert_mutability() del self.attributes[name] @@ -584,26 +584,26 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): copy.freeze() return copy - def __iter__(self) -> Iterator[_SettingsKeyT]: + def __iter__(self) -> Iterator[_SettingsKey]: return iter(self.attributes) def __len__(self) -> int: return len(self.attributes) - def _to_dict(self) -> dict[_SettingsKeyT, Any]: + def _to_dict(self) -> dict[_SettingsKey, Any]: return { self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) for k, v in self.items() } - def _get_key(self, key_value: Any) -> _SettingsKeyT: + def _get_key(self, key_value: Any) -> _SettingsKey: return ( key_value if isinstance(key_value, (bool, float, int, str, type(None))) else str(key_value) ) - def copy_to_dict(self) -> dict[_SettingsKeyT, Any]: + def copy_to_dict(self) -> dict[_SettingsKey, Any]: """ Make a copy of current settings and convert to a dict. @@ -626,7 +626,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): else: p.text(pformat(self.copy_to_dict())) - def pop(self, name: _SettingsKeyT, default: Any = __default) -> Any: + def pop(self, name: _SettingsKey, default: Any = __default) -> Any: try: value = self.attributes[name].value except KeyError: @@ -648,7 +648,7 @@ class Settings(BaseSettings): described on :ref:`topics-settings-ref` already populated. """ - def __init__(self, values: _SettingsInputT = None, priority: int | str = "project"): + def __init__(self, values: _SettingsInput = None, priority: int | str = "project"): # Do not pass kwarg values here. We don't want to promote user-defined # dicts, and we want to update, not replace, default dicts with the # values given by the user @@ -670,7 +670,7 @@ def iter_default_settings() -> Iterable[tuple[str, Any]]: def overridden_settings( - settings: Mapping[_SettingsKeyT, Any], + settings: Mapping[_SettingsKey, Any], ) -> Iterable[tuple[str, Any]]: """Return an iterable of the settings that have been overridden""" for name, defvalue in iter_default_settings(): diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 8db00a1ca..018e510c5 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -26,7 +26,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.http.request import CallbackT - from scrapy.settings import BaseSettings, _SettingsKeyT + from scrapy.settings import BaseSettings, _SettingsKey from scrapy.utils.log import SpiderLoggerAdapter @@ -39,7 +39,7 @@ class Spider(object_ref): """ name: str - custom_settings: dict[_SettingsKeyT, Any] | None = None + custom_settings: dict[_SettingsKey, Any] | None = None #: Start URLs. See :meth:`start`. start_urls: list[str] diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 0cbd6b3cc..89421cbe4 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -10,7 +10,7 @@ from __future__ import annotations import copy import warnings from collections.abc import AsyncIterator, Awaitable, Callable -from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast +from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast from scrapy.http import HtmlResponse, Request, Response from scrapy.link import Link @@ -34,8 +34,8 @@ if TYPE_CHECKING: _T = TypeVar("_T") -ProcessLinksT = Callable[[list[Link]], list[Link]] -ProcessRequestT = Callable[[Request, Response], Optional[Request]] +ProcessLinksT: TypeAlias = Callable[[list[Link]], list[Link]] +ProcessRequestT: TypeAlias = Callable[[Request, Response], Request | None] def _identity(x: _T) -> _T: diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 9ea58c729..15e7736ec 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -6,7 +6,7 @@ import asyncio import logging import time from collections.abc import AsyncIterator, Callable, Coroutine, Iterable -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar from twisted.internet.defer import Deferred from twisted.internet.task import LoopingCall @@ -17,15 +17,14 @@ from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_instal if TYPE_CHECKING: from twisted.internet.base import DelayedCall - # typing.Concatenate and typing.ParamSpec require Python 3.10 # typing.Self, typing.TypeVarTuple and typing.Unpack require Python 3.11 - from typing_extensions import Concatenate, ParamSpec, Self, TypeVarTuple, Unpack + from typing_extensions import Self, TypeVarTuple, Unpack - _P = ParamSpec("_P") _Ts = TypeVarTuple("_Ts") _T = TypeVar("_T") +_P = ParamSpec("_P") logger = logging.getLogger(__name__) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 5a627fc83..5869cf52e 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -6,7 +6,7 @@ import sys from configparser import ConfigParser from operator import itemgetter from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, cast +from typing import TYPE_CHECKING, Any, cast from scrapy.exceptions import UsageError from scrapy.settings import BaseSettings @@ -14,7 +14,7 @@ from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values if TYPE_CHECKING: - from collections.abc import Collection, Iterable, Mapping, MutableMapping + from collections.abc import Callable, Collection, Iterable, Mapping, MutableMapping def build_component_list( diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 374c8ecb8..2dde0aa55 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -3,7 +3,7 @@ from __future__ import annotations import inspect import warnings from functools import wraps -from typing import TYPE_CHECKING, Any, TypeVar, overload +from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, overload from twisted.internet.defer import Deferred, maybeDeferred from twisted.internet.threads import deferToThread @@ -13,13 +13,9 @@ from scrapy.exceptions import ScrapyDeprecationWarning if TYPE_CHECKING: from collections.abc import AsyncGenerator, Callable, Coroutine - # typing.ParamSpec requires Python 3.10 - from typing_extensions import ParamSpec - - _P = ParamSpec("_P") - _T = TypeVar("_T") +_P = ParamSpec("_P") def deprecated( diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 7481e26ed..39cd6e540 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -10,7 +10,16 @@ import warnings from asyncio import Future from collections.abc import Awaitable, Coroutine, Iterable, Iterator from functools import wraps -from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast, overload +from typing import ( + TYPE_CHECKING, + Any, + Concatenate, + Generic, + ParamSpec, + TypeVar, + cast, + overload, +) from twisted.internet.defer import Deferred, DeferredList, fail, succeed from twisted.internet.task import Cooperator @@ -24,14 +33,10 @@ if TYPE_CHECKING: from twisted.python.failure import Failure - # typing.Concatenate and typing.ParamSpec require Python 3.10 - from typing_extensions import Concatenate, ParamSpec - - _P = ParamSpec("_P") - _T = TypeVar("_T") _T2 = TypeVar("_T2") +_P = ParamSpec("_P") _DEFER_DELAY = 0.1 @@ -327,7 +332,7 @@ def process_chain_both( stacklevel=2, ) d: Deferred = Deferred() - for cb, eb in zip(callbacks, errbacks): + for cb, eb in zip(callbacks, errbacks, strict=False): d.addCallback(cb, *a, **kw) d.addErrback(eb, *a, **kw) if isinstance(input, failure.Failure): diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c70a0d327..ad9640e47 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -212,7 +212,7 @@ def csviter( }, ) continue - yield dict(zip(headers, row)) + yield dict(zip(headers, row, strict=False)) @overload diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index e45d34ac5..6d3bb1db9 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -5,14 +5,14 @@ import pprint import sys from collections.abc import MutableMapping from logging.config import dictConfig -from typing import TYPE_CHECKING, Any, Optional, cast +from typing import TYPE_CHECKING, Any, cast from twisted.internet import asyncioreactor from twisted.python import log as twisted_log from twisted.python.failure import Failure import scrapy -from scrapy.settings import Settings, _SettingsKeyT +from scrapy.settings import Settings, _SettingsKey from scrapy.utils.versions import get_versions if TYPE_CHECKING: @@ -35,7 +35,7 @@ def failure_to_exc_info( return ( failure.type, failure.value, - cast("Optional[TracebackType]", failure.getTracebackObject()), + cast("TracebackType | None", failure.getTracebackObject()), ) return None @@ -83,7 +83,7 @@ DEFAULT_LOGGING = { def configure_logging( - settings: Settings | dict[_SettingsKeyT, Any] | None = None, + settings: Settings | dict[_SettingsKey, Any] | None = None, install_root_handler: bool = True, ) -> None: """ diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index ad758b783..4eda29e25 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -3,12 +3,12 @@ from __future__ import annotations import signal from collections.abc import Callable from types import FrameType -from typing import Any, Optional, Union +from typing import Any, TypeAlias # copy of _HANDLER from typeshed/stdlib/signal.pyi -SignalHandlerT = Union[ - Callable[[int, Optional[FrameType]], Any], int, signal.Handlers, None -] +SignalHandlerT: TypeAlias = ( + Callable[[int, FrameType | None], Any] | int | signal.Handlers | None +) signal_names: dict[int, str] = {} for signame in dir(signal): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 69ab97298..d9a5007cc 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -13,7 +13,7 @@ import weakref from collections.abc import AsyncIterator, Iterable, Mapping from functools import partial, wraps from itertools import chain -from typing import TYPE_CHECKING, Any, TypeVar, overload +from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator @@ -22,15 +22,14 @@ if TYPE_CHECKING: from collections.abc import Callable, Iterator from re import Pattern - # typing.Concatenate and typing.ParamSpec require Python 3.10 # typing.Self requires Python 3.11 - from typing_extensions import Concatenate, ParamSpec, Self + from typing_extensions import Self - _P = ParamSpec("_P") _T = TypeVar("_T") _KT = TypeVar("_KT") _VT = TypeVar("_VT") +_P = ParamSpec("_P") def flatten(x: Iterable[Any]) -> list[Any]: @@ -286,7 +285,7 @@ def get_spec(func: Callable[..., Any]) -> tuple[list[str], dict[str, Any]]: firstdefault = len(spec.args) - len(defaults) args = spec.args[:firstdefault] - kwargs = dict(zip(spec.args[firstdefault:], defaults)) + kwargs = dict(zip(spec.args[firstdefault:], defaults, strict=False)) return args, kwargs diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 132f88c74..f37c48a7d 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -3,7 +3,7 @@ from __future__ import annotations import asyncio import sys from contextlib import suppress -from typing import TYPE_CHECKING, Any, Generic, TypeVar +from typing import TYPE_CHECKING, Any, Generic, ParamSpec, TypeVar from warnings import catch_warnings, filterwarnings from twisted.internet import asyncioreactor, error @@ -19,14 +19,11 @@ if TYPE_CHECKING: from twisted.internet.protocol import ServerFactory from twisted.internet.tcp import Port - # typing.ParamSpec requires Python 3.10 - from typing_extensions import ParamSpec - from scrapy.utils.asyncio import CallLaterResult - _P = ParamSpec("_P") _T = TypeVar("_T") +_P = ParamSpec("_P") def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # pylint: disable=inconsistent-return-statements # noqa: RET503 diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 1348cc992..8cfa5cce4 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -8,7 +8,7 @@ from __future__ import annotations import re import warnings from importlib import import_module -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, TypeAlias from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse from warnings import warn @@ -37,7 +37,7 @@ if TYPE_CHECKING: from scrapy import Spider -UrlT = Union[str, bytes, ParseResult] +UrlT: TypeAlias = str | bytes | ParseResult def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 47bed925a..ecd784c4c 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -369,7 +369,7 @@ with multiples lines est = [x for sublist in est for x in sublist] # flatten est = [x.lstrip().rstrip() for x in est] it = iter(est) - s = dict(zip(it, it)) + s = dict(zip(it, it, strict=False)) assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == "1" diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index b86f3c7f2..8d6c22ad7 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,13 +1,17 @@ from __future__ import annotations import datetime -from typing import Any, Callable +from typing import TYPE_CHECKING, Any from scrapy.extensions.periodic_log import PeriodicLog from scrapy.utils.test import get_crawler from .spiders import MetaSpider +if TYPE_CHECKING: + from collections.abc import Callable + + stats_dump_1 = { "log_count/INFO": 10, "log_count/WARNING": 1, diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index aa4477210..7eb9f78e8 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2628,7 +2628,7 @@ class TestBatchDeliveries(TestFeedExportBase): } data = await self.exported_data(items, settings) for fmt, expected in formats.items(): - for expected_batch, got_batch in zip(expected, data[fmt]): + for expected_batch, got_batch in zip(expected, data[fmt], strict=False): assert got_batch == expected_batch @deferred_f_from_coro_f @@ -2652,7 +2652,7 @@ class TestBatchDeliveries(TestFeedExportBase): } data = await self.exported_data(items, settings) for fmt, expected in formats.items(): - for expected_batch, got_batch in zip(expected, data[fmt]): + for expected_batch, got_batch in zip(expected, data[fmt], strict=False): assert got_batch == expected_batch @deferred_f_from_coro_f diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index d100d6b71..dcc067ec9 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -6,7 +6,7 @@ import re import string from ipaddress import IPv4Address from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, cast +from typing import TYPE_CHECKING, Any, cast from unittest import mock from urllib.parse import urlencode @@ -39,7 +39,7 @@ from tests.mockserver.http_resources import LeafResource, Status from tests.mockserver.utils import ssl_context_factory if TYPE_CHECKING: - from collections.abc import AsyncGenerator, Coroutine, Generator + from collections.abc import AsyncGenerator, Callable, Coroutine, Generator from scrapy.core.http2.protocol import H2ClientProtocol diff --git a/tests/test_http_response.py b/tests/test_http_response.py index fdef5adea..ac321c1ad 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -333,7 +333,7 @@ class TestResponseBase: if response is None: response = self._links_response() followed = response.follow_all(follow_obj) - for req, target in zip(followed, target_urls): + for req, target in zip(followed, target_urls, strict=False): assert req.url == target yield req @@ -647,7 +647,7 @@ class TestTextResponse(TestResponseBase): # select elements for sellist in [resp.css("a"), resp.xpath("//a")]: - for sel, url in zip(sellist, urls): + for sel, url in zip(sellist, urls, strict=False): self._assert_followed_url(sel, url, response=resp) # select elements @@ -659,7 +659,7 @@ class TestTextResponse(TestResponseBase): # href attributes should work for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]: - for sel, url in zip(sellist, urls): + for sel, url in zip(sellist, urls, strict=False): self._assert_followed_url(sel, url, response=resp) # non-a elements are not supported diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing index 3926c830f..95f03e17c 100644 --- a/tests_typing/test_http_request.mypy-testing +++ b/tests_typing/test_http_request.mypy-testing @@ -16,7 +16,7 @@ class MyRequest2(Request): @pytest.mark.mypy_testing def mypy_test_headers(): - Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[tuple[str, Any]], None]" + Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" Request("data:,", headers=None) Request("data:,", headers={}) Request("data:,", headers=[]) diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing index 88aedbd3e..630754c1e 100644 --- a/tests_typing/test_http_response.mypy-testing +++ b/tests_typing/test_http_response.mypy-testing @@ -7,7 +7,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse @pytest.mark.mypy_testing def mypy_test_headers(): - Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[tuple[str, Any]], None]" + Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" Response("data:,", headers=None) Response("data:,", headers={}) Response("data:,", headers=[]) diff --git a/tox.ini b/tox.ini index 5f49e6a5a..eb406c89a 100644 --- a/tox.ini +++ b/tox.ini @@ -43,7 +43,7 @@ commands = pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 scrapy tests --doctest-modules} [testenv:typing] -basepython = python3.9 +basepython = python3.10 deps = mypy==1.16.1 typing-extensions==4.14.1 @@ -64,7 +64,7 @@ commands = mypy {posargs:scrapy tests} [testenv:typing-tests] -basepython = python3.9 +basepython = python3.10 deps = {[test-requirements]deps} {[testenv:typing]deps} @@ -97,7 +97,7 @@ commands = twine check dist/* [pinned] -basepython = python3.9 +basepython = python3.10 deps = # pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0 pytest==8.4.0 @@ -106,7 +106,7 @@ deps = cryptography==37.0.0 cssselect==0.9.1 itemadapter==0.1.0 - lxml==4.6.0 + lxml==4.6.4 parsel==1.5.0 pyOpenSSL==22.0.0 queuelib==1.4.2 @@ -151,17 +151,17 @@ deps = basepython = {[pinned]basepython} deps = {[pinned]deps} - Pillow==8.0.0 + Pillow==8.3.2 boto3==1.20.0 bpython==0.7.1 brotli==0.5.2; implementation_name != "pypy" brotlicffi==0.8.0; implementation_name == "pypy" brotlipy google-cloud-storage==1.29.0 - ipython==2.0.0 + ipython==7.1.0 robotexclusionrulesparser==1.6.2 - uvloop==0.14.0; platform_system != "Windows" and implementation_name != "pypy" - zstandard==0.1; implementation_name != "pypy" + uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy" + zstandard==0.16.0; implementation_name != "pypy" setenv = {[pinned]setenv} commands = {[pinned]commands} @@ -254,7 +254,7 @@ commands = [testenv:botocore] deps = {[testenv]deps} - botocore>=1.4.87 + botocore>=1.13.45 commands = pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy -m requires_botocore} @@ -262,7 +262,7 @@ commands = basepython = {[pinned]basepython} deps = {[pinned]deps} - botocore==1.4.87 + botocore==1.13.45 setenv = {[pinned]setenv} commands =