Remove Python 3.9 support (#7121)

.github/workflows/checks.yml

@@ -20,10 +20,10 @@ jobs:
         - python-version: "3.13"
           env:
             TOXENV: pylint
-        - python-version: "3.9"
+        - python-version: "3.10"
           env:
             TOXENV: typing
-        - python-version: "3.9"
+        - python-version: "3.10"
           env:
             TOXENV: typing-tests
         - python-version: "3.13"  # Keep in sync with .readthedocs.yml

.github/workflows/tests-macos.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
 
     steps:
     - uses: actions/checkout@v4

.github/workflows/tests-ubuntu.yml

@@ -17,9 +17,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: "3.9"
-          env:
-            TOXENV: py
         - python-version: "3.10"
           env:
             TOXENV: py
@@ -40,19 +37,19 @@ jobs:
             TOXENV: pypy3
 
         # pinned deps
-        - python-version: "3.9.21"
+        - python-version: "3.10.19"
           env:
             TOXENV: pinned
-        - python-version: "3.9.21"
+        - python-version: "3.10.19"
           env:
             TOXENV: default-reactor-pinned
         - python-version: pypy3.11
           env:
             TOXENV: pypy3-pinned
-        - python-version: "3.9.21"
+        - python-version: "3.10.19"
           env:
             TOXENV: extra-deps-pinned
-        - python-version: "3.9.21"
+        - python-version: "3.10.19"
           env:
             TOXENV: botocore-pinned
 

.github/workflows/tests-windows.yml

@@ -17,9 +17,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: "3.9"
-          env:
-            TOXENV: py
         - python-version: "3.10"
           env:
             TOXENV: py
@@ -37,10 +34,10 @@ jobs:
             TOXENV: default-reactor
 
         # pinned deps
-        - python-version: "3.9.13"
+        - python-version: "3.10.11"
           env:
             TOXENV: pinned
-        - python-version: "3.9.13"
+        - python-version: "3.10.11"
           env:
             TOXENV: extra-deps-pinned
 

README.rst

@@ -40,7 +40,7 @@
    :alt: Ask DeepWiki
 
 Scrapy_ is a web scraping framework to extract structured data from websites.
-It is cross-platform, and requires Python 3.9+. It is maintained by Zyte_
+It is cross-platform, and requires Python 3.10+. It is maintained by Zyte_
 (formerly Scrapinghub) and `many other contributors`_.
 
 .. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors

docs/intro/install.rst

@@ -9,7 +9,7 @@ Installation guide
 Supported Python versions
 =========================
 
-Scrapy requires Python 3.9+, either the CPython implementation (default) or
+Scrapy requires Python 3.10+, either the CPython implementation (default) or
 the PyPy implementation (see :ref:`python:implementations`).
 
 .. _intro-install-scrapy:

docs/topics/media-pipeline.rst

@@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
 can configure some extra functions like generating thumbnails and filtering
 the images based on their size.
 
-The Images Pipeline requires Pillow_ 8.0.0 or greater. It is used for
+The Images Pipeline requires Pillow_ 8.3.2 or greater. It is used for
 thumbnailing and normalizing images to JPEG/RGB format.
 
 .. _Pillow: https://github.com/python-pillow/Pillow
@@ -238,7 +238,7 @@ Amazon S3 storage
 .. setting:: FILES_STORE_S3_ACL
 .. setting:: IMAGES_STORE_S3_ACL
 
-If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and
+If botocore_ >= 1.13.45 is installed, :setting:`FILES_STORE` and
 :setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will
 automatically upload the files to the bucket.
 

pyproject.toml

@@ -13,7 +13,7 @@ dependencies = [
     "defusedxml>=0.7.1",
     "itemadapter>=0.1.0",
     "itemloaders>=1.0.1",
-    "lxml>=4.6.0",
+    "lxml>=4.6.4",
     "packaging",
     "parsel>=1.5.0",
     "protego>=0.1.15",
@@ -35,7 +35,6 @@ classifiers = [
     "Operating System :: OS Independent",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
@@ -49,7 +48,7 @@ classifiers = [
 license = "BSD-3-Clause"
 license-files = ["LICENSE", "AUTHORS"]
 readme = "README.rst"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 authors = [{ name = "Scrapy developers", email = "pablo@pablohoffman.com" }]
 maintainers = [{ name = "Pablo Hoffman", email = "pablo@pablohoffman.com" }]
 
@@ -375,6 +374,8 @@ ignore = [
     "SIM115",
     # Yoda condition detected
     "SIM300",
+    # removed in recent ruff
+    "UP038",
 
     # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")
 

scrapy/cmdline.py

@@ -6,7 +6,7 @@ import inspect
 import os
 import sys
 from importlib.metadata import entry_points
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, ParamSpec
 
 import scrapy
 from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
@@ -20,9 +20,6 @@ from scrapy.utils.reactor import _asyncio_reactor_path
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterable
 
-    # typing.ParamSpec requires Python 3.10
-    from typing_extensions import ParamSpec
-
     from scrapy.settings import BaseSettings, Settings
 
 _P = ParamSpec("_P")
@@ -67,11 +64,7 @@ def _get_commands_from_entry_points(
     inproject: bool, group: str = "scrapy.commands"
 ) -> dict[str, ScrapyCommand]:
     cmds: dict[str, ScrapyCommand] = {}
-    if sys.version_info >= (3, 10):
+    for entry_point in entry_points(group=group):
-        eps = entry_points(group=group)
-    else:
-        eps = entry_points().get(group, ())
-    for entry_point in eps:
         obj = entry_point.load()
         if inspect.isclass(obj):
             cmds[entry_point.name] = obj()

scrapy/contracts/default.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import json
-from typing import Any, Callable
+from typing import TYPE_CHECKING, Any
 
 from itemadapter import ItemAdapter, is_item
 
@@ -9,6 +9,9 @@ from scrapy.contracts import Contract
 from scrapy.exceptions import ContractFail
 from scrapy.http import Request
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
 
 # contracts
 class UrlContract(Contract):

scrapy/core/scraper.py

@@ -7,7 +7,7 @@ import logging
 import warnings
 from collections import deque
 from collections.abc import AsyncIterator
-from typing import TYPE_CHECKING, Any, TypeVar, Union
+from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar
 
 from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.python.failure import Failure
@@ -53,7 +53,7 @@ logger = logging.getLogger(__name__)
 
 
 _T = TypeVar("_T")
-QueueTuple = tuple[Union[Response, Failure], Request, Deferred[None]]
+QueueTuple: TypeAlias = tuple[Response | Failure, Request, Deferred[None]]
 
 
 class Slot:

scrapy/core/spidermw.py

@@ -11,7 +11,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
 from functools import wraps
 from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice
-from typing import TYPE_CHECKING, Any, TypeVar, Union, cast
+from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast
 from warnings import warn
 
 from twisted.internet.defer import Deferred, inlineCallbacks
@@ -41,9 +41,9 @@ logger = logging.getLogger(__name__)
 
 
 _T = TypeVar("_T")
-ScrapeFunc = Callable[
+ScrapeFunc: TypeAlias = Callable[
-    [Union[Response, Failure], Request],
+    [Response | Failure, Request],
-    Coroutine[Any, Any, Union[Iterable[_T], AsyncIterator[_T]]],
+    Coroutine[Any, Any, Iterable[_T] | AsyncIterator[_T]],
 ]
 
 
@@ -179,7 +179,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 yield from iterable
             except Exception as ex:
                 exception_result = cast(
-                    "Union[Failure, MutableChain[_T]]",
+                    "Failure | MutableChain[_T]",
                     self._process_spider_exception(
                         response, ex, exception_processor_index
                     ),
@@ -195,7 +195,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                     yield r
             except Exception as ex:
                 exception_result = cast(
-                    "Union[Failure, MutableAsyncChain[_T]]",
+                    "Failure | MutableAsyncChain[_T]",
                     self._process_spider_exception(
                         response, ex, exception_processor_index
                     ),
@@ -241,9 +241,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 # simplified when downgrading is removed.
                 if dfd.called:
                     # the result is available immediately if _process_spider_output didn't do downgrading
-                    return cast(
+                    return cast("MutableChain[_T] | MutableAsyncChain[_T]", dfd.result)
-                        "Union[MutableChain[_T], MutableAsyncChain[_T]]", dfd.result
-                    )
                 # we forbid waiting here because otherwise we would need to return a deferred from
                 # _process_spider_exception too, which complicates the architecture
                 msg = f"Async iterable returned from {global_object_name(method)} cannot be downgraded"

scrapy/extensions/feedexport.py

@@ -16,7 +16,7 @@ from collections.abc import Callable
 from datetime import datetime, timezone
 from pathlib import Path, PureWindowsPath
 from tempfile import NamedTemporaryFile
-from typing import IO, TYPE_CHECKING, Any, Optional, Protocol, TypeVar, cast
+from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, TypeVar, cast
 from urllib.parse import unquote, urlparse
 
 from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
@@ -50,7 +50,9 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
-UriParamsCallableT = Callable[[dict[str, Any], Spider], Optional[dict[str, Any]]]
+UriParamsCallableT: TypeAlias = Callable[
+    [dict[str, Any], Spider], dict[str, Any] | None
+]
 
 _StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol")
 

scrapy/extensions/httpcache.py

@@ -7,7 +7,7 @@ from email.utils import mktime_tz, parsedate_tz
 from importlib import import_module
 from pathlib import Path
 from time import time
-from typing import IO, TYPE_CHECKING, Any, cast
+from typing import IO, TYPE_CHECKING, Any, Concatenate, cast
 from weakref import WeakKeyDictionary
 
 from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
@@ -23,9 +23,6 @@ if TYPE_CHECKING:
     from collections.abc import Callable
     from types import ModuleType
 
-    # typing.Concatenate requires Python 3.10
-    from typing_extensions import Concatenate
-
     from scrapy.http.request import Request
     from scrapy.settings import BaseSettings
     from scrapy.spiders import Spider

scrapy/http/headers.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, AnyStr, Union, cast
+from typing import TYPE_CHECKING, Any, AnyStr, TypeAlias, cast
 
 from w3lib.http import headers_dict_to_raw
 
@@ -15,7 +15,7 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
 
-_RawValueT = Union[bytes, str, int]
+_RawValue: TypeAlias = bytes | str | int
 
 
 # isn't fully compatible typing-wise with either dict or CaselessDict,
@@ -44,9 +44,9 @@ class Headers(CaselessDict):
         """Normalize key to bytes"""
         return self._tobytes(key.title())
 
-    def normvalue(self, value: _RawValueT | Iterable[_RawValueT]) -> list[bytes]:
+    def normvalue(self, value: _RawValue | Iterable[_RawValue]) -> list[bytes]:
         """Normalize values to bytes"""
-        _value: Iterable[_RawValueT]
+        _value: Iterable[_RawValue]
         if value is None:
             _value = []
         elif isinstance(value, (str, bytes)):
@@ -58,7 +58,7 @@ class Headers(CaselessDict):
 
         return [self._tobytes(x) for x in _value]
 
-    def _tobytes(self, x: _RawValueT) -> bytes:
+    def _tobytes(self, x: _RawValue) -> bytes:
         if isinstance(x, bytes):
             return x
         if isinstance(x, str):
@@ -87,15 +87,15 @@ class Headers(CaselessDict):
                 return self.normvalue(def_val)
             return []
 
-    def setlist(self, key: AnyStr, list_: Iterable[_RawValueT]) -> None:
+    def setlist(self, key: AnyStr, list_: Iterable[_RawValue]) -> None:
         self[key] = list_
 
     def setlistdefault(
-        self, key: AnyStr, default_list: Iterable[_RawValueT] = ()
+        self, key: AnyStr, default_list: Iterable[_RawValue] = ()
     ) -> Any:
         return self.setdefault(key, default_list)
 
-    def appendlist(self, key: AnyStr, value: Iterable[_RawValueT]) -> None:
+    def appendlist(self, key: AnyStr, value: Iterable[_RawValue]) -> None:
         lst = self.getlist(key)
         lst.extend(self.normvalue(value))
         self[key] = lst

scrapy/http/request/__init__.py

@@ -12,10 +12,11 @@ from typing import (
     TYPE_CHECKING,
     Any,
     AnyStr,
+    Concatenate,
     NoReturn,
+    TypeAlias,
     TypedDict,
     TypeVar,
-    Union,
     overload,
 )
 
@@ -33,13 +34,13 @@ if TYPE_CHECKING:
 
     from twisted.python.failure import Failure
 
-    # typing.Concatenate requires Python 3.10
     # typing.NotRequired and typing.Self require Python 3.11
-    from typing_extensions import Concatenate, NotRequired, Self
+    from typing_extensions import NotRequired, Self
 
+    # circular import
     from scrapy.http import Response
 
-    CallbackT = Callable[Concatenate[Response, ...], Any]
+    CallbackT: TypeAlias = Callable[Concatenate[Response, ...], Any]
 
 
 class VerboseCookie(TypedDict):
@@ -50,7 +51,7 @@ class VerboseCookie(TypedDict):
     secure: NotRequired[bool]
 
 
-CookiesT = Union[dict[str, str], list[VerboseCookie]]
+CookiesT: TypeAlias = dict[str, str] | list[VerboseCookie]
 
 
 RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")

scrapy/http/request/form.py

@@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst
 from __future__ import annotations
 
 from collections.abc import Iterable
-from typing import TYPE_CHECKING, Any, Optional, Union, cast
+from typing import TYPE_CHECKING, Any, TypeAlias, cast
 from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
 
 from parsel.csstranslator import HTMLTranslator
@@ -31,9 +31,9 @@ if TYPE_CHECKING:
     from scrapy.http.response.text import TextResponse
 
 
-FormdataVType = Union[str, Iterable[str]]
+FormdataVType: TypeAlias = str | Iterable[str]
-FormdataKVType = tuple[str, FormdataVType]
+FormdataKVType: TypeAlias = tuple[str, FormdataVType]
-FormdataType = Optional[Union[dict[str, FormdataVType], list[FormdataKVType]]]
+FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None
 
 
 class FormRequest(Request):

scrapy/linkextractors/lxmlhtml.py

@@ -9,7 +9,7 @@ import operator
 import re
 from collections.abc import Callable, Iterable
 from functools import partial
-from typing import TYPE_CHECKING, Any, Union, cast
+from typing import TYPE_CHECKING, Any, TypeAlias, cast
 from urllib.parse import urljoin, urlparse
 
 from lxml import etree
@@ -157,8 +157,8 @@ class LxmlParserLinkExtractor:
         return links
 
 
-_RegexT = Union[str, re.Pattern[str]]
+_Regex: TypeAlias = str | re.Pattern[str]
-_RegexOrSeveralT = Union[_RegexT, Iterable[_RegexT]]
+_RegexOrSeveral: TypeAlias = _Regex | Iterable[_Regex]
 
 
 class LxmlLinkExtractor:
@@ -166,8 +166,8 @@ class LxmlLinkExtractor:
 
     def __init__(
         self,
-        allow: _RegexOrSeveralT = (),
+        allow: _RegexOrSeveral = (),
-        deny: _RegexOrSeveralT = (),
+        deny: _RegexOrSeveral = (),
         allow_domains: str | Iterable[str] = (),
         deny_domains: str | Iterable[str] = (),
         restrict_xpaths: str | Iterable[str] = (),
@@ -179,7 +179,7 @@ class LxmlLinkExtractor:
         deny_extensions: str | Iterable[str] | None = None,
         restrict_css: str | Iterable[str] = (),
         strip: bool = True,
-        restrict_text: _RegexOrSeveralT | None = None,
+        restrict_text: _RegexOrSeveral | None = None,
     ):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
         self.link_extractor = LxmlParserLinkExtractor(
@@ -208,7 +208,7 @@ class LxmlLinkExtractor:
         self.restrict_text: list[re.Pattern[str]] = self._compile_regexes(restrict_text)
 
     @staticmethod
-    def _compile_regexes(value: _RegexOrSeveralT | None) -> list[re.Pattern[str]]:
+    def _compile_regexes(value: _RegexOrSeveral | None) -> list[re.Pattern[str]]:
         return [
             x if isinstance(x, re.Pattern) else re.compile(x)
             for x in arg_to_iter(value)

scrapy/middleware.py

@@ -5,7 +5,7 @@ import pprint
 import warnings
 from abc import ABC, abstractmethod
 from collections import defaultdict, deque
-from typing import TYPE_CHECKING, Any, TypeVar, cast
+from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, cast
 
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.utils.defer import ensure_awaitable
@@ -18,20 +18,18 @@ if TYPE_CHECKING:
 
     from twisted.internet.defer import Deferred
 
-    # typing.Concatenate and typing.ParamSpec require Python 3.10
     # typing.Self requires Python 3.11
-    from typing_extensions import Concatenate, ParamSpec, Self
+    from typing_extensions import Self
 
     from scrapy import Spider
     from scrapy.crawler import Crawler
     from scrapy.settings import BaseSettings, Settings
 
-    _P = ParamSpec("_P")
-
 
 logger = logging.getLogger(__name__)
 
 _T = TypeVar("_T")
+_P = ParamSpec("_P")
 
 
 class MiddlewareManager(ABC):

scrapy/pipelines/__init__.py

@@ -7,7 +7,7 @@ See documentation in docs/item-pipeline.rst
 from __future__ import annotations
 
 import warnings
-from typing import TYPE_CHECKING, Any, Callable, cast
+from typing import TYPE_CHECKING, Any, cast
 
 from twisted.internet.defer import Deferred, DeferredList
 
@@ -22,7 +22,7 @@ from scrapy.utils.defer import (
 from scrapy.utils.python import global_object_name
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable
+    from collections.abc import Callable, Iterable
 
     from twisted.python.failure import Failure
 

scrapy/pipelines/images.py

@@ -69,7 +69,7 @@ class ImagesPipeline(FilesPipeline):
             self._ImageOps = ImageOps
         except ImportError:
             raise NotConfigured(
-                "ImagesPipeline requires installing Pillow 8.0.0 or later"
+                "ImagesPipeline requires installing Pillow 8.3.2 or later"
             )
 
         super().__init__(

scrapy/pipelines/media.py

@@ -5,7 +5,7 @@ import logging
 import warnings
 from abc import ABC, abstractmethod
 from collections import defaultdict
-from typing import TYPE_CHECKING, Any, Literal, TypedDict, Union, cast
+from typing import TYPE_CHECKING, Any, Literal, TypeAlias, TypedDict, cast
 
 from twisted import version as twisted_version
 from twisted.internet.defer import (
@@ -47,7 +47,9 @@ class FileInfo(TypedDict):
     status: str
 
 
-FileInfoOrError = Union[tuple[Literal[True], FileInfo], tuple[Literal[False], Failure]]
+FileInfoOrError: TypeAlias = (
+    tuple[Literal[True], FileInfo] | tuple[Literal[False], Failure]
+)
 
 logger = logging.getLogger(__name__)
 

scrapy/settings/__init__.py

@@ -6,7 +6,7 @@ import warnings
 from collections.abc import Iterable, Iterator, Mapping, MutableMapping
 from importlib import import_module
 from pprint import pformat
-from typing import TYPE_CHECKING, Any, Union, cast
+from typing import TYPE_CHECKING, Any, TypeAlias, cast
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import default_settings
@@ -14,7 +14,7 @@ from scrapy.utils.misc import load_object
 
 # The key types are restricted in BaseSettings._get_key() to ones supported by JSON,
 # see https://github.com/scrapy/scrapy/issues/5383.
-_SettingsKeyT = Union[bool, float, int, str, None]
+_SettingsKey: TypeAlias = bool | float | int | str | None
 
 if TYPE_CHECKING:
     from types import ModuleType
@@ -25,7 +25,7 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
-    _SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None]
+    _SettingsInput: TypeAlias = SupportsItems[_SettingsKey, Any] | str | None
 
 
 SETTINGS_PRIORITIES: dict[str, int] = {
@@ -76,7 +76,7 @@ class SettingsAttribute:
         return f"<SettingsAttribute value={self.value!r} priority={self.priority}>"
 
 
-class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
+class BaseSettings(MutableMapping[_SettingsKey, Any]):
     """
     Instances of this class behave like dictionaries, but store priorities
     along with their ``(key, value)`` pairs, and can be frozen (i.e. marked
@@ -100,13 +100,13 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
 
     __default = object()
 
-    def __init__(self, values: _SettingsInputT = None, priority: int | str = "project"):
+    def __init__(self, values: _SettingsInput = None, priority: int | str = "project"):
         self.frozen: bool = False
-        self.attributes: dict[_SettingsKeyT, SettingsAttribute] = {}
+        self.attributes: dict[_SettingsKey, SettingsAttribute] = {}
         if values:
             self.update(values, priority)
 
-    def __getitem__(self, opt_name: _SettingsKeyT) -> Any:
+    def __getitem__(self, opt_name: _SettingsKey) -> Any:
         if opt_name not in self:
             return None
         return self.attributes[opt_name].value
@@ -114,7 +114,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
     def __contains__(self, name: Any) -> bool:
         return name in self.attributes
 
-    def add_to_list(self, name: _SettingsKeyT, item: Any) -> None:
+    def add_to_list(self, name: _SettingsKey, item: Any) -> None:
         """Append *item* to the :class:`list` setting with the specified *name*
         if *item* is not already in that list.
 
@@ -125,7 +125,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         if item not in value:
             self.set(name, [*value, item], self.getpriority(name) or 0)
 
-    def remove_from_list(self, name: _SettingsKeyT, item: Any) -> None:
+    def remove_from_list(self, name: _SettingsKey, item: Any) -> None:
         """Remove *item* from the :class:`list` setting with the specified
         *name*.
 
@@ -139,7 +139,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
             raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).")
         self.set(name, [v for v in value if v != item], self.getpriority(name) or 0)
 
-    def get(self, name: _SettingsKeyT, default: Any = None) -> Any:
+    def get(self, name: _SettingsKey, default: Any = None) -> Any:
         """
         Get a setting value without affecting its original type.
 
@@ -160,7 +160,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
 
         return self[name] if self[name] is not None else default
 
-    def getbool(self, name: _SettingsKeyT, default: bool = False) -> bool:
+    def getbool(self, name: _SettingsKey, default: bool = False) -> bool:
         """
         Get a setting value as a boolean.
 
@@ -190,7 +190,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
                 "'True'/'False' and 'true'/'false'"
             )
 
-    def getint(self, name: _SettingsKeyT, default: int = 0) -> int:
+    def getint(self, name: _SettingsKey, default: int = 0) -> int:
         """
         Get a setting value as an int.
 
@@ -202,7 +202,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         """
         return int(self.get(name, default))
 
-    def getfloat(self, name: _SettingsKeyT, default: float = 0.0) -> float:
+    def getfloat(self, name: _SettingsKey, default: float = 0.0) -> float:
         """
         Get a setting value as a float.
 
@@ -215,7 +215,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         return float(self.get(name, default))
 
     def getlist(
-        self, name: _SettingsKeyT, default: list[Any] | None = None
+        self, name: _SettingsKey, default: list[Any] | None = None
     ) -> list[Any]:
         """
         Get a setting value as a list. If the setting original type is a list,
@@ -239,7 +239,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         return list(value)
 
     def getdict(
-        self, name: _SettingsKeyT, default: dict[Any, Any] | None = None
+        self, name: _SettingsKey, default: dict[Any, Any] | None = None
     ) -> dict[Any, Any]:
         """
         Get a setting value as a dictionary. If the setting original type is a
@@ -263,7 +263,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
 
     def getdictorlist(
         self,
-        name: _SettingsKeyT,
+        name: _SettingsKey,
         default: dict[Any, Any] | list[Any] | tuple[Any] | None = None,
     ) -> dict[Any, Any] | list[Any]:
         """Get a setting value as either a :class:`dict` or a :class:`list`.
@@ -310,7 +310,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
             )
         return copy.deepcopy(value)
 
-    def getwithbase(self, name: _SettingsKeyT) -> BaseSettings:
+    def getwithbase(self, name: _SettingsKey) -> BaseSettings:
         """Get a composition of a dictionary-like setting and its `_BASE`
         counterpart.
 
@@ -324,7 +324,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         compbs.update(self[name])
         return compbs
 
-    def getpriority(self, name: _SettingsKeyT) -> int | None:
+    def getpriority(self, name: _SettingsKey) -> int | None:
         """
         Return the current numerical priority value of a setting, or ``None`` if
         the given ``name`` does not exist.
@@ -349,7 +349,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
 
     def replace_in_component_priority_dict(
         self,
-        name: _SettingsKeyT,
+        name: _SettingsKey,
         old_cls: type,
         new_cls: type,
         priority: int | None = None,
@@ -388,11 +388,11 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         )
         self.set(name, component_priority_dict, priority=self.getpriority(name) or 0)
 
-    def __setitem__(self, name: _SettingsKeyT, value: Any) -> None:
+    def __setitem__(self, name: _SettingsKey, value: Any) -> None:
         self.set(name, value)
 
     def set(
-        self, name: _SettingsKeyT, value: Any, priority: int | str = "project"
+        self, name: _SettingsKey, value: Any, priority: int | str = "project"
     ) -> None:
         """
         Store a key/value attribute with a given priority.
@@ -422,7 +422,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
             self.attributes[name].set(value, priority)
 
     def set_in_component_priority_dict(
-        self, name: _SettingsKeyT, cls: type, priority: int | None
+        self, name: _SettingsKey, cls: type, priority: int | None
     ) -> None:
         """Set the *cls* component in the *name* :ref:`component priority
         dictionary <component-priority-dictionaries>` setting with *priority*.
@@ -447,7 +447,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
 
     def setdefault(
         self,
-        name: _SettingsKeyT,
+        name: _SettingsKey,
         default: Any = None,
         priority: int | str = "project",
     ) -> Any:
@@ -458,7 +458,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         return self.attributes[name].value
 
     def setdefault_in_component_priority_dict(
-        self, name: _SettingsKeyT, cls: type, priority: int | None
+        self, name: _SettingsKey, cls: type, priority: int | None
     ) -> None:
         """Set the *cls* component in the *name* :ref:`component priority
         dictionary <component-priority-dictionaries>` setting with *priority*
@@ -475,7 +475,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         component_priority_dict[cls] = priority
         self.set(name, component_priority_dict, self.getpriority(name) or 0)
 
-    def setdict(self, values: _SettingsInputT, priority: int | str = "project") -> None:
+    def setdict(self, values: _SettingsInput, priority: int | str = "project") -> None:
         self.update(values, priority)
 
     def setmodule(
@@ -503,7 +503,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
                 self.set(key, getattr(module, key), priority)
 
     # BaseSettings.update() doesn't support all inputs that MutableMapping.update() supports
-    def update(self, values: _SettingsInputT, priority: int | str = "project") -> None:  # type: ignore[override]
+    def update(self, values: _SettingsInput, priority: int | str = "project") -> None:  # type: ignore[override]
         """
         Store key/value pairs with a given priority.
 
@@ -527,7 +527,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         """
         self._assert_mutability()
         if isinstance(values, str):
-            values = cast("dict[_SettingsKeyT, Any]", json.loads(values))
+            values = cast("dict[_SettingsKey, Any]", json.loads(values))
         if values is not None:
             if isinstance(values, BaseSettings):
                 for name, value in values.items():
@@ -536,7 +536,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
                 for name, value in values.items():
                     self.set(name, value, priority)
 
-    def delete(self, name: _SettingsKeyT, priority: int | str = "project") -> None:
+    def delete(self, name: _SettingsKey, priority: int | str = "project") -> None:
         if name not in self:
             raise KeyError(name)
         self._assert_mutability()
@@ -544,7 +544,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         if priority >= cast("int", self.getpriority(name)):
             del self.attributes[name]
 
-    def __delitem__(self, name: _SettingsKeyT) -> None:
+    def __delitem__(self, name: _SettingsKey) -> None:
         self._assert_mutability()
         del self.attributes[name]
 
@@ -584,26 +584,26 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         copy.freeze()
         return copy
 
-    def __iter__(self) -> Iterator[_SettingsKeyT]:
+    def __iter__(self) -> Iterator[_SettingsKey]:
         return iter(self.attributes)
 
     def __len__(self) -> int:
         return len(self.attributes)
 
-    def _to_dict(self) -> dict[_SettingsKeyT, Any]:
+    def _to_dict(self) -> dict[_SettingsKey, Any]:
         return {
             self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
             for k, v in self.items()
         }
 
-    def _get_key(self, key_value: Any) -> _SettingsKeyT:
+    def _get_key(self, key_value: Any) -> _SettingsKey:
         return (
             key_value
             if isinstance(key_value, (bool, float, int, str, type(None)))
             else str(key_value)
         )
 
-    def copy_to_dict(self) -> dict[_SettingsKeyT, Any]:
+    def copy_to_dict(self) -> dict[_SettingsKey, Any]:
         """
         Make a copy of current settings and convert to a dict.
 
@@ -626,7 +626,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         else:
             p.text(pformat(self.copy_to_dict()))
 
-    def pop(self, name: _SettingsKeyT, default: Any = __default) -> Any:
+    def pop(self, name: _SettingsKey, default: Any = __default) -> Any:
         try:
             value = self.attributes[name].value
         except KeyError:
@@ -648,7 +648,7 @@ class Settings(BaseSettings):
     described on :ref:`topics-settings-ref` already populated.
     """
 
-    def __init__(self, values: _SettingsInputT = None, priority: int | str = "project"):
+    def __init__(self, values: _SettingsInput = None, priority: int | str = "project"):
         # Do not pass kwarg values here. We don't want to promote user-defined
         # dicts, and we want to update, not replace, default dicts with the
         # values given by the user
@@ -670,7 +670,7 @@ def iter_default_settings() -> Iterable[tuple[str, Any]]:
 
 
 def overridden_settings(
-    settings: Mapping[_SettingsKeyT, Any],
+    settings: Mapping[_SettingsKey, Any],
 ) -> Iterable[tuple[str, Any]]:
     """Return an iterable of the settings that have been overridden"""
     for name, defvalue in iter_default_settings():

scrapy/spiders/__init__.py

@@ -26,7 +26,7 @@ if TYPE_CHECKING:
 
     from scrapy.crawler import Crawler
     from scrapy.http.request import CallbackT
-    from scrapy.settings import BaseSettings, _SettingsKeyT
+    from scrapy.settings import BaseSettings, _SettingsKey
     from scrapy.utils.log import SpiderLoggerAdapter
 
 
@@ -39,7 +39,7 @@ class Spider(object_ref):
     """
 
     name: str
-    custom_settings: dict[_SettingsKeyT, Any] | None = None
+    custom_settings: dict[_SettingsKey, Any] | None = None
 
     #: Start URLs. See :meth:`start`.
     start_urls: list[str]

scrapy/spiders/crawl.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 import copy
 import warnings
 from collections.abc import AsyncIterator, Awaitable, Callable
-from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast
+from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast
 
 from scrapy.http import HtmlResponse, Request, Response
 from scrapy.link import Link
@@ -34,8 +34,8 @@ if TYPE_CHECKING:
 
 
 _T = TypeVar("_T")
-ProcessLinksT = Callable[[list[Link]], list[Link]]
+ProcessLinksT: TypeAlias = Callable[[list[Link]], list[Link]]
-ProcessRequestT = Callable[[Request, Response], Optional[Request]]
+ProcessRequestT: TypeAlias = Callable[[Request, Response], Request | None]
 
 
 def _identity(x: _T) -> _T:

scrapy/utils/asyncio.py

@@ -6,7 +6,7 @@ import asyncio
 import logging
 import time
 from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
-from typing import TYPE_CHECKING, Any, TypeVar
+from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar
 
 from twisted.internet.defer import Deferred
 from twisted.internet.task import LoopingCall
@@ -17,15 +17,14 @@ from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_instal
 if TYPE_CHECKING:
     from twisted.internet.base import DelayedCall
 
-    # typing.Concatenate and typing.ParamSpec require Python 3.10
     # typing.Self, typing.TypeVarTuple and typing.Unpack require Python 3.11
-    from typing_extensions import Concatenate, ParamSpec, Self, TypeVarTuple, Unpack
+    from typing_extensions import Self, TypeVarTuple, Unpack
 
-    _P = ParamSpec("_P")
     _Ts = TypeVarTuple("_Ts")
 
 
 _T = TypeVar("_T")
+_P = ParamSpec("_P")
 
 
 logger = logging.getLogger(__name__)

scrapy/utils/conf.py

@@ -6,7 +6,7 @@ import sys
 from configparser import ConfigParser
 from operator import itemgetter
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, cast
+from typing import TYPE_CHECKING, Any, cast
 
 from scrapy.exceptions import UsageError
 from scrapy.settings import BaseSettings
@@ -14,7 +14,7 @@ from scrapy.utils.deprecate import update_classpath
 from scrapy.utils.python import without_none_values
 
 if TYPE_CHECKING:
-    from collections.abc import Collection, Iterable, Mapping, MutableMapping
+    from collections.abc import Callable, Collection, Iterable, Mapping, MutableMapping
 
 
 def build_component_list(

scrapy/utils/decorators.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import inspect
 import warnings
 from functools import wraps
-from typing import TYPE_CHECKING, Any, TypeVar, overload
+from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, overload
 
 from twisted.internet.defer import Deferred, maybeDeferred
 from twisted.internet.threads import deferToThread
@@ -13,13 +13,9 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator, Callable, Coroutine
 
-    # typing.ParamSpec requires Python 3.10
-    from typing_extensions import ParamSpec
-
-    _P = ParamSpec("_P")
-
 
 _T = TypeVar("_T")
+_P = ParamSpec("_P")
 
 
 def deprecated(

scrapy/utils/defer.py

@@ -10,7 +10,16 @@ import warnings
 from asyncio import Future
 from collections.abc import Awaitable, Coroutine, Iterable, Iterator
 from functools import wraps
-from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast, overload
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Concatenate,
+    Generic,
+    ParamSpec,
+    TypeVar,
+    cast,
+    overload,
+)
 
 from twisted.internet.defer import Deferred, DeferredList, fail, succeed
 from twisted.internet.task import Cooperator
@@ -24,14 +33,10 @@ if TYPE_CHECKING:
 
     from twisted.python.failure import Failure
 
-    # typing.Concatenate and typing.ParamSpec require Python 3.10
-    from typing_extensions import Concatenate, ParamSpec
-
-    _P = ParamSpec("_P")
-
 
 _T = TypeVar("_T")
 _T2 = TypeVar("_T2")
+_P = ParamSpec("_P")
 
 
 _DEFER_DELAY = 0.1
@@ -327,7 +332,7 @@ def process_chain_both(
         stacklevel=2,
     )
     d: Deferred = Deferred()
-    for cb, eb in zip(callbacks, errbacks):
+    for cb, eb in zip(callbacks, errbacks, strict=False):
         d.addCallback(cb, *a, **kw)
         d.addErrback(eb, *a, **kw)
     if isinstance(input, failure.Failure):

scrapy/utils/iterators.py

@@ -212,7 +212,7 @@ def csviter(
                 },
             )
             continue
-        yield dict(zip(headers, row))
+        yield dict(zip(headers, row, strict=False))
 
 
 @overload

scrapy/utils/log.py

@@ -5,14 +5,14 @@ import pprint
 import sys
 from collections.abc import MutableMapping
 from logging.config import dictConfig
-from typing import TYPE_CHECKING, Any, Optional, cast
+from typing import TYPE_CHECKING, Any, cast
 
 from twisted.internet import asyncioreactor
 from twisted.python import log as twisted_log
 from twisted.python.failure import Failure
 
 import scrapy
-from scrapy.settings import Settings, _SettingsKeyT
+from scrapy.settings import Settings, _SettingsKey
 from scrapy.utils.versions import get_versions
 
 if TYPE_CHECKING:
@@ -35,7 +35,7 @@ def failure_to_exc_info(
         return (
             failure.type,
             failure.value,
-            cast("Optional[TracebackType]", failure.getTracebackObject()),
+            cast("TracebackType | None", failure.getTracebackObject()),
         )
     return None
 
@@ -83,7 +83,7 @@ DEFAULT_LOGGING = {
 
 
 def configure_logging(
-    settings: Settings | dict[_SettingsKeyT, Any] | None = None,
+    settings: Settings | dict[_SettingsKey, Any] | None = None,
     install_root_handler: bool = True,
 ) -> None:
     """

scrapy/utils/ossignal.py

@@ -3,12 +3,12 @@ from __future__ import annotations
 import signal
 from collections.abc import Callable
 from types import FrameType
-from typing import Any, Optional, Union
+from typing import Any, TypeAlias
 
 # copy of _HANDLER from typeshed/stdlib/signal.pyi
-SignalHandlerT = Union[
+SignalHandlerT: TypeAlias = (
-    Callable[[int, Optional[FrameType]], Any], int, signal.Handlers, None
+    Callable[[int, FrameType | None], Any] | int | signal.Handlers | None
-]
+)
 
 signal_names: dict[int, str] = {}
 for signame in dir(signal):

scrapy/utils/python.py

@@ -13,7 +13,7 @@ import weakref
 from collections.abc import AsyncIterator, Iterable, Mapping
 from functools import partial, wraps
 from itertools import chain
-from typing import TYPE_CHECKING, Any, TypeVar, overload
+from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.asyncgen import as_async_generator
@@ -22,15 +22,14 @@ if TYPE_CHECKING:
     from collections.abc import Callable, Iterator
     from re import Pattern
 
-    # typing.Concatenate and typing.ParamSpec require Python 3.10
     # typing.Self requires Python 3.11
-    from typing_extensions import Concatenate, ParamSpec, Self
+    from typing_extensions import Self
 
-    _P = ParamSpec("_P")
 
 _T = TypeVar("_T")
 _KT = TypeVar("_KT")
 _VT = TypeVar("_VT")
+_P = ParamSpec("_P")
 
 
 def flatten(x: Iterable[Any]) -> list[Any]:
@@ -286,7 +285,7 @@ def get_spec(func: Callable[..., Any]) -> tuple[list[str], dict[str, Any]]:
 
     firstdefault = len(spec.args) - len(defaults)
     args = spec.args[:firstdefault]
-    kwargs = dict(zip(spec.args[firstdefault:], defaults))
+    kwargs = dict(zip(spec.args[firstdefault:], defaults, strict=False))
     return args, kwargs
 
 

scrapy/utils/reactor.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import asyncio
 import sys
 from contextlib import suppress
-from typing import TYPE_CHECKING, Any, Generic, TypeVar
+from typing import TYPE_CHECKING, Any, Generic, ParamSpec, TypeVar
 from warnings import catch_warnings, filterwarnings
 
 from twisted.internet import asyncioreactor, error
@@ -19,14 +19,11 @@ if TYPE_CHECKING:
     from twisted.internet.protocol import ServerFactory
     from twisted.internet.tcp import Port
 
-    # typing.ParamSpec requires Python 3.10
-    from typing_extensions import ParamSpec
-
     from scrapy.utils.asyncio import CallLaterResult
 
-    _P = ParamSpec("_P")
 
 _T = TypeVar("_T")
+_P = ParamSpec("_P")
 
 
 def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:  # type: ignore[return]  # pylint: disable=inconsistent-return-statements  # noqa: RET503

scrapy/utils/url.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 import re
 import warnings
 from importlib import import_module
-from typing import TYPE_CHECKING, Union
+from typing import TYPE_CHECKING, TypeAlias
 from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
 from warnings import warn
 
@@ -37,7 +37,7 @@ if TYPE_CHECKING:
 
     from scrapy import Spider
 
-UrlT = Union[str, bytes, ParseResult]
+UrlT: TypeAlias = str | bytes | ParseResult
 
 
 def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:

tests/test_crawl.py

@@ -369,7 +369,7 @@ with multiples lines
         est = [x for sublist in est for x in sublist]  # flatten
         est = [x.lstrip().rstrip() for x in est]
         it = iter(est)
-        s = dict(zip(it, it))
+        s = dict(zip(it, it, strict=False))
 
         assert s["engine.spider.name"] == crawler.spider.name
         assert s["len(engine.scraper.slot.active)"] == "1"

tests/test_extension_periodic_log.py

@@ -1,13 +1,17 @@
 from __future__ import annotations
 
 import datetime
-from typing import Any, Callable
+from typing import TYPE_CHECKING, Any
 
 from scrapy.extensions.periodic_log import PeriodicLog
 from scrapy.utils.test import get_crawler
 
 from .spiders import MetaSpider
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+
 stats_dump_1 = {
     "log_count/INFO": 10,
     "log_count/WARNING": 1,

tests/test_feedexport.py

@@ -2628,7 +2628,7 @@ class TestBatchDeliveries(TestFeedExportBase):
         }
         data = await self.exported_data(items, settings)
         for fmt, expected in formats.items():
-            for expected_batch, got_batch in zip(expected, data[fmt]):
+            for expected_batch, got_batch in zip(expected, data[fmt], strict=False):
                 assert got_batch == expected_batch
 
     @deferred_f_from_coro_f
@@ -2652,7 +2652,7 @@ class TestBatchDeliveries(TestFeedExportBase):
         }
         data = await self.exported_data(items, settings)
         for fmt, expected in formats.items():
-            for expected_batch, got_batch in zip(expected, data[fmt]):
+            for expected_batch, got_batch in zip(expected, data[fmt], strict=False):
                 assert got_batch == expected_batch
 
     @deferred_f_from_coro_f

tests/test_http2_client_protocol.py

@@ -6,7 +6,7 @@ import re
 import string
 from ipaddress import IPv4Address
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, cast
+from typing import TYPE_CHECKING, Any, cast
 from unittest import mock
 from urllib.parse import urlencode
 
@@ -39,7 +39,7 @@ from tests.mockserver.http_resources import LeafResource, Status
 from tests.mockserver.utils import ssl_context_factory
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncGenerator, Coroutine, Generator
+    from collections.abc import AsyncGenerator, Callable, Coroutine, Generator
 
     from scrapy.core.http2.protocol import H2ClientProtocol
 

tests/test_http_response.py

@@ -333,7 +333,7 @@ class TestResponseBase:
         if response is None:
             response = self._links_response()
         followed = response.follow_all(follow_obj)
-        for req, target in zip(followed, target_urls):
+        for req, target in zip(followed, target_urls, strict=False):
             assert req.url == target
             yield req
 
@@ -647,7 +647,7 @@ class TestTextResponse(TestResponseBase):
 
         # select <a> elements
         for sellist in [resp.css("a"), resp.xpath("//a")]:
-            for sel, url in zip(sellist, urls):
+            for sel, url in zip(sellist, urls, strict=False):
                 self._assert_followed_url(sel, url, response=resp)
 
         # select <link> elements
@@ -659,7 +659,7 @@ class TestTextResponse(TestResponseBase):
 
         # href attributes should work
         for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]:
-            for sel, url in zip(sellist, urls):
+            for sel, url in zip(sellist, urls, strict=False):
                 self._assert_followed_url(sel, url, response=resp)
 
         # non-a elements are not supported

tests_typing/test_http_request.mypy-testing

@@ -16,7 +16,7 @@ class MyRequest2(Request):
 
 @pytest.mark.mypy_testing
 def mypy_test_headers():
-    Request("data:,", headers=1)  # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[tuple[str, Any]], None]"
+    Request("data:,", headers=1)  # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
     Request("data:,", headers=None)
     Request("data:,", headers={})
     Request("data:,", headers=[])

tests_typing/test_http_response.mypy-testing

@@ -7,7 +7,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse
 
 @pytest.mark.mypy_testing
 def mypy_test_headers():
-    Response("data:,", headers=1)  # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[tuple[str, Any]], None]"
+    Response("data:,", headers=1)  # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
     Response("data:,", headers=None)
     Response("data:,", headers={})
     Response("data:,", headers=[])

tox.ini

@@ -43,7 +43,7 @@ commands =
     pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 scrapy tests --doctest-modules}
 
 [testenv:typing]
-basepython = python3.9
+basepython = python3.10
 deps =
     mypy==1.16.1
     typing-extensions==4.14.1
@@ -64,7 +64,7 @@ commands =
     mypy {posargs:scrapy tests}
 
 [testenv:typing-tests]
-basepython = python3.9
+basepython = python3.10
 deps =
     {[test-requirements]deps}
     {[testenv:typing]deps}
@@ -97,7 +97,7 @@ commands =
     twine check dist/*
 
 [pinned]
-basepython = python3.9
+basepython = python3.10
 deps =
     # pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0
     pytest==8.4.0
@@ -106,7 +106,7 @@ deps =
     cryptography==37.0.0
     cssselect==0.9.1
     itemadapter==0.1.0
-    lxml==4.6.0
+    lxml==4.6.4
     parsel==1.5.0
     pyOpenSSL==22.0.0
     queuelib==1.4.2
@@ -151,17 +151,17 @@ deps =
 basepython = {[pinned]basepython}
 deps =
     {[pinned]deps}
-    Pillow==8.0.0
+    Pillow==8.3.2
     boto3==1.20.0
     bpython==0.7.1
     brotli==0.5.2; implementation_name != "pypy"
     brotlicffi==0.8.0; implementation_name == "pypy"
     brotlipy
     google-cloud-storage==1.29.0
-    ipython==2.0.0
+    ipython==7.1.0
     robotexclusionrulesparser==1.6.2
-    uvloop==0.14.0; platform_system != "Windows" and implementation_name != "pypy"
+    uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"
-    zstandard==0.1; implementation_name != "pypy"
+    zstandard==0.16.0; implementation_name != "pypy"
 setenv =
     {[pinned]setenv}
 commands = {[pinned]commands}
@@ -254,7 +254,7 @@ commands =
 [testenv:botocore]
 deps =
     {[testenv]deps}
-    botocore>=1.4.87
+    botocore>=1.13.45
 commands =
     pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy -m requires_botocore}
 
@@ -262,7 +262,7 @@ commands =
 basepython = {[pinned]basepython}
 deps =
     {[pinned]deps}
-    botocore==1.4.87
+    botocore==1.13.45
 setenv =
     {[pinned]setenv}
 commands =