RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove Python 3.9 support (#7121)

This commit is contained in:
Andrey Rakhmatullin 2025-10-27 16:37:49 +05:00 committed by GitHub
parent e48a1bdde3
commit d414d393d4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
43 changed files with 175 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/checks.yml vendored
View File

@ -20,10 +20,10 @@ jobs:
- python-version: "3.13"
env:
TOXENV: pylint
- python-version: "3.9"
- python-version: "3.10"
env:
TOXENV: typing
- python-version: "3.9"
- python-version: "3.10"
env:
TOXENV: typing-tests
- python-version: "3.13" # Keep in sync with .readthedocs.yml

2
.github/workflows/tests-macos.yml vendored
View File

@ -16,7 +16,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
python-version: ["3.10", "3.11", "3.12", "3.13"]
steps:
- uses: actions/checkout@v4

11
.github/workflows/tests-ubuntu.yml vendored
View File

@ -17,9 +17,6 @@ jobs:
fail-fast: false
matrix:
include:
- python-version: "3.9"
env:
TOXENV: py
- python-version: "3.10"
env:
TOXENV: py
@ -40,19 +37,19 @@ jobs:
TOXENV: pypy3
# pinned deps
- python-version: "3.9.21"
- python-version: "3.10.19"
env:
TOXENV: pinned
- python-version: "3.9.21"
- python-version: "3.10.19"
env:
TOXENV: default-reactor-pinned
- python-version: pypy3.11
env:
TOXENV: pypy3-pinned
- python-version: "3.9.21"
- python-version: "3.10.19"
env:
TOXENV: extra-deps-pinned
- python-version: "3.9.21"
- python-version: "3.10.19"
env:
TOXENV: botocore-pinned

7
.github/workflows/tests-windows.yml vendored
View File

@ -17,9 +17,6 @@ jobs:
fail-fast: false
matrix:
include:
- python-version: "3.9"
env:
TOXENV: py
- python-version: "3.10"
env:
TOXENV: py
@ -37,10 +34,10 @@ jobs:
TOXENV: default-reactor
# pinned deps
- python-version: "3.9.13"
- python-version: "3.10.11"
env:
TOXENV: pinned
- python-version: "3.9.13"
- python-version: "3.10.11"
env:
TOXENV: extra-deps-pinned

2
README.rst
View File

@ -40,7 +40,7 @@
:alt: Ask DeepWiki
Scrapy_ is a web scraping framework to extract structured data from websites.
It is cross-platform, and requires Python 3.9+. It is maintained by Zyte_
It is cross-platform, and requires Python 3.10+. It is maintained by Zyte_
(formerly Scrapinghub) and `many other contributors`_.
.. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors

2
docs/intro/install.rst
View File

@ -9,7 +9,7 @@ Installation guide
Supported Python versions
=========================
Scrapy requires Python 3.9+, either the CPython implementation (default) or
Scrapy requires Python 3.10+, either the CPython implementation (default) or
the PyPy implementation (see :ref:`python:implementations`).
.. _intro-install-scrapy:

4
docs/topics/media-pipeline.rst
View File

@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
can configure some extra functions like generating thumbnails and filtering
the images based on their size.
The Images Pipeline requires Pillow_ 8.0.0 or greater. It is used for
The Images Pipeline requires Pillow_ 8.3.2 or greater. It is used for
thumbnailing and normalizing images to JPEG/RGB format.
.. _Pillow: https://github.com/python-pillow/Pillow
@ -238,7 +238,7 @@ Amazon S3 storage
.. setting:: FILES_STORE_S3_ACL
.. setting:: IMAGES_STORE_S3_ACL
If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and
If botocore_ >= 1.13.45 is installed, :setting:`FILES_STORE` and
:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will
automatically upload the files to the bucket.

7
pyproject.toml
View File

@ -13,7 +13,7 @@ dependencies = [
"defusedxml>=0.7.1",
"itemadapter>=0.1.0",
"itemloaders>=1.0.1",
"lxml>=4.6.0",
"lxml>=4.6.4",
"packaging",
"parsel>=1.5.0",
"protego>=0.1.15",
@ -35,7 +35,6 @@ classifiers = [
"Operating System :: OS Independent",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
@ -49,7 +48,7 @@ classifiers = [
license = "BSD-3-Clause"
license-files = ["LICENSE", "AUTHORS"]
readme = "README.rst"
requires-python = ">=3.9"
requires-python = ">=3.10"
authors = [{ name = "Scrapy developers", email = "pablo@pablohoffman.com" }]
maintainers = [{ name = "Pablo Hoffman", email = "pablo@pablohoffman.com" }]
@ -375,6 +374,8 @@ ignore = [
"SIM115",
# Yoda condition detected
"SIM300",
# removed in recent ruff
"UP038",
# Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")

11
scrapy/cmdline.py
View File

@ -6,7 +6,7 @@ import inspect
import os
import sys
from importlib.metadata import entry_points
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, ParamSpec
import scrapy
from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
@ -20,9 +20,6 @@ from scrapy.utils.reactor import _asyncio_reactor_path
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
# typing.ParamSpec requires Python 3.10
from typing_extensions import ParamSpec
from scrapy.settings import BaseSettings, Settings
_P = ParamSpec("_P")
@ -67,11 +64,7 @@ def _get_commands_from_entry_points(
inproject: bool, group: str = "scrapy.commands"
) -> dict[str, ScrapyCommand]:
cmds: dict[str, ScrapyCommand] = {}
if sys.version_info >= (3, 10):
eps = entry_points(group=group)
else:
eps = entry_points().get(group, ())
for entry_point in eps:
for entry_point in entry_points(group=group):
obj = entry_point.load()
if inspect.isclass(obj):
cmds[entry_point.name] = obj()

5
scrapy/contracts/default.py
View File

@ -1,7 +1,7 @@
from __future__ import annotations
import json
from typing import Any, Callable
from typing import TYPE_CHECKING, Any
from itemadapter import ItemAdapter, is_item
@ -9,6 +9,9 @@ from scrapy.contracts import Contract
from scrapy.exceptions import ContractFail
from scrapy.http import Request
if TYPE_CHECKING:
from collections.abc import Callable
# contracts
class UrlContract(Contract):

4
scrapy/core/scraper.py
View File

@ -7,7 +7,7 @@ import logging
import warnings
from collections import deque
from collections.abc import AsyncIterator
from typing import TYPE_CHECKING, Any, TypeVar, Union
from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.python.failure import Failure
@ -53,7 +53,7 @@ logger = logging.getLogger(__name__)
_T = TypeVar("_T")
QueueTuple = tuple[Union[Response, Failure], Request, Deferred[None]]
QueueTuple: TypeAlias = tuple[Response | Failure, Request, Deferred[None]]
class Slot:

16
scrapy/core/spidermw.py
View File

@ -11,7 +11,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
from functools import wraps
from inspect import isasyncgenfunction, iscoroutine
from itertools import islice
from typing import TYPE_CHECKING, Any, TypeVar, Union, cast
from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast
from warnings import warn
from twisted.internet.defer import Deferred, inlineCallbacks
@ -41,9 +41,9 @@ logger = logging.getLogger(__name__)
_T = TypeVar("_T")
ScrapeFunc = Callable[
[Union[Response, Failure], Request],
Coroutine[Any, Any, Union[Iterable[_T], AsyncIterator[_T]]],
ScrapeFunc: TypeAlias = Callable[
[Response | Failure, Request],
Coroutine[Any, Any, Iterable[_T] | AsyncIterator[_T]],
]
@ -179,7 +179,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
yield from iterable
except Exception as ex:
exception_result = cast(
"Union[Failure, MutableChain[_T]]",
"Failure | MutableChain[_T]",
self._process_spider_exception(
response, ex, exception_processor_index
),
@ -195,7 +195,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
yield r
except Exception as ex:
exception_result = cast(
"Union[Failure, MutableAsyncChain[_T]]",
"Failure | MutableAsyncChain[_T]",
self._process_spider_exception(
response, ex, exception_processor_index
),
@ -241,9 +241,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
# simplified when downgrading is removed.
if dfd.called:
# the result is available immediately if _process_spider_output didn't do downgrading
return cast(
"Union[MutableChain[_T], MutableAsyncChain[_T]]", dfd.result
)
return cast("MutableChain[_T] | MutableAsyncChain[_T]", dfd.result)
# we forbid waiting here because otherwise we would need to return a deferred from
# _process_spider_exception too, which complicates the architecture
msg = f"Async iterable returned from {global_object_name(method)} cannot be downgraded"

6
scrapy/extensions/feedexport.py
View File

@ -16,7 +16,7 @@ from collections.abc import Callable
from datetime import datetime, timezone
from pathlib import Path, PureWindowsPath
from tempfile import NamedTemporaryFile
from typing import IO, TYPE_CHECKING, Any, Optional, Protocol, TypeVar, cast
from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, TypeVar, cast
from urllib.parse import unquote, urlparse
from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
@ -50,7 +50,9 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
UriParamsCallableT = Callable[[dict[str, Any], Spider], Optional[dict[str, Any]]]
UriParamsCallableT: TypeAlias = Callable[
[dict[str, Any], Spider], dict[str, Any] | None
]
_StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol")

5
scrapy/extensions/httpcache.py
View File

@ -7,7 +7,7 @@ from email.utils import mktime_tz, parsedate_tz
from importlib import import_module
from pathlib import Path
from time import time
from typing import IO, TYPE_CHECKING, Any, cast
from typing import IO, TYPE_CHECKING, Any, Concatenate, cast
from weakref import WeakKeyDictionary
from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
@ -23,9 +23,6 @@ if TYPE_CHECKING:
from collections.abc import Callable
from types import ModuleType
# typing.Concatenate requires Python 3.10
from typing_extensions import Concatenate
from scrapy.http.request import Request
from scrapy.settings import BaseSettings
from scrapy.spiders import Spider

16
scrapy/http/headers.py
View File

@ -1,7 +1,7 @@
from __future__ import annotations
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, AnyStr, Union, cast
from typing import TYPE_CHECKING, Any, AnyStr, TypeAlias, cast
from w3lib.http import headers_dict_to_raw
@ -15,7 +15,7 @@ if TYPE_CHECKING:
from typing_extensions import Self
_RawValueT = Union[bytes, str, int]
_RawValue: TypeAlias = bytes | str | int
# isn't fully compatible typing-wise with either dict or CaselessDict,
@ -44,9 +44,9 @@ class Headers(CaselessDict):
"""Normalize key to bytes"""
return self._tobytes(key.title())
def normvalue(self, value: _RawValueT | Iterable[_RawValueT]) -> list[bytes]:
def normvalue(self, value: _RawValue | Iterable[_RawValue]) -> list[bytes]:
"""Normalize values to bytes"""
_value: Iterable[_RawValueT]
_value: Iterable[_RawValue]
if value is None:
_value = []
elif isinstance(value, (str, bytes)):
@ -58,7 +58,7 @@ class Headers(CaselessDict):
return [self._tobytes(x) for x in _value]
def _tobytes(self, x: _RawValueT) -> bytes:
def _tobytes(self, x: _RawValue) -> bytes:
if isinstance(x, bytes):
return x
if isinstance(x, str):
@ -87,15 +87,15 @@ class Headers(CaselessDict):
return self.normvalue(def_val)
return []
def setlist(self, key: AnyStr, list_: Iterable[_RawValueT]) -> None:
def setlist(self, key: AnyStr, list_: Iterable[_RawValue]) -> None:
self[key] = list_
def setlistdefault(
self, key: AnyStr, default_list: Iterable[_RawValueT] = ()
self, key: AnyStr, default_list: Iterable[_RawValue] = ()
) -> Any:
return self.setdefault(key, default_list)
def appendlist(self, key: AnyStr, value: Iterable[_RawValueT]) -> None:
def appendlist(self, key: AnyStr, value: Iterable[_RawValue]) -> None:
lst = self.getlist(key)
lst.extend(self.normvalue(value))
self[key] = lst

11
scrapy/http/request/__init__.py
View File

@ -12,10 +12,11 @@ from typing import (
TYPE_CHECKING,
Any,
AnyStr,
Concatenate,
NoReturn,
TypeAlias,
TypedDict,
TypeVar,
Union,
overload,
)
@ -33,13 +34,13 @@ if TYPE_CHECKING:
from twisted.python.failure import Failure
# typing.Concatenate requires Python 3.10
# typing.NotRequired and typing.Self require Python 3.11
from typing_extensions import Concatenate, NotRequired, Self
from typing_extensions import NotRequired, Self
# circular import
from scrapy.http import Response
CallbackT = Callable[Concatenate[Response, ...], Any]
CallbackT: TypeAlias = Callable[Concatenate[Response, ...], Any]
class VerboseCookie(TypedDict):
@ -50,7 +51,7 @@ class VerboseCookie(TypedDict):
secure: NotRequired[bool]
CookiesT = Union[dict[str, str], list[VerboseCookie]]
CookiesT: TypeAlias = dict[str, str] | list[VerboseCookie]
RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")

8
scrapy/http/request/form.py
View File

@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst
from __future__ import annotations
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, Optional, Union, cast
from typing import TYPE_CHECKING, Any, TypeAlias, cast
from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
from parsel.csstranslator import HTMLTranslator
@ -31,9 +31,9 @@ if TYPE_CHECKING:
from scrapy.http.response.text import TextResponse
FormdataVType = Union[str, Iterable[str]]
FormdataKVType = tuple[str, FormdataVType]
FormdataType = Optional[Union[dict[str, FormdataVType], list[FormdataKVType]]]
FormdataVType: TypeAlias = str | Iterable[str]
FormdataKVType: TypeAlias = tuple[str, FormdataVType]
FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None
class FormRequest(Request):

14
scrapy/linkextractors/lxmlhtml.py
View File

@ -9,7 +9,7 @@ import operator
import re
from collections.abc import Callable, Iterable
from functools import partial
from typing import TYPE_CHECKING, Any, Union, cast
from typing import TYPE_CHECKING, Any, TypeAlias, cast
from urllib.parse import urljoin, urlparse
from lxml import etree
@ -157,8 +157,8 @@ class LxmlParserLinkExtractor:
return links
_RegexT = Union[str, re.Pattern[str]]
_RegexOrSeveralT = Union[_RegexT, Iterable[_RegexT]]
_Regex: TypeAlias = str | re.Pattern[str]
_RegexOrSeveral: TypeAlias = _Regex | Iterable[_Regex]
class LxmlLinkExtractor:
@ -166,8 +166,8 @@ class LxmlLinkExtractor:
def __init__(
self,
allow: _RegexOrSeveralT = (),
deny: _RegexOrSeveralT = (),
allow: _RegexOrSeveral = (),
deny: _RegexOrSeveral = (),
allow_domains: str | Iterable[str] = (),
deny_domains: str | Iterable[str] = (),
restrict_xpaths: str | Iterable[str] = (),
@ -179,7 +179,7 @@ class LxmlLinkExtractor:
deny_extensions: str | Iterable[str] | None = None,
restrict_css: str | Iterable[str] = (),
strip: bool = True,
restrict_text: _RegexOrSeveralT | None = None,
restrict_text: _RegexOrSeveral | None = None,
):
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
self.link_extractor = LxmlParserLinkExtractor(
@ -208,7 +208,7 @@ class LxmlLinkExtractor:
self.restrict_text: list[re.Pattern[str]] = self._compile_regexes(restrict_text)
@staticmethod
def _compile_regexes(value: _RegexOrSeveralT | None) -> list[re.Pattern[str]]:
def _compile_regexes(value: _RegexOrSeveral | None) -> list[re.Pattern[str]]:
return [
x if isinstance(x, re.Pattern) else re.compile(x)
for x in arg_to_iter(value)

8
scrapy/middleware.py
View File

@ -5,7 +5,7 @@ import pprint
import warnings
from abc import ABC, abstractmethod
from collections import defaultdict, deque
from typing import TYPE_CHECKING, Any, TypeVar, cast
from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, cast
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.utils.defer import ensure_awaitable
@ -18,20 +18,18 @@ if TYPE_CHECKING:
from twisted.internet.defer import Deferred
# typing.Concatenate and typing.ParamSpec require Python 3.10
# typing.Self requires Python 3.11
from typing_extensions import Concatenate, ParamSpec, Self
from typing_extensions import Self
from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.settings import BaseSettings, Settings
_P = ParamSpec("_P")
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
_P = ParamSpec("_P")
class MiddlewareManager(ABC):

4
scrapy/pipelines/__init__.py
View File

@ -7,7 +7,7 @@ See documentation in docs/item-pipeline.rst
from __future__ import annotations
import warnings
from typing import TYPE_CHECKING, Any, Callable, cast
from typing import TYPE_CHECKING, Any, cast
from twisted.internet.defer import Deferred, DeferredList
@ -22,7 +22,7 @@ from scrapy.utils.defer import (
from scrapy.utils.python import global_object_name
if TYPE_CHECKING:
from collections.abc import Iterable
from collections.abc import Callable, Iterable
from twisted.python.failure import Failure

2
scrapy/pipelines/images.py
View File

@ -69,7 +69,7 @@ class ImagesPipeline(FilesPipeline):
self._ImageOps = ImageOps
except ImportError:
raise NotConfigured(
"ImagesPipeline requires installing Pillow 8.0.0 or later"
"ImagesPipeline requires installing Pillow 8.3.2 or later"
)
super().__init__(

6
scrapy/pipelines/media.py
View File

@ -5,7 +5,7 @@ import logging
import warnings
from abc import ABC, abstractmethod
from collections import defaultdict
from typing import TYPE_CHECKING, Any, Literal, TypedDict, Union, cast
from typing import TYPE_CHECKING, Any, Literal, TypeAlias, TypedDict, cast
from twisted import version as twisted_version
from twisted.internet.defer import (
@ -47,7 +47,9 @@ class FileInfo(TypedDict):
status: str
FileInfoOrError = Union[tuple[Literal[True], FileInfo], tuple[Literal[False], Failure]]
FileInfoOrError: TypeAlias = (
tuple[Literal[True], FileInfo] | tuple[Literal[False], Failure]
)
logger = logging.getLogger(__name__)

72
scrapy/settings/__init__.py
View File

@ -6,7 +6,7 @@ import warnings
from collections.abc import Iterable, Iterator, Mapping, MutableMapping
from importlib import import_module
from pprint import pformat
from typing import TYPE_CHECKING, Any, Union, cast
from typing import TYPE_CHECKING, Any, TypeAlias, cast
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import default_settings
@ -14,7 +14,7 @@ from scrapy.utils.misc import load_object
# The key types are restricted in BaseSettings._get_key() to ones supported by JSON,
# see https://github.com/scrapy/scrapy/issues/5383.
_SettingsKeyT = Union[bool, float, int, str, None]
_SettingsKey: TypeAlias = bool | float | int | str | None
if TYPE_CHECKING:
from types import ModuleType
@ -25,7 +25,7 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
_SettingsInputT = Union[SupportsItems[_SettingsKeyT, Any], str, None]
_SettingsInput: TypeAlias = SupportsItems[_SettingsKey, Any] | str | None
SETTINGS_PRIORITIES: dict[str, int] = {
@ -76,7 +76,7 @@ class SettingsAttribute:
return f"<SettingsAttribute value={self.value!r} priority={self.priority}>"
class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
class BaseSettings(MutableMapping[_SettingsKey, Any]):
"""
Instances of this class behave like dictionaries, but store priorities
along with their ``(key, value)`` pairs, and can be frozen (i.e. marked
@ -100,13 +100,13 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
__default = object()
def __init__(self, values: _SettingsInputT = None, priority: int | str = "project"):
def __init__(self, values: _SettingsInput = None, priority: int | str = "project"):
self.frozen: bool = False
self.attributes: dict[_SettingsKeyT, SettingsAttribute] = {}
self.attributes: dict[_SettingsKey, SettingsAttribute] = {}
if values:
self.update(values, priority)
def __getitem__(self, opt_name: _SettingsKeyT) -> Any:
def __getitem__(self, opt_name: _SettingsKey) -> Any:
if opt_name not in self:
return None
return self.attributes[opt_name].value
@ -114,7 +114,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
def __contains__(self, name: Any) -> bool:
return name in self.attributes
def add_to_list(self, name: _SettingsKeyT, item: Any) -> None:
def add_to_list(self, name: _SettingsKey, item: Any) -> None:
"""Append *item* to the :class:`list` setting with the specified *name*
if *item* is not already in that list.
@ -125,7 +125,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
if item not in value:
self.set(name, [*value, item], self.getpriority(name) or 0)
def remove_from_list(self, name: _SettingsKeyT, item: Any) -> None:
def remove_from_list(self, name: _SettingsKey, item: Any) -> None:
"""Remove *item* from the :class:`list` setting with the specified
*name*.
@ -139,7 +139,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).")
self.set(name, [v for v in value if v != item], self.getpriority(name) or 0)
def get(self, name: _SettingsKeyT, default: Any = None) -> Any:
def get(self, name: _SettingsKey, default: Any = None) -> Any:
"""
Get a setting value without affecting its original type.
@ -160,7 +160,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
return self[name] if self[name] is not None else default
def getbool(self, name: _SettingsKeyT, default: bool = False) -> bool:
def getbool(self, name: _SettingsKey, default: bool = False) -> bool:
"""
Get a setting value as a boolean.
@ -190,7 +190,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
"'True'/'False' and 'true'/'false'"
)
def getint(self, name: _SettingsKeyT, default: int = 0) -> int:
def getint(self, name: _SettingsKey, default: int = 0) -> int:
"""
Get a setting value as an int.
@ -202,7 +202,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
"""
return int(self.get(name, default))
def getfloat(self, name: _SettingsKeyT, default: float = 0.0) -> float:
def getfloat(self, name: _SettingsKey, default: float = 0.0) -> float:
"""
Get a setting value as a float.
@ -215,7 +215,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
return float(self.get(name, default))
def getlist(
self, name: _SettingsKeyT, default: list[Any] | None = None
self, name: _SettingsKey, default: list[Any] | None = None
) -> list[Any]:
"""
Get a setting value as a list. If the setting original type is a list,
@ -239,7 +239,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
return list(value)
def getdict(
self, name: _SettingsKeyT, default: dict[Any, Any] | None = None
self, name: _SettingsKey, default: dict[Any, Any] | None = None
) -> dict[Any, Any]:
"""
Get a setting value as a dictionary. If the setting original type is a
@ -263,7 +263,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
def getdictorlist(
self,
name: _SettingsKeyT,
name: _SettingsKey,
default: dict[Any, Any] | list[Any] | tuple[Any] | None = None,
) -> dict[Any, Any] | list[Any]:
"""Get a setting value as either a :class:`dict` or a :class:`list`.
@ -310,7 +310,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
)
return copy.deepcopy(value)
def getwithbase(self, name: _SettingsKeyT) -> BaseSettings:
def getwithbase(self, name: _SettingsKey) -> BaseSettings:
"""Get a composition of a dictionary-like setting and its `_BASE`
counterpart.
@ -324,7 +324,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
compbs.update(self[name])
return compbs
def getpriority(self, name: _SettingsKeyT) -> int | None:
def getpriority(self, name: _SettingsKey) -> int | None:
"""
Return the current numerical priority value of a setting, or ``None`` if
the given ``name`` does not exist.
@ -349,7 +349,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
def replace_in_component_priority_dict(
self,
name: _SettingsKeyT,
name: _SettingsKey,
old_cls: type,
new_cls: type,
priority: int | None = None,
@ -388,11 +388,11 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
)
self.set(name, component_priority_dict, priority=self.getpriority(name) or 0)
def __setitem__(self, name: _SettingsKeyT, value: Any) -> None:
def __setitem__(self, name: _SettingsKey, value: Any) -> None:
self.set(name, value)
def set(
self, name: _SettingsKeyT, value: Any, priority: int | str = "project"
self, name: _SettingsKey, value: Any, priority: int | str = "project"
) -> None:
"""
Store a key/value attribute with a given priority.
@ -422,7 +422,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
self.attributes[name].set(value, priority)
def set_in_component_priority_dict(
self, name: _SettingsKeyT, cls: type, priority: int | None
self, name: _SettingsKey, cls: type, priority: int | None
) -> None:
"""Set the *cls* component in the *name* :ref:`component priority
dictionary <component-priority-dictionaries>` setting with *priority*.
@ -447,7 +447,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
def setdefault(
self,
name: _SettingsKeyT,
name: _SettingsKey,
default: Any = None,
priority: int | str = "project",
) -> Any:
@ -458,7 +458,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
return self.attributes[name].value
def setdefault_in_component_priority_dict(
self, name: _SettingsKeyT, cls: type, priority: int | None
self, name: _SettingsKey, cls: type, priority: int | None
) -> None:
"""Set the *cls* component in the *name* :ref:`component priority
dictionary <component-priority-dictionaries>` setting with *priority*
@ -475,7 +475,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
component_priority_dict[cls] = priority
self.set(name, component_priority_dict, self.getpriority(name) or 0)
def setdict(self, values: _SettingsInputT, priority: int | str = "project") -> None:
def setdict(self, values: _SettingsInput, priority: int | str = "project") -> None:
self.update(values, priority)
def setmodule(
@ -503,7 +503,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
self.set(key, getattr(module, key), priority)
# BaseSettings.update() doesn't support all inputs that MutableMapping.update() supports
def update(self, values: _SettingsInputT, priority: int | str = "project") -> None: # type: ignore[override]
def update(self, values: _SettingsInput, priority: int | str = "project") -> None: # type: ignore[override]
"""
Store key/value pairs with a given priority.
@ -527,7 +527,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
"""
self._assert_mutability()
if isinstance(values, str):
values = cast("dict[_SettingsKeyT, Any]", json.loads(values))
values = cast("dict[_SettingsKey, Any]", json.loads(values))
if values is not None:
if isinstance(values, BaseSettings):
for name, value in values.items():
@ -536,7 +536,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
for name, value in values.items():
self.set(name, value, priority)
def delete(self, name: _SettingsKeyT, priority: int | str = "project") -> None:
def delete(self, name: _SettingsKey, priority: int | str = "project") -> None:
if name not in self:
raise KeyError(name)
self._assert_mutability()
@ -544,7 +544,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
if priority >= cast("int", self.getpriority(name)):
del self.attributes[name]
def __delitem__(self, name: _SettingsKeyT) -> None:
def __delitem__(self, name: _SettingsKey) -> None:
self._assert_mutability()
del self.attributes[name]
@ -584,26 +584,26 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
copy.freeze()
return copy
def __iter__(self) -> Iterator[_SettingsKeyT]:
def __iter__(self) -> Iterator[_SettingsKey]:
return iter(self.attributes)
def __len__(self) -> int:
return len(self.attributes)
def _to_dict(self) -> dict[_SettingsKeyT, Any]:
def _to_dict(self) -> dict[_SettingsKey, Any]:
return {
self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
for k, v in self.items()
}
def _get_key(self, key_value: Any) -> _SettingsKeyT:
def _get_key(self, key_value: Any) -> _SettingsKey:
return (
key_value
if isinstance(key_value, (bool, float, int, str, type(None)))
else str(key_value)
)
def copy_to_dict(self) -> dict[_SettingsKeyT, Any]:
def copy_to_dict(self) -> dict[_SettingsKey, Any]:
"""
Make a copy of current settings and convert to a dict.
@ -626,7 +626,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
else:
p.text(pformat(self.copy_to_dict()))
def pop(self, name: _SettingsKeyT, default: Any = __default) -> Any:
def pop(self, name: _SettingsKey, default: Any = __default) -> Any:
try:
value = self.attributes[name].value
except KeyError:
@ -648,7 +648,7 @@ class Settings(BaseSettings):
described on :ref:`topics-settings-ref` already populated.
"""
def __init__(self, values: _SettingsInputT = None, priority: int | str = "project"):
def __init__(self, values: _SettingsInput = None, priority: int | str = "project"):
# Do not pass kwarg values here. We don't want to promote user-defined
# dicts, and we want to update, not replace, default dicts with the
# values given by the user
@ -670,7 +670,7 @@ def iter_default_settings() -> Iterable[tuple[str, Any]]:
def overridden_settings(
settings: Mapping[_SettingsKeyT, Any],
settings: Mapping[_SettingsKey, Any],
) -> Iterable[tuple[str, Any]]:
"""Return an iterable of the settings that have been overridden"""
for name, defvalue in iter_default_settings():

4
scrapy/spiders/__init__.py
View File

@ -26,7 +26,7 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
from scrapy.http.request import CallbackT
from scrapy.settings import BaseSettings, _SettingsKeyT
from scrapy.settings import BaseSettings, _SettingsKey
from scrapy.utils.log import SpiderLoggerAdapter
@ -39,7 +39,7 @@ class Spider(object_ref):
"""
name: str
custom_settings: dict[_SettingsKeyT, Any] | None = None
custom_settings: dict[_SettingsKey, Any] | None = None
#: Start URLs. See :meth:`start`.
start_urls: list[str]

6
scrapy/spiders/crawl.py
View File

@ -10,7 +10,7 @@ from __future__ import annotations
import copy
import warnings
from collections.abc import AsyncIterator, Awaitable, Callable
from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast
from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast
from scrapy.http import HtmlResponse, Request, Response
from scrapy.link import Link
@ -34,8 +34,8 @@ if TYPE_CHECKING:
_T = TypeVar("_T")
ProcessLinksT = Callable[[list[Link]], list[Link]]
ProcessRequestT = Callable[[Request, Response], Optional[Request]]
ProcessLinksT: TypeAlias = Callable[[list[Link]], list[Link]]
ProcessRequestT: TypeAlias = Callable[[Request, Response], Request | None]
def _identity(x: _T) -> _T:

7
scrapy/utils/asyncio.py
View File

@ -6,7 +6,7 @@ import asyncio
import logging
import time
from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
from typing import TYPE_CHECKING, Any, TypeVar
from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar
from twisted.internet.defer import Deferred
from twisted.internet.task import LoopingCall
@ -17,15 +17,14 @@ from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_instal
if TYPE_CHECKING:
from twisted.internet.base import DelayedCall
# typing.Concatenate and typing.ParamSpec require Python 3.10
# typing.Self, typing.TypeVarTuple and typing.Unpack require Python 3.11
from typing_extensions import Concatenate, ParamSpec, Self, TypeVarTuple, Unpack
from typing_extensions import Self, TypeVarTuple, Unpack
_P = ParamSpec("_P")
_Ts = TypeVarTuple("_Ts")
_T = TypeVar("_T")
_P = ParamSpec("_P")
logger = logging.getLogger(__name__)

4
scrapy/utils/conf.py
View File

@ -6,7 +6,7 @@ import sys
from configparser import ConfigParser
from operator import itemgetter
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, cast
from typing import TYPE_CHECKING, Any, cast
from scrapy.exceptions import UsageError
from scrapy.settings import BaseSettings
@ -14,7 +14,7 @@ from scrapy.utils.deprecate import update_classpath
from scrapy.utils.python import without_none_values
if TYPE_CHECKING:
from collections.abc import Collection, Iterable, Mapping, MutableMapping
from collections.abc import Callable, Collection, Iterable, Mapping, MutableMapping
def build_component_list(

8
scrapy/utils/decorators.py
View File

@ -3,7 +3,7 @@ from __future__ import annotations
import inspect
import warnings
from functools import wraps
from typing import TYPE_CHECKING, Any, TypeVar, overload
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, overload
from twisted.internet.defer import Deferred, maybeDeferred
from twisted.internet.threads import deferToThread
@ -13,13 +13,9 @@ from scrapy.exceptions import ScrapyDeprecationWarning
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Callable, Coroutine
# typing.ParamSpec requires Python 3.10
from typing_extensions import ParamSpec
_P = ParamSpec("_P")
_T = TypeVar("_T")
_P = ParamSpec("_P")
def deprecated(

19
scrapy/utils/defer.py
View File

@ -10,7 +10,16 @@ import warnings
from asyncio import Future
from collections.abc import Awaitable, Coroutine, Iterable, Iterator
from functools import wraps
from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast, overload
from typing import (
TYPE_CHECKING,
Any,
Concatenate,
Generic,
ParamSpec,
TypeVar,
cast,
overload,
)
from twisted.internet.defer import Deferred, DeferredList, fail, succeed
from twisted.internet.task import Cooperator
@ -24,14 +33,10 @@ if TYPE_CHECKING:
from twisted.python.failure import Failure
# typing.Concatenate and typing.ParamSpec require Python 3.10
from typing_extensions import Concatenate, ParamSpec
_P = ParamSpec("_P")
_T = TypeVar("_T")
_T2 = TypeVar("_T2")
_P = ParamSpec("_P")
_DEFER_DELAY = 0.1
@ -327,7 +332,7 @@ def process_chain_both(
stacklevel=2,
)
d: Deferred = Deferred()
for cb, eb in zip(callbacks, errbacks):
for cb, eb in zip(callbacks, errbacks, strict=False):
d.addCallback(cb, *a, **kw)
d.addErrback(eb, *a, **kw)
if isinstance(input, failure.Failure):

2
scrapy/utils/iterators.py
View File

@ -212,7 +212,7 @@ def csviter(
},
)
continue
yield dict(zip(headers, row))
yield dict(zip(headers, row, strict=False))
@overload

8
scrapy/utils/log.py
View File

@ -5,14 +5,14 @@ import pprint
import sys
from collections.abc import MutableMapping
from logging.config import dictConfig
from typing import TYPE_CHECKING, Any, Optional, cast
from typing import TYPE_CHECKING, Any, cast
from twisted.internet import asyncioreactor
from twisted.python import log as twisted_log
from twisted.python.failure import Failure
import scrapy
from scrapy.settings import Settings, _SettingsKeyT
from scrapy.settings import Settings, _SettingsKey
from scrapy.utils.versions import get_versions
if TYPE_CHECKING:
@ -35,7 +35,7 @@ def failure_to_exc_info(
return (
failure.type,
failure.value,
cast("Optional[TracebackType]", failure.getTracebackObject()),
cast("TracebackType | None", failure.getTracebackObject()),
)
return None
@ -83,7 +83,7 @@ DEFAULT_LOGGING = {
def configure_logging(
settings: Settings | dict[_SettingsKeyT, Any] | None = None,
settings: Settings | dict[_SettingsKey, Any] | None = None,
install_root_handler: bool = True,
) -> None:
"""

8
scrapy/utils/ossignal.py
View File

@ -3,12 +3,12 @@ from __future__ import annotations
import signal
from collections.abc import Callable
from types import FrameType
from typing import Any, Optional, Union
from typing import Any, TypeAlias
# copy of _HANDLER from typeshed/stdlib/signal.pyi
SignalHandlerT = Union[
Callable[[int, Optional[FrameType]], Any], int, signal.Handlers, None
]
SignalHandlerT: TypeAlias = (
Callable[[int, FrameType | None], Any] | int | signal.Handlers | None
)
signal_names: dict[int, str] = {}
for signame in dir(signal):

9
scrapy/utils/python.py
View File

@ -13,7 +13,7 @@ import weakref
from collections.abc import AsyncIterator, Iterable, Mapping
from functools import partial, wraps
from itertools import chain
from typing import TYPE_CHECKING, Any, TypeVar, overload
from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.asyncgen import as_async_generator
@ -22,15 +22,14 @@ if TYPE_CHECKING:
from collections.abc import Callable, Iterator
from re import Pattern
# typing.Concatenate and typing.ParamSpec require Python 3.10
# typing.Self requires Python 3.11
from typing_extensions import Concatenate, ParamSpec, Self
from typing_extensions import Self
_P = ParamSpec("_P")
_T = TypeVar("_T")
_KT = TypeVar("_KT")
_VT = TypeVar("_VT")
_P = ParamSpec("_P")
def flatten(x: Iterable[Any]) -> list[Any]:
@ -286,7 +285,7 @@ def get_spec(func: Callable[..., Any]) -> tuple[list[str], dict[str, Any]]:
firstdefault = len(spec.args) - len(defaults)
args = spec.args[:firstdefault]
kwargs = dict(zip(spec.args[firstdefault:], defaults))
kwargs = dict(zip(spec.args[firstdefault:], defaults, strict=False))
return args, kwargs

7
scrapy/utils/reactor.py
View File

@ -3,7 +3,7 @@ from __future__ import annotations
import asyncio
import sys
from contextlib import suppress
from typing import TYPE_CHECKING, Any, Generic, TypeVar
from typing import TYPE_CHECKING, Any, Generic, ParamSpec, TypeVar
from warnings import catch_warnings, filterwarnings
from twisted.internet import asyncioreactor, error
@ -19,14 +19,11 @@ if TYPE_CHECKING:
from twisted.internet.protocol import ServerFactory
from twisted.internet.tcp import Port
# typing.ParamSpec requires Python 3.10
from typing_extensions import ParamSpec
from scrapy.utils.asyncio import CallLaterResult
_P = ParamSpec("_P")
_T = TypeVar("_T")
_P = ParamSpec("_P")
def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # pylint: disable=inconsistent-return-statements # noqa: RET503

4
scrapy/utils/url.py
View File

@ -8,7 +8,7 @@ from __future__ import annotations
import re
import warnings
from importlib import import_module
from typing import TYPE_CHECKING, Union
from typing import TYPE_CHECKING, TypeAlias
from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
from warnings import warn
@ -37,7 +37,7 @@ if TYPE_CHECKING:
from scrapy import Spider
UrlT = Union[str, bytes, ParseResult]
UrlT: TypeAlias = str | bytes | ParseResult
def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:

2
tests/test_crawl.py
View File

@ -369,7 +369,7 @@ with multiples lines
est = [x for sublist in est for x in sublist] # flatten
est = [x.lstrip().rstrip() for x in est]
it = iter(est)
s = dict(zip(it, it))
s = dict(zip(it, it, strict=False))
assert s["engine.spider.name"] == crawler.spider.name
assert s["len(engine.scraper.slot.active)"] == "1"

6
tests/test_extension_periodic_log.py
View File

@ -1,13 +1,17 @@
from __future__ import annotations
import datetime
from typing import Any, Callable
from typing import TYPE_CHECKING, Any
from scrapy.extensions.periodic_log import PeriodicLog
from scrapy.utils.test import get_crawler
from .spiders import MetaSpider
if TYPE_CHECKING:
from collections.abc import Callable
stats_dump_1 = {
"log_count/INFO": 10,
"log_count/WARNING": 1,

4
tests/test_feedexport.py
View File

@ -2628,7 +2628,7 @@ class TestBatchDeliveries(TestFeedExportBase):
}
data = await self.exported_data(items, settings)
for fmt, expected in formats.items():
for expected_batch, got_batch in zip(expected, data[fmt]):
for expected_batch, got_batch in zip(expected, data[fmt], strict=False):
assert got_batch == expected_batch
@deferred_f_from_coro_f
@ -2652,7 +2652,7 @@ class TestBatchDeliveries(TestFeedExportBase):
}
data = await self.exported_data(items, settings)
for fmt, expected in formats.items():
for expected_batch, got_batch in zip(expected, data[fmt]):
for expected_batch, got_batch in zip(expected, data[fmt], strict=False):
assert got_batch == expected_batch
@deferred_f_from_coro_f

4
tests/test_http2_client_protocol.py
View File

@ -6,7 +6,7 @@ import re
import string
from ipaddress import IPv4Address
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, cast
from typing import TYPE_CHECKING, Any, cast
from unittest import mock
from urllib.parse import urlencode
@ -39,7 +39,7 @@ from tests.mockserver.http_resources import LeafResource, Status
from tests.mockserver.utils import ssl_context_factory
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Coroutine, Generator
from collections.abc import AsyncGenerator, Callable, Coroutine, Generator
from scrapy.core.http2.protocol import H2ClientProtocol

6
tests/test_http_response.py
View File

@ -333,7 +333,7 @@ class TestResponseBase:
if response is None:
response = self._links_response()
followed = response.follow_all(follow_obj)
for req, target in zip(followed, target_urls):
for req, target in zip(followed, target_urls, strict=False):
assert req.url == target
yield req
@ -647,7 +647,7 @@ class TestTextResponse(TestResponseBase):
# select <a> elements
for sellist in [resp.css("a"), resp.xpath("//a")]:
for sel, url in zip(sellist, urls):
for sel, url in zip(sellist, urls, strict=False):
self._assert_followed_url(sel, url, response=resp)
# select <link> elements
@ -659,7 +659,7 @@ class TestTextResponse(TestResponseBase):
# href attributes should work
for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]:
for sel, url in zip(sellist, urls):
for sel, url in zip(sellist, urls, strict=False):
self._assert_followed_url(sel, url, response=resp)
# non-a elements are not supported

2
tests_typing/test_http_request.mypy-testing
View File

@ -16,7 +16,7 @@ class MyRequest2(Request):
@pytest.mark.mypy_testing
def mypy_test_headers():
Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[tuple[str, Any]], None]"
Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
Request("data:,", headers=None)
Request("data:,", headers={})
Request("data:,", headers=[])

2
tests_typing/test_http_response.mypy-testing
View File

@ -7,7 +7,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse
@pytest.mark.mypy_testing
def mypy_test_headers():
Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[tuple[str, Any]], None]"
Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
Response("data:,", headers=None)
Response("data:,", headers={})
Response("data:,", headers=[])

20
tox.ini
View File

@ -43,7 +43,7 @@ commands =
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 scrapy tests --doctest-modules}
[testenv:typing]
basepython = python3.9
basepython = python3.10
deps =
mypy==1.16.1
typing-extensions==4.14.1
@ -64,7 +64,7 @@ commands =
mypy {posargs:scrapy tests}
[testenv:typing-tests]
basepython = python3.9
basepython = python3.10
deps =
{[test-requirements]deps}
{[testenv:typing]deps}
@ -97,7 +97,7 @@ commands =
twine check dist/*
[pinned]
basepython = python3.9
basepython = python3.10
deps =
# pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0
pytest==8.4.0
@ -106,7 +106,7 @@ deps =
cryptography==37.0.0
cssselect==0.9.1
itemadapter==0.1.0
lxml==4.6.0
lxml==4.6.4
parsel==1.5.0
pyOpenSSL==22.0.0
queuelib==1.4.2
@ -151,17 +151,17 @@ deps =
basepython = {[pinned]basepython}
deps =
{[pinned]deps}
Pillow==8.0.0
Pillow==8.3.2
boto3==1.20.0
bpython==0.7.1
brotli==0.5.2; implementation_name != "pypy"
brotlicffi==0.8.0; implementation_name == "pypy"
brotlipy
google-cloud-storage==1.29.0
ipython==2.0.0
ipython==7.1.0
robotexclusionrulesparser==1.6.2
uvloop==0.14.0; platform_system != "Windows" and implementation_name != "pypy"
zstandard==0.1; implementation_name != "pypy"
uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"
zstandard==0.16.0; implementation_name != "pypy"
setenv =
{[pinned]setenv}
commands = {[pinned]commands}
@ -254,7 +254,7 @@ commands =
[testenv:botocore]
deps =
{[testenv]deps}
botocore>=1.4.87
botocore>=1.13.45
commands =
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy -m requires_botocore}
@ -262,7 +262,7 @@ commands =
basepython = {[pinned]basepython}
deps =
{[pinned]deps}
botocore==1.4.87
botocore==1.13.45
setenv =
{[pinned]setenv}
commands =