From cc484efd43b0f8ba0dc89904a7d38086775c44ae Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 29 Dec 2024 14:15:16 +0500 Subject: [PATCH 1/3] Replace isort with the ruff isort rules. --- .pre-commit-config.yaml | 4 ---- pyproject.toml | 5 ++--- scrapy/core/downloader/handlers/http11.py | 9 +++++++-- scrapy/http/cookies.py | 3 +-- tests/CrawlerRunner/ip_address.py | 3 +-- 5 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 39b9a33aa..d253f61c6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,10 +8,6 @@ repos: rev: 24.4.2 hooks: - id: black -- repo: https://github.com/pycqa/isort - rev: 5.13.2 - hooks: - - id: isort - repo: https://github.com/adamchainz/blacken-docs rev: 1.18.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 7dc1f6ec3..a2dabcf4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,9 +116,6 @@ disable_warnings = ["include-ignored"] # https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185 exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"] -[tool.isort] -profile = "black" - [tool.pylint.MASTER] persistent = "no" jobs = 1 # >1 hides results @@ -226,6 +223,8 @@ extend-select = [ "FA", # refurb "FURB", + # isort + "I", # flake8-implicit-str-concat "ISC", # flake8-logging diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index bd3200e9f..9f65794fe 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -17,9 +17,14 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError from twisted.internet.protocol import Factory, Protocol, connectionDone from twisted.python.failure import Failure -from twisted.web.client import URI, Agent, HTTPConnectionPool +from twisted.web.client import ( + URI, + Agent, + HTTPConnectionPool, + ResponseDone, + ResponseFailed, +) from twisted.web.client import Response as TxResponse -from twisted.web.client import ResponseDone, ResponseFailed from twisted.web.http import PotentialDataLoss, _DataLoss from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 56941ad51..60322fe6e 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -2,9 +2,8 @@ from __future__ import annotations import re import time -from http.cookiejar import Cookie +from http.cookiejar import Cookie, CookiePolicy, DefaultCookiePolicy from http.cookiejar import CookieJar as _CookieJar -from http.cookiejar import CookiePolicy, DefaultCookiePolicy from typing import TYPE_CHECKING, Any, cast from scrapy.utils.httpobj import urlparse_cached diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 5bf7512bc..2f1bb7713 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -1,9 +1,8 @@ from urllib.parse import urlparse from twisted.internet import reactor -from twisted.names import cache +from twisted.names import cache, resolve from twisted.names import hosts as hostsModule -from twisted.names import resolve from twisted.names.client import Resolver from twisted.python.runtime import platform From 4a0c05749c72662bb82cdb01d9a0ef1ff6416b6a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 29 Dec 2024 14:29:27 +0500 Subject: [PATCH 2/3] Bump tool versions. --- .github/workflows/checks.yml | 2 +- .pre-commit-config.yaml | 8 ++++---- .readthedocs.yml | 2 +- pyproject.toml | 1 + tox.ini | 6 +++--- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ff279e9fd..a064bf5b2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -21,7 +21,7 @@ jobs: - python-version: "3.9" env: TOXENV: typing-tests - - python-version: "3.12" # Keep in sync with .readthedocs.yml + - python-version: "3.13" # Keep in sync with .readthedocs.yml env: TOXENV: docs - python-version: "3.13" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d253f61c6..c76c613d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,16 +1,16 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.1 + rev: v0.8.4 hooks: - id: ruff args: [ --fix ] - repo: https://github.com/psf/black.git - rev: 24.4.2 + rev: 24.10.0 hooks: - id: black - repo: https://github.com/adamchainz/blacken-docs - rev: 1.18.0 + rev: 1.19.1 hooks: - id: blacken-docs additional_dependencies: - - black==24.4.2 + - black==24.10.0 diff --git a/.readthedocs.yml b/.readthedocs.yml index 0c544df7e..5ec6eafbb 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ build: tools: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.12" # Keep in sync with .github/workflows/checks.yml + python: "3.13" # Keep in sync with .github/workflows/checks.yml python: install: diff --git a/pyproject.toml b/pyproject.toml index a2dabcf4b..ad85e5c75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -170,6 +170,7 @@ disable = [ "too-many-instance-attributes", "too-many-lines", "too-many-locals", + "too-many-positional-arguments", "too-many-public-methods", "too-many-return-statements", "unbalanced-tuple-unpacking", diff --git a/tox.ini b/tox.ini index 4e1a99473..24b674085 100644 --- a/tox.ini +++ b/tox.ini @@ -77,15 +77,15 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==3.2.5 + pylint==3.3.3 commands = pylint conftest.py docs extras scrapy tests [testenv:twinecheck] basepython = python3 deps = - twine==5.1.1 - build==1.2.1 + twine==6.0.1 + build==1.2.2.post1 commands = python -m build --sdist twine check dist/* From f7af7b282d6d3b36689cc192e6f78c065e32fb89 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 29 Dec 2024 16:45:26 +0500 Subject: [PATCH 3/3] Bump mypy and stubs. --- scrapy/downloadermiddlewares/cookies.py | 13 +++++++------ scrapy/extensions/telnet.py | 2 +- scrapy/http/request/__init__.py | 8 ++++---- scrapy/utils/sitemap.py | 4 +++- tox.ini | 8 ++++---- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 545dcaac9..43348f632 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -131,25 +131,26 @@ class CookiesMiddleware: decoded = {} flags = set() for key in ("name", "value", "path", "domain"): - if cookie.get(key) is None: + value = cookie.get(key) + if value is None: if key in ("name", "value"): msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)" logger.warning(msg) return None continue - # https://github.com/python/mypy/issues/7178, https://github.com/python/mypy/issues/9168 - if isinstance(cookie[key], (bool, float, int, str)): # type: ignore[literal-required] - decoded[key] = str(cookie[key]) # type: ignore[literal-required] + if isinstance(value, (bool, float, int, str)): + decoded[key] = str(value) else: + assert isinstance(value, bytes) try: - decoded[key] = cookie[key].decode("utf8") # type: ignore[literal-required] + decoded[key] = value.decode("utf8") except UnicodeDecodeError: logger.warning( "Non UTF-8 encoded cookie found in request %s: %s", request, cookie, ) - decoded[key] = cookie[key].decode("latin1", errors="replace") # type: ignore[literal-required] + decoded[key] = value.decode("latin1", errors="replace") for flag in ("secure",): value = cookie.get(flag, _UNSET) if value is _UNSET or not value: diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 89c83d20d..ee28d86ba 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -75,7 +75,7 @@ class TelnetConsole(protocol.ServerFactory): def stop_listening(self) -> None: self.port.stopListening() - def protocol(self) -> telnet.TelnetTransport: # type: ignore[override] + def protocol(self) -> telnet.TelnetTransport: # these import twisted.internet.reactor from twisted.conch import manhole, telnet from twisted.conch.insults import insults diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a96a215f4..3d6cf4816 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -44,10 +44,10 @@ if TYPE_CHECKING: class VerboseCookie(TypedDict): - name: str - value: str - domain: NotRequired[str] - path: NotRequired[str] + name: str | bytes + value: str | bytes | bool | float | int + domain: NotRequired[str | bytes] + path: NotRequired[str | bytes] secure: NotRequired[bool] diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index b60fe929e..e0d9f4595 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -26,13 +26,15 @@ class Sitemap: ) self._root = lxml.etree.fromstring(xmltext, parser=xmlp) # noqa: S320 rt = self._root.tag - self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt + assert isinstance(rt, str) + self.type = rt.split("}", 1)[1] if "}" in rt else rt def __iter__(self) -> Iterator[dict[str, Any]]: for elem in self._root.getchildren(): d: dict[str, Any] = {} for el in elem.getchildren(): tag = el.tag + assert isinstance(tag, str) name = tag.split("}", 1)[1] if "}" in tag else tag if name == "link": diff --git a/tox.ini b/tox.ini index 24b674085..39ab1ccd4 100644 --- a/tox.ini +++ b/tox.ini @@ -43,12 +43,12 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.12.0 + mypy==1.14.0 typing-extensions==4.12.2 - types-lxml==2024.9.16 + types-lxml==2024.12.13 types-Pygments==2.18.0.20240506 - botocore-stubs==1.35.39 - boto3-stubs[s3]==1.35.39 + botocore-stubs==1.35.90 + boto3-stubs[s3]==1.35.90 attrs >= 18.2.0 Pillow >= 10.3.0 pyOpenSSL >= 24.2.1