Merge pull request #6595 from wRAR/update-tools

Update tool versions

.github/workflows/checks.yml

@@ -21,7 +21,7 @@ jobs:
         - python-version: "3.9"
           env:
             TOXENV: typing-tests
-        - python-version: "3.12"  # Keep in sync with .readthedocs.yml
+        - python-version: "3.13"  # Keep in sync with .readthedocs.yml
           env:
             TOXENV: docs
         - python-version: "3.13"

.pre-commit-config.yaml

@@ -1,20 +1,16 @@
 repos:
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.8.1
+  rev: v0.8.4
   hooks:
     - id: ruff
       args: [ --fix ]
 - repo: https://github.com/psf/black.git
-  rev: 24.4.2
+  rev: 24.10.0
   hooks:
   - id: black
-- repo: https://github.com/pycqa/isort
-  rev: 5.13.2
-  hooks:
-  - id: isort
 - repo: https://github.com/adamchainz/blacken-docs
-  rev: 1.18.0
+  rev: 1.19.1
   hooks:
   - id: blacken-docs
     additional_dependencies:
-    - black==24.4.2
+    - black==24.10.0

.readthedocs.yml

@@ -9,7 +9,7 @@ build:
   tools:
     # For available versions, see:
     # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
-    python: "3.12"  # Keep in sync with .github/workflows/checks.yml
+    python: "3.13"  # Keep in sync with .github/workflows/checks.yml
 
 python:
   install:

pyproject.toml

@@ -116,9 +116,6 @@ disable_warnings = ["include-ignored"]
 # https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185
 exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"]
 
-[tool.isort]
-profile = "black"
-
 [tool.pylint.MASTER]
 persistent = "no"
 jobs = 1          # >1 hides results
@@ -173,6 +170,7 @@ disable = [
     "too-many-instance-attributes",
     "too-many-lines",
     "too-many-locals",
+    "too-many-positional-arguments",
     "too-many-public-methods",
     "too-many-return-statements",
     "unbalanced-tuple-unpacking",
@@ -226,6 +224,8 @@ extend-select = [
     "FA",
     # refurb
     "FURB",
+    # isort
+    "I",
     # flake8-implicit-str-concat
     "ISC",
     # flake8-logging

scrapy/core/downloader/handlers/http11.py

@@ -17,9 +17,14 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
 from twisted.internet.error import TimeoutError
 from twisted.internet.protocol import Factory, Protocol, connectionDone
 from twisted.python.failure import Failure
-from twisted.web.client import URI, Agent, HTTPConnectionPool
+from twisted.web.client import (
+    URI,
+    Agent,
+    HTTPConnectionPool,
+    ResponseDone,
+    ResponseFailed,
+)
 from twisted.web.client import Response as TxResponse
-from twisted.web.client import ResponseDone, ResponseFailed
 from twisted.web.http import PotentialDataLoss, _DataLoss
 from twisted.web.http_headers import Headers as TxHeaders
 from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS

scrapy/downloadermiddlewares/cookies.py

@@ -131,25 +131,26 @@ class CookiesMiddleware:
         decoded = {}
         flags = set()
         for key in ("name", "value", "path", "domain"):
-            if cookie.get(key) is None:
+            value = cookie.get(key)
+            if value is None:
                 if key in ("name", "value"):
                     msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
                     logger.warning(msg)
                     return None
                 continue
-            # https://github.com/python/mypy/issues/7178, https://github.com/python/mypy/issues/9168
+            if isinstance(value, (bool, float, int, str)):
-            if isinstance(cookie[key], (bool, float, int, str)):  # type: ignore[literal-required]
+                decoded[key] = str(value)
-                decoded[key] = str(cookie[key])  # type: ignore[literal-required]
             else:
+                assert isinstance(value, bytes)
                 try:
-                    decoded[key] = cookie[key].decode("utf8")  # type: ignore[literal-required]
+                    decoded[key] = value.decode("utf8")
                 except UnicodeDecodeError:
                     logger.warning(
                         "Non UTF-8 encoded cookie found in request %s: %s",
                         request,
                         cookie,
                     )
-                    decoded[key] = cookie[key].decode("latin1", errors="replace")  # type: ignore[literal-required]
+                    decoded[key] = value.decode("latin1", errors="replace")
         for flag in ("secure",):
             value = cookie.get(flag, _UNSET)
             if value is _UNSET or not value:

scrapy/extensions/telnet.py

@@ -75,7 +75,7 @@ class TelnetConsole(protocol.ServerFactory):
     def stop_listening(self) -> None:
         self.port.stopListening()
 
-    def protocol(self) -> telnet.TelnetTransport:  # type: ignore[override]
+    def protocol(self) -> telnet.TelnetTransport:
         # these import twisted.internet.reactor
         from twisted.conch import manhole, telnet
         from twisted.conch.insults import insults

scrapy/http/cookies.py

@@ -2,9 +2,8 @@ from __future__ import annotations
 
 import re
 import time
-from http.cookiejar import Cookie
+from http.cookiejar import Cookie, CookiePolicy, DefaultCookiePolicy
 from http.cookiejar import CookieJar as _CookieJar
-from http.cookiejar import CookiePolicy, DefaultCookiePolicy
 from typing import TYPE_CHECKING, Any, cast
 
 from scrapy.utils.httpobj import urlparse_cached

scrapy/http/request/__init__.py

@@ -44,10 +44,10 @@ if TYPE_CHECKING:
 
 
 class VerboseCookie(TypedDict):
-    name: str
+    name: str | bytes
-    value: str
+    value: str | bytes | bool | float | int
-    domain: NotRequired[str]
+    domain: NotRequired[str | bytes]
-    path: NotRequired[str]
+    path: NotRequired[str | bytes]
     secure: NotRequired[bool]
 
 

scrapy/utils/sitemap.py

@@ -26,13 +26,15 @@ class Sitemap:
         )
         self._root = lxml.etree.fromstring(xmltext, parser=xmlp)  # noqa: S320
         rt = self._root.tag
-        self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt
+        assert isinstance(rt, str)
+        self.type = rt.split("}", 1)[1] if "}" in rt else rt
 
     def __iter__(self) -> Iterator[dict[str, Any]]:
         for elem in self._root.getchildren():
             d: dict[str, Any] = {}
             for el in elem.getchildren():
                 tag = el.tag
+                assert isinstance(tag, str)
                 name = tag.split("}", 1)[1] if "}" in tag else tag
 
                 if name == "link":

tests/CrawlerRunner/ip_address.py

@@ -1,9 +1,8 @@
 from urllib.parse import urlparse
 
 from twisted.internet import reactor
-from twisted.names import cache
+from twisted.names import cache, resolve
 from twisted.names import hosts as hostsModule
-from twisted.names import resolve
 from twisted.names.client import Resolver
 from twisted.python.runtime import platform
 

tox.ini

@@ -43,12 +43,12 @@ install_command =
 [testenv:typing]
 basepython = python3
 deps =
-    mypy==1.12.0
+    mypy==1.14.0
     typing-extensions==4.12.2
-    types-lxml==2024.9.16
+    types-lxml==2024.12.13
     types-Pygments==2.18.0.20240506
-    botocore-stubs==1.35.39
+    botocore-stubs==1.35.90
-    boto3-stubs[s3]==1.35.39
+    boto3-stubs[s3]==1.35.90
     attrs >= 18.2.0
     Pillow >= 10.3.0
     pyOpenSSL >= 24.2.1
@@ -77,15 +77,15 @@ commands =
 basepython = python3
 deps =
     {[testenv:extra-deps]deps}
-    pylint==3.2.5
+    pylint==3.3.3
 commands =
     pylint conftest.py docs extras scrapy tests
 
 [testenv:twinecheck]
 basepython = python3
 deps =
-    twine==5.1.1
+    twine==6.0.1
-    build==1.2.1
+    build==1.2.2.post1
 commands =
     python -m build --sdist
     twine check dist/*