Update tool versions (#7127)

.git-blame-ignore-revs

@@ -4,4 +4,4 @@ e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d
 # reapplying black to the code with default line length
 303f0a70fcf8067adf0a909c2096a5009162383a
 # reapplying black again and removing line length on pre-commit black config
-c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962
+c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962

.github/workflows/checks.yml

@@ -34,10 +34,10 @@ jobs:
             TOXENV: twinecheck
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -50,5 +50,5 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
     - uses: pre-commit/action@v3.0.1

.github/workflows/publish.yml

@@ -18,8 +18,8 @@ jobs:
     permissions:
       id-token: write
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@v6
         with:
           python-version: "3.13"
       - run: |

.github/workflows/tests-macos.yml

@@ -19,10 +19,10 @@ jobs:
         python-version: ["3.10", "3.11", "3.12", "3.13"]
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/tests-ubuntu.yml

@@ -64,10 +64,10 @@ jobs:
             TOXENV: botocore
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/tests-windows.yml

@@ -46,10 +46,10 @@ jobs:
             TOXENV: extra-deps
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
 

.pre-commit-config.yaml

@@ -1,19 +1,26 @@
+exclude: |
+  (?x)(
+      ^docs/_static|
+      ^docs/_tests|
+      ^tests/sample_data
+  )
 repos:
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.12.2
+  rev: v0.14.2
   hooks:
     - id: ruff-check
       args: [ --fix ]
     - id: ruff-format
 - repo: https://github.com/adamchainz/blacken-docs
-  rev: 1.19.1
+  rev: 1.20.0
   hooks:
   - id: blacken-docs
     additional_dependencies:
-    - black==25.1.0
+    - black==25.9.0
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v5.0.0
+  rev: v6.0.0
   hooks:
+  - id: end-of-file-fixer
   - id: trailing-whitespace
 - repo: https://github.com/sphinx-contrib/sphinx-lint
   rev: v1.0.0

docs/_ext/scrapydocs.py

@@ -29,14 +29,14 @@ def is_setting_index(node: Node) -> bool:
     if node.tagname == "index" and node["entries"]:  # type: ignore[index,attr-defined]
         # index entries for setting directives look like:
         # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
-        entry_type, info, refid = node["entries"][0][:3]  # type: ignore[index]
+        entry_type, info, _ = node["entries"][0][:3]  # type: ignore[index]
         return entry_type == "pair" and info.endswith("; setting")
     return False
 
 
 def get_setting_name_and_refid(node: Node) -> tuple[str, str]:
     """Extract setting name from directive index node"""
-    entry_type, info, refid = node["entries"][0][:3]  # type: ignore[index]
+    _, info, refid = node["entries"][0][:3]  # type: ignore[index]
     return info.replace("; setting", ""), refid
 
 

docs/topics/developer-tools.rst

@@ -317,4 +317,3 @@ to identifying the correct request and replicating it in your spider.
 .. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll
 .. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10
 .. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions
-

docs/topics/stats.rst

@@ -121,4 +121,3 @@ DummyStatsCollector
     setting, to disable stats collect in order to improve performance. However,
     the performance penalty of stats collection is usually marginal compared to
     other Scrapy workload like parsing pages.
-

docs/versioning.rst

@@ -66,4 +66,3 @@ the :ref:`release notes <news>`.
 
 
 .. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases
-

pyproject.toml

@@ -374,8 +374,6 @@ ignore = [
     "SIM115",
     # Yoda condition detected
     "SIM300",
-    # removed in recent ruff
-    "UP038",
 
     # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")
 

scrapy/utils/spider.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import inspect
 import logging
-from typing import TYPE_CHECKING, Any, Literal, TypeVar, overload
+from typing import TYPE_CHECKING, Any, TypeVar, overload
 
 from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
@@ -76,7 +76,7 @@ def spidercls_for_request(
 def spidercls_for_request(
     spider_loader: SpiderLoaderProtocol,
     request: Request,
-    default_spidercls: Literal[None],
+    default_spidercls: None,
     log_none: bool = ...,
     log_multiple: bool = ...,
 ) -> type[Spider] | None: ...

tests/keys/example-com.gen.README

@@ -21,4 +21,3 @@ Locality Name (eg, city) [New York]:The Internet
 Organization Name (eg, company) [Example, LLC]:Scrapy
 Common Name (e.g. server FQDN or YOUR name) [Example Company]:www.example.com
 Email Address [test@example.com]:
-

tests/keys/localhost-ip.gen.README

@@ -18,4 +18,3 @@ Organization Name (eg, company) [Internet Widgits Pty Ltd]:Scrapy
 Organizational Unit Name (eg, section) []:.
 Common Name (e.g. server FQDN or YOUR name) []:127.0.0.1
 Email Address []:.
-

tests/keys/localhost.gen.README

@@ -18,4 +18,3 @@ Organization Name (eg, company) [Internet Widgits Pty Ltd]:Scrapy
 Organizational Unit Name (eg, section) []:.
 Common Name (e.g. server FQDN or YOUR name) []:localhost
 Email Address []:.
-

tests/test_cmdline_crawl_with_pipeline/__init__.py

@@ -14,7 +14,7 @@ class TestCmdlineCrawlPipeline:
         return proc.returncode, stderr
 
     def test_open_spider_normally_in_pipeline(self):
-        returncode, stderr = self._execute("normal")
+        returncode, _ = self._execute("normal")
         assert returncode == 0
 
     def test_exception_at_open_spider_in_pipeline(self):

tests/test_command_genspider.py

@@ -172,7 +172,7 @@ class TestGenspiderStandaloneCommand:
     def test_same_name_as_existing_file(self, force: bool, tmp_path: Path) -> None:
         file_name = "example"
         file_path = Path(tmp_path, file_name + ".py")
-        p, out, err = proc("genspider", file_name, "example.com", cwd=tmp_path)
+        _, out, _ = proc("genspider", file_name, "example.com", cwd=tmp_path)
         assert f"Created spider {file_name!r} using template 'basic' " in out
         assert file_path.exists()
         modify_time_before = file_path.stat().st_mtime
@@ -180,7 +180,7 @@ class TestGenspiderStandaloneCommand:
 
         if force:
             # use different template to ensure contents were changed
-            p, out, err = proc(
+            _, out, _ = proc(
                 "genspider",
                 "--force",
                 "-t",

tests/test_command_startproject.py

@@ -232,7 +232,7 @@ class TestStartprojectTemplates:
         project_dir.mkdir(parents=True)
 
         existing_nodes = {
-            oct(permissions)[2:] + extension: permissions
+            f"{permissions:o}{extension}": permissions
             for extension in ("", ".d")
             for permissions in (
                 0o444,

tests/test_crawl.py

@@ -612,7 +612,7 @@ class TestCrawlSpider:
     @pytest.mark.only_asyncio
     @deferred_f_from_coro_f
     async def test_async_def_deferred_wrapped(self):
-        log, items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider)
+        _, items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider)
         assert items == [{"code": 200}]
 
     @deferred_f_from_coro_f

tests/test_crawler.py

@@ -785,7 +785,7 @@ class ScriptRunnerMixin(ABC):
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )
-        stdout, stderr = p.communicate()
+        _, stderr = p.communicate()
         return stderr.decode("utf-8")
 
 

tests/test_extension_statsmailer.py

@@ -74,7 +74,7 @@ def test_spider_closed_sends_email(dummy_stats):
     spider = DefaultSpider(name="dummy")
     ext.spider_closed(spider)
 
-    args, kwargs = mail.send.call_args
+    args, _ = mail.send.call_args
     to, subject, body = args
     assert to == recipients
     assert "Scrapy stats for: dummy" in subject

tests/test_http_headers.py

@@ -56,12 +56,12 @@ class TestHeaders:
 
     def test_encode_latin1(self):
         h = Headers({"key": "\xa3"}, encoding="latin1")
-        key, val = dict(h).popitem()
+        _, val = dict(h).popitem()
         assert val[0] == b"\xa3"
 
     def test_encode_multiple(self):
         h = Headers({"key": ["\xa3"]}, encoding="utf-8")
-        key, val = dict(h).popitem()
+        _, val = dict(h).popitem()
         assert val[0] == b"\xc2\xa3"
 
     def test_delete_and_contains(self):

tests/test_http_request.py

@@ -881,7 +881,7 @@ class TestFormRequest(TestRequest):
         )
         with pytest.raises(
             ValueError,
-            match="Multiple elements found .* matching the criteria in clickdata",
+            match=r"Multiple elements found .* matching the criteria in clickdata",
         ):
             self.request_class.from_response(response, clickdata={"type": "submit"})
 

tests/test_http_response.py

@@ -813,7 +813,7 @@ class TestTextResponse(TestResponseBase):
         text_body = b"""<html><body>text</body></html>"""
         text_response = self.response_class("http://www.example.com", body=text_body)
         with pytest.raises(
-            ValueError, match="(Expecting value|Unexpected '<'): line 1"
+            ValueError, match=r"(Expecting value|Unexpected '<'): line 1"
         ):
             text_response.json()
 

tests/test_item.py

@@ -284,9 +284,9 @@ class TestItemMeta:
 
         (first_call, second_call) = new_mock.call_args_list[-2:]
 
-        mcs, class_name, bases, attrs = first_call[0]
+        *_, attrs = first_call[0]
         assert "__classcell__" not in attrs
-        mcs, class_name, bases, attrs = second_call[0]
+        *_, attrs = second_call[0]
         assert "__classcell__" in attrs
 
 

tests/test_pipeline_images.py

@@ -159,7 +159,7 @@ class TestImagesPipeline:
         self.pipeline.thumbs = {"small": (20, 20)}
 
         orig_im, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
-        orig_thumb, orig_thumb_buf = _create_image("JPEG", "RGB", (20, 20), (0, 0, 0))
+        _, orig_thumb_buf = _create_image("JPEG", "RGB", (20, 20), (0, 0, 0))
         resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue())
         req = Request(url="https://dev.mydeco.com/mydeco.gif")
 
@@ -172,7 +172,7 @@ class TestImagesPipeline:
         assert orig_im.copy() == new_im
         assert buf.getvalue() == new_buf.getvalue()
 
-        thumb_path, thumb_img, thumb_buf = next(get_images_gen)
+        thumb_path, _, thumb_buf = next(get_images_gen)
         assert thumb_path == "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
         assert orig_thumb_buf.getvalue() == thumb_buf.getvalue()
 

tests/test_pipelines.py

@@ -371,13 +371,13 @@ class TestMiddlewareManagerSpider:
             ),
             pytest.raises(
                 ValueError,
-                match="ItemPipelineManager needs to access self.crawler.spider but it is None",
+                match=r"ItemPipelineManager needs to access self\.crawler\.spider but it is None",
             ),
         ):
             mwman.open_spider(DefaultSpider())
         with pytest.raises(
             ValueError,
-            match="ItemPipelineManager needs to access self.crawler.spider but it is None",
+            match=r"ItemPipelineManager needs to access self\.crawler\.spider but it is None",
         ):
             await mwman.open_spider_async()
         with (
@@ -387,13 +387,13 @@ class TestMiddlewareManagerSpider:
             ),
             pytest.raises(
                 ValueError,
-                match="ItemPipelineManager needs to access self.crawler.spider but it is None",
+                match=r"ItemPipelineManager needs to access self\.crawler\.spider but it is None",
             ),
         ):
             mwman.close_spider(DefaultSpider())
         with pytest.raises(
             ValueError,
-            match="ItemPipelineManager needs to access self.crawler.spider but it is None",
+            match=r"ItemPipelineManager needs to access self\.crawler\.spider but it is None",
         ):
             await mwman.close_spider_async()
 

tests/test_scheduler_base.py

@@ -35,7 +35,7 @@ class MinimalScheduler:
 
     def next_request(self) -> Request | None:
         if self.has_pending_requests():
-            fp, request = self.requests.popitem()
+            _, request = self.requests.popitem()
             return request
         return None
 

tests/test_settings/__init__.py

@@ -311,7 +311,7 @@ class TestBaseSettings:
         assert settings.getdict("TEST_DICT3", {"key1": 5}) == {"key1": 5}
         with pytest.raises(
             ValueError,
-            match="dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected",
+            match=r"dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected",
         ):
             settings.getdict("TEST_LIST1")
         with pytest.raises(

tests/test_spidermiddleware.py

@@ -568,7 +568,8 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
 
     async def _test_asyncgen_nodowngrade(self, *mw_classes: type[Any]) -> None:
         with pytest.raises(
-            _InvalidOutput, match="Async iterable returned from .+ cannot be downgraded"
+            _InvalidOutput,
+            match=r"Async iterable returned from .+ cannot be downgraded",
         ):
             await self._get_middleware_result(*mw_classes)
 

tests/test_squeues.py

@@ -33,13 +33,13 @@ def nonserializable_object_test(self):
     q = self.queue()
     with pytest.raises(
         ValueError,
-        match="unmarshallable object|Can't (get|pickle) local object|Can't pickle .*: it's not found as",
+        match=r"unmarshallable object|Can't (get|pickle) local object|Can't pickle .*: it's not found as",
     ):
         q.push(lambda x: x)
     # Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
     sel = Selector(text="<html><body><p>some text</p></body></html>")
     with pytest.raises(
-        ValueError, match="unmarshallable object|can't pickle Selector objects"
+        ValueError, match=r"unmarshallable object|can't pickle Selector objects"
     ):
         q.push(sel)
 
@@ -117,7 +117,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
         q = self.queue()
         with pytest.raises(
             ValueError,
-            match="Can't (get|pickle) local object|Can't pickle .*: it's not found as",
+            match=r"Can't (get|pickle) local object|Can't pickle .*: it's not found as",
         ) as exc_info:
             q.push(lambda x: x)
         if hasattr(sys, "pypy_version_info"):

tests/test_utils_conf.py

@@ -31,7 +31,7 @@ class TestBuildComponentList:
         # Same priority raises ValueError
         duplicate_bs.set("ONE", duplicate_bs["ONE"], priority=20)
         with pytest.raises(
-            ValueError, match="Some paths in .* convert to the same object"
+            ValueError, match=r"Some paths in .* convert to the same object"
         ):
             build_component_list(duplicate_bs, convert=lambda x: x.lower())
 

tests/test_utils_curl.py

@@ -220,7 +220,7 @@ class TestCurlToRequestKwargs:
             assert curl_to_request_kwargs(curl_command) == expected_result
 
         # case 2: ignore_unknown_options=False (raise exception):
-        with pytest.raises(ValueError, match="Unrecognized options:.*--bar.*--baz"):
+        with pytest.raises(ValueError, match=r"Unrecognized options:.*--bar.*--baz"):
             curl_to_request_kwargs(
                 "curl --bar --baz http://www.example.com", ignore_unknown_options=False
             )

tox.ini

@@ -45,15 +45,15 @@ commands =
 [testenv:typing]
 basepython = python3.10
 deps =
-    mypy==1.16.1
+    mypy==1.18.2
-    typing-extensions==4.14.1
+    typing-extensions==4.15.0
-    types-defusedxml==0.7.0.20250516
+    types-defusedxml==0.7.0.20250822
-    types-lxml==2025.3.30
+    types-lxml==2025.8.25
-    types-pexpect==4.9.0.20250516
+    types-pexpect==4.9.0.20250916
-    types-Pygments==2.19.0.20250516
+    types-Pygments==2.19.0.20250809
-    botocore-stubs==1.38.46
+    botocore-stubs==1.40.59
-    boto3-stubs[s3]==1.39.3
+    boto3-stubs[s3]==1.40.59
-    itemadapter==0.11.0
+    itemadapter==0.12.2
     Protego==0.5.0
     w3lib==2.3.1
     attrs >= 18.2.0
@@ -83,15 +83,15 @@ commands =
 basepython = python3
 deps =
     {[testenv:extra-deps]deps}
-    pylint==3.3.7
+    pylint==4.0.2
 commands =
     pylint conftest.py docs extras scrapy tests
 
 [testenv:twinecheck]
 basepython = python3
 deps =
-    twine==6.1.0
+    twine==6.2.0
-    build==1.2.2.post1
+    build==1.3.0
 commands =
     python -m build --sdist
     twine check dist/*