From 4ca92f4c49c3cd55f6f455a95d6cdc3bf32af6a7 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Tue, 30 Jun 2026 16:44:20 +0200 Subject: [PATCH 1/6] WIP --- pyproject.toml | 20 ++++++++++++++++++-- tests/test_command_shell.py | 8 ++++++++ tests/test_feedexport_storages.py | 8 ++++---- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b8d9067f9..7caad834d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -273,9 +273,25 @@ markers = [ "requires_internet: marks tests that need real Internet access", ] filterwarnings = [ + "error", "ignore::DeprecationWarning:twisted.web.static", - # Twisted doesn't close failed sockets after CannotListenError: https://github.com/twisted/twisted/issues/6108 - "ignore:Exception ignored in. Date: Fri, 31 Jul 2026 03:03:46 +0200 Subject: [PATCH 2/6] Ignore min envs --- pyproject.toml | 9 +++------ tests/ignores.txt | 1 + tests/test_command_shell.py | 31 +++++++++++++------------------ tox.ini | 3 +++ 4 files changed, 20 insertions(+), 24 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d0c0ecf09..53a1221fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -359,6 +359,7 @@ markers = [ "requires_internet: marks tests that need real Internet access", ] filterwarnings = [ + # Envs with pinned old dependencies opt out of this, see tox.ini. "error", "ignore::DeprecationWarning:twisted.web.static", # Twisted leaves the listening socket open after a failed bind @@ -372,12 +373,8 @@ filterwarnings = [ # queuelib's test helpers (reused by tests/test_squeues.py) leave queue # files open; the resulting warning surfaces at an arbitrary GC point. "ignore:.*<_io\\.FileIO name=.*queuelib-tests-:pytest.PytestUnraisableExceptionWarning", - # pyftpdlib imports the deprecated asyncore/asynchat modules on Python < 3.12. - "ignore:The async(ore|hat) module is deprecated:DeprecationWarning", - # Old w3lib (min env) compiles a regex with inline flags not at the start. - "ignore:Flags not at the start of the expression:DeprecationWarning", - # Old Twisted (min env) calls the deprecated threading.currentThread(). - "ignore:currentThread\\(\\) is deprecated:DeprecationWarning", + # itemadapter imports pydantic.v1, which warns on Python 3.14 and higher. + "ignore:Core Pydantic V1 functionality isn't compatible:UserWarning", ] [tool.ruff.lint] diff --git a/tests/ignores.txt b/tests/ignores.txt index 3717bbc95..5be8576f3 100644 --- a/tests/ignores.txt +++ b/tests/ignores.txt @@ -1,3 +1,4 @@ scrapy/core/downloader/handlers/http.py scrapy/extensions/statsmailer.py +scrapy/interfaces.py scrapy/mail.py diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 5f8ad5a35..501f7132e 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -162,6 +162,16 @@ class TestShellCommand: assert ret == 0, out +def _stop(p: PopenSpawn[str]) -> None: + p.sendeof() + p.wait() # type: ignore[no-untyped-call] + # PopenSpawn leaves the subprocess pipes open, which triggers + # ResourceWarning at an arbitrary garbage collection point. + for pipe in (p.proc.stdin, p.proc.stdout): + if pipe: + pipe.close() + + class TestInteractiveShell: def test_fetch(self, mockserver: MockServer) -> None: args = ( @@ -179,12 +189,7 @@ class TestInteractiveShell: p.sendline(f"fetch('{mockserver.url('/')}')") p.sendline("type(response)") p.expect_exact("HtmlResponse") - p.sendeof() - p.wait() # type: ignore[no-untyped-call] - if p.proc.stdin: - p.proc.stdin.close() - if p.proc.stdout: - p.proc.stdout.close() + _stop(p) logfile.seek(0) assert "Traceback" not in logfile.read().decode() @@ -210,12 +215,7 @@ class TestInteractiveShell: p = PopenSpawn(args, env=env, timeout=5) p.logfile_read = logfile p.expect_exact("Available Scrapy objects") - p.sendeof() - p.wait() # type: ignore[no-untyped-call] - if p.proc.stdin: - p.proc.stdin.close() - if p.proc.stdout: - p.proc.stdout.close() + _stop(p) logfile.seek(0) return logfile.read().decode() @@ -240,12 +240,7 @@ class TestInteractiveShell: # shell=python was honored, regardless of platform-specific prompts. p.sendline("import sys; print('IPYMODULE', 'IPython' in sys.modules)") p.expect_exact("IPYMODULE False") - p.sendeof() - p.wait() # type: ignore[no-untyped-call] - if p.proc.stdin: - p.proc.stdin.close() - if p.proc.stdout: - p.proc.stdout.close() + _stop(p) logfile.seek(0) assert "Traceback" not in logfile.read().decode() diff --git a/tox.ini b/tox.ini index e10a7cc0c..b63295518 100644 --- a/tox.ini +++ b/tox.ini @@ -147,6 +147,9 @@ deps = {[test-requirements]deps} setenv = _SCRAPY_MIN=true + # Pinned old Python and library versions trigger warnings that we cannot + # fix, so these envs do not turn warnings into errors. + PYTEST_ADDOPTS=-W default {env:PYTEST_ADDOPTS:} commands = pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=min.junit.xml -o junit_family=legacy --durations=10 scrapy tests} From 69209001e7379d1f463d401d655eaae4f13f3d97 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 31 Jul 2026 04:09:05 +0200 Subject: [PATCH 3/6] Track additional warning --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 53a1221fa..180debfff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -375,6 +375,8 @@ filterwarnings = [ "ignore:.*<_io\\.FileIO name=.*queuelib-tests-:pytest.PytestUnraisableExceptionWarning", # itemadapter imports pydantic.v1, which warns on Python 3.14 and higher. "ignore:Core Pydantic V1 functionality isn't compatible:UserWarning", + # pyftpdlib imports asynchat, removed in Python 3.12, on lower versions. + "ignore:The async(hat|ore) module is deprecated:DeprecationWarning", ] [tool.ruff.lint] From 18114a458ca46ca69a4d89c8aa6d7108aca74334 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 31 Jul 2026 05:15:16 +0200 Subject: [PATCH 4/6] Ignore expected warnings from PyPy --- pyproject.toml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 180debfff..cb7f96a23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -366,10 +366,14 @@ filterwarnings = [ # (CannotListenError): https://github.com/twisted/twisted/issues/6108 # The asyncio reactor likewise leaves client sockets and transports open on # teardown. These surface as unraisable exceptions during finalization; the - # message wording varies across Python versions (sometimes there is no - # "Exception ignored ..." prefix at all), so match on the object repr. + # message wording varies across Python implementations and versions + # (PyPy leaves the prefix empty), so match on the object repr. "ignore:.*\\.remove:pytest.PytestUnraisableExceptionWarning", # queuelib's test helpers (reused by tests/test_squeues.py) leave queue # files open; the resulting warning surfaces at an arbitrary GC point. "ignore:.*<_io\\.FileIO name=.*queuelib-tests-:pytest.PytestUnraisableExceptionWarning", From daff5abef7374e2e737ff79a039ffa80d0abef30 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 31 Jul 2026 14:57:28 +0200 Subject: [PATCH 5/6] Silence no-cov warnings --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index cb7f96a23..736cfd3df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -381,6 +381,8 @@ filterwarnings = [ "ignore:Core Pydantic V1 functionality isn't compatible:UserWarning", # pyftpdlib imports asynchat, removed in Python 3.12, on lower versions. "ignore:The async(hat|ore) module is deprecated:DeprecationWarning", + # CI runs without coverage pass --no-cov, which pytest-cov reports. + "ignore::pytest_cov.CovDisabledWarning", ] [tool.ruff.lint] From 3c4c868d3d81f67f5bdbd22b9222f91f40692067 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 31 Jul 2026 15:53:41 +0200 Subject: [PATCH 6/6] Improve the filtering of --no-cov warnings --- pyproject.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 736cfd3df..8c6bc402a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -381,8 +381,10 @@ filterwarnings = [ "ignore:Core Pydantic V1 functionality isn't compatible:UserWarning", # pyftpdlib imports asynchat, removed in Python 3.12, on lower versions. "ignore:The async(hat|ore) module is deprecated:DeprecationWarning", - # CI runs without coverage pass --no-cov, which pytest-cov reports. - "ignore::pytest_cov.CovDisabledWarning", + # CI runs without coverage pass --no-cov, which pytest-cov reports. Matched + # by message because some tox envs do not install pytest-cov, and pytest + # warns when a filter names a module it cannot import. + "ignore:Coverage disabled via --no-cov switch!", ] [tool.ruff.lint]