Merge e5086d410e into e28e56aa61

pyproject.toml

@@ -318,12 +318,32 @@ markers = [
     "requires_internet: marks tests that need real Internet access",
 ]
 filterwarnings = [
+    # Envs with pinned old dependencies opt out of this, see tox.ini.
+    "error",
     "ignore::DeprecationWarning:twisted.web.static",
-    # Jobs that do not report coverage disable it with --no-cov, which pytest-cov
-    # warns about because the coverage options below stay in place.
-    "ignore::pytest_cov.CovDisabledWarning",
-    # Twisted doesn't close failed sockets after CannotListenError: https://github.com/twisted/twisted/issues/6108
-    "ignore:Exception ignored in. <socket\\.socket.*laddr=..0\\.0\\.0\\.0., 0.:pytest.PytestUnraisableExceptionWarning",
+    # Twisted leaves the listening socket open after a failed bind
+    # (CannotListenError): https://github.com/twisted/twisted/issues/6108
+    # The asyncio reactor likewise leaves client sockets and transports open on
+    # teardown. These surface as unraisable exceptions during finalization; the
+    # message wording varies across Python implementations and versions
+    # (PyPy leaves the prefix empty), so match on the object repr.
+    "ignore:.*<socket\\.socket fd=:pytest.PytestUnraisableExceptionWarning",
+    "ignore:.*<function (_SelectorTransport|_ProactorBasePipeTransport|_ProactorSocketTransport|BaseEventLoop)\\.__del__:pytest.PytestUnraisableExceptionWarning",
+    # PyDispatcher iterates its connection registry while a weakref callback
+    # may mutate it, which raises RuntimeError on PyPy, where several weakrefs
+    # can die within a single collection.
+    "ignore:.*<function connect\\.<locals>\\.remove:pytest.PytestUnraisableExceptionWarning",
+    # queuelib's test helpers (reused by tests/test_squeues.py) leave queue
+    # files open; the resulting warning surfaces at an arbitrary GC point.
+    "ignore:.*<_io\\.FileIO name=.*queuelib-tests-:pytest.PytestUnraisableExceptionWarning",
+    # itemadapter imports pydantic.v1, which warns on Python 3.14 and higher.
+    "ignore:Core Pydantic V1 functionality isn't compatible:UserWarning",
+    # pyftpdlib imports asynchat, removed in Python 3.12, on lower versions.
+    "ignore:The async(hat|ore) module is deprecated:DeprecationWarning",
+    # CI runs without coverage pass --no-cov, which pytest-cov reports. Matched
+    # by message because some tox envs do not install pytest-cov, and pytest
+    # warns when a filter names a module it cannot import.
+    "ignore:Coverage disabled via --no-cov switch!",
 ]
 
 [tool.ruff.lint]

tests/test_command_shell.py

@@ -163,6 +163,16 @@ class TestShellCommand:
         assert ret == 0, out
 
 
+def _stop(p: PopenSpawn[str]) -> None:
+    p.sendeof()
+    p.wait()  # type: ignore[no-untyped-call]
+    # PopenSpawn leaves the subprocess pipes open, which triggers
+    # ResourceWarning at an arbitrary garbage collection point.
+    for pipe in (p.proc.stdin, p.proc.stdout):
+        if pipe:
+            pipe.close()
+
+
 class TestShellCommandWithSpider(TestProjectBase):
     @pytest.fixture(autouse=True)
     def create_files(self, proj_path: Path) -> None:
@@ -207,12 +217,7 @@ class TestInteractiveShell:
         p.sendline(f"fetch('{mockserver.url('/')}')")
         p.sendline("type(response)")
         p.expect_exact("HtmlResponse")
-        p.sendeof()
-        p.wait()  # type: ignore[no-untyped-call]
-        if p.proc.stdin:
-            p.proc.stdin.close()
-        if p.proc.stdout:
-            p.proc.stdout.close()
+        _stop(p)
         logfile.seek(0)
         assert "Traceback" not in logfile.read().decode()
 
@@ -238,8 +243,7 @@ class TestInteractiveShell:
         p = PopenSpawn(args, env=env, timeout=60)
         p.logfile_read = logfile
         p.expect_exact("Available Scrapy objects")
-        p.sendeof()
-        p.wait()  # type: ignore[no-untyped-call]
+        _stop(p)
         logfile.seek(0)
         return logfile.read().decode()
 
@@ -264,8 +268,7 @@ class TestInteractiveShell:
         # shell=python was honored, regardless of platform-specific prompts.
         p.sendline("import sys; print('IPYMODULE', 'IPython' in sys.modules)")
         p.expect_exact("IPYMODULE False")
-        p.sendeof()
-        p.wait()  # type: ignore[no-untyped-call]
+        _stop(p)
         logfile.seek(0)
         assert "Traceback" not in logfile.read().decode()
 

tests/test_feedexport_storages.py

@@ -212,16 +212,16 @@ class TestBlockingFeedStorage:
     def test_default_temp_dir(self):
         b = MyBlockingFeedStorage()
 
-        storage_file = b.open(get_test_spider())
-        storage_dir = Path(storage_file.name).parent
+        with b.open(get_test_spider()) as storage_file:
+            storage_dir = Path(storage_file.name).parent
         assert str(storage_dir) == tempfile.gettempdir()
 
     def test_temp_file(self, tmp_path):
         b = MyBlockingFeedStorage()
 
         spider = get_test_spider({"FEED_TEMPDIR": str(tmp_path)})
-        storage_file = b.open(spider)
-        storage_dir = Path(storage_file.name).parent
+        with b.open(spider) as storage_file:
+            storage_dir = Path(storage_file.name).parent
         assert storage_dir == tmp_path
 
     def test_invalid_folder(self, tmp_path):

tox.ini

@@ -154,6 +154,9 @@ deps =
     {[test-requirements]deps}
 setenv =
     _SCRAPY_MIN=true
+    # Pinned old Python and library versions trigger warnings that we cannot
+    # fix, so these envs do not turn warnings into errors.
+    PYTEST_ADDOPTS=-W default {env:PYTEST_ADDOPTS:}
 commands =
     pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=min.junit.xml -o junit_family=legacy --durations=10 scrapy tests}