Refactor and improve catching warnings in tests. (#7643)

scrapy/spidermiddlewares/referer.py

@@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, cast
 from urllib.parse import urlparse
 from warnings import warn
 
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
 from scrapy.utils.misc import load_object
@@ -349,7 +349,7 @@ class RefererMiddleware(BaseSpiderMiddleware):
             response = kwargs.pop("resp_or_url")
             warn(
                 "Passing 'resp_or_url' is deprecated, use 'response' instead.",
-                DeprecationWarning,
+                ScrapyDeprecationWarning,
                 stacklevel=2,
             )
         if response is None:
@@ -360,7 +360,7 @@ class RefererMiddleware(BaseSpiderMiddleware):
             warn(
                 "Passing a response URL to RefererMiddleware.policy() instead "
                 "of a Response object is deprecated.",
-                DeprecationWarning,
+                ScrapyDeprecationWarning,
                 stacklevel=2,
             )
         allow_import_path = True

scrapy/spiders/crawl.py

@@ -110,6 +110,7 @@ class CrawlSpider(Spider):
                 "deprecated: it will be removed in future Scrapy releases. "
                 "Please override the CrawlSpider.parse_with_rules method "
                 "instead.",
+                ScrapyDeprecationWarning,
                 stacklevel=2,
             )
 
@@ -199,6 +200,7 @@ class CrawlSpider(Spider):
             "The CrawlSpider._parse_response method is deprecated: "
             "it will be removed in future Scrapy releases. "
             "Please use the CrawlSpider.parse_with_rules method instead.",
+            ScrapyDeprecationWarning,
             stacklevel=2,
         )
         return self.parse_with_rules(response, callback, cb_kwargs, follow)

tests/test_crawler.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import asyncio
 import logging
 import re
-import warnings
 from pathlib import Path
 from typing import Any, ClassVar
 
@@ -88,9 +87,7 @@ class TestCrawler(TestBaseCrawler):
         self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
 
     def test_crawler_accepts_None(self) -> None:
-        with warnings.catch_warnings():
+        crawler = Crawler(DefaultSpider)
-            warnings.simplefilter("ignore", ScrapyDeprecationWarning)
-            crawler = Crawler(DefaultSpider)
         self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
 
     def test_crawler_rejects_spider_objects(self) -> None:

tests/test_dupefilters.py

@@ -3,8 +3,8 @@ import shutil
 import sys
 import tempfile
 from pathlib import Path
-from warnings import catch_warnings
 
+import pytest
 from testfixtures import LogCapture
 
 from scrapy.core.scheduler import Scheduler
@@ -260,11 +260,8 @@ class TestBaseDupeFilter:
         dupefilter = _get_dupefilter(
             settings={"DUPEFILTER_CLASS": BaseDupeFilter},
         )
-        with catch_warnings(record=True) as warning_list:
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match=r"Calling BaseDupeFilter\.log\(\) is deprecated.",
+        ):
             dupefilter.log(None, None)
-        assert len(warning_list) == 1
-        assert (
-            str(warning_list[0].message)
-            == "Calling BaseDupeFilter.log() is deprecated."
-        )
-        assert warning_list[0].category == ScrapyDeprecationWarning

tests/test_http_request.py

@@ -460,11 +460,13 @@ class TestRequest:
     def test_from_curl_ignore_unknown_options(self):
         # By default: it works and ignores the unknown options: --foo and -z
         with warnings.catch_warnings():  # avoid warning when executing tests
-            warnings.simplefilter("ignore")
+            warnings.filterwarnings(
+                "ignore", category=UserWarning, message="Unrecognized options:"
+            )
             r = self.request_class.from_curl(
                 'curl -X DELETE "http://example.org" --foo -z',
             )
-            assert r.method == "DELETE"
+        assert r.method == "DELETE"
 
         # If `ignore_unknown_options` is set to `False` it raises an error with
         # the unknown options: --foo and -z

tests/test_http_request_json.py

@@ -4,6 +4,8 @@ import json
 import warnings
 from unittest import mock
 
+import pytest
+
 from scrapy.http import JsonRequest
 from scrapy.utils.python import to_bytes
 from tests.test_http_request import TestRequest
@@ -63,40 +65,40 @@ class TestJsonRequest(TestRequest):
         data = {
             "name": "value",
         }
-        with warnings.catch_warnings(record=True) as _warnings:
+        with pytest.warns(UserWarning, match="data will be ignored"):
             r5 = self.request_class(url="http://www.example.com/", body=body, data=data)
-            assert r5.body == body
+        assert r5.body == body
-            assert r5.method == "GET"
+        assert r5.method == "GET"
-            assert len(_warnings) == 1
-            assert "data will be ignored" in str(_warnings[0].message)
 
     def test_empty_body_data(self):
         """passing any body value and data should result a warning"""
         data = {
             "name": "value",
         }
-        with warnings.catch_warnings(record=True) as _warnings:
+        with pytest.warns(UserWarning, match="data will be ignored"):
             r6 = self.request_class(url="http://www.example.com/", body=b"", data=data)
-            assert r6.body == b""
+        assert r6.body == b""
-            assert r6.method == "GET"
+        assert r6.method == "GET"
-            assert len(_warnings) == 1
-            assert "data will be ignored" in str(_warnings[0].message)
 
     def test_body_none_data(self):
         data = {
             "name": "value",
         }
-        with warnings.catch_warnings(record=True) as _warnings:
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "error", category=UserWarning, message="Both body and data passed"
+            )
             r7 = self.request_class(url="http://www.example.com/", body=None, data=data)
-            assert r7.body == to_bytes(json.dumps(data))
+        assert r7.body == to_bytes(json.dumps(data))
-            assert r7.method == "POST"
+        assert r7.method == "POST"
-            assert len(_warnings) == 0
 
     def test_body_data_none(self):
-        with warnings.catch_warnings(record=True) as _warnings:
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "error", category=UserWarning, message="Both body and data passed"
+            )
             r8 = self.request_class(url="http://www.example.com/", body=None, data=None)
-            assert r8.method == "GET"
+        assert r8.method == "GET"
-            assert len(_warnings) == 0
 
     def test_dumps_sort_keys(self):
         """Test that sort_keys=True is passed to json.dumps by default"""
@@ -183,8 +185,5 @@ class TestJsonRequest(TestRequest):
         }
         r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1)
 
-        with warnings.catch_warnings(record=True) as _warnings:
+        with pytest.warns(UserWarning, match="data will be ignored"):
             r1.replace(data=data2, body=body2)
-            assert "Both body and data passed. data will be ignored" in str(
-                _warnings[0].message
-            )

tests/test_pipeline_files.py

@@ -2,7 +2,6 @@ import dataclasses
 import os
 import random
 import time
-import warnings
 from abc import ABC, abstractmethod
 from datetime import datetime
 from ftplib import FTP
@@ -786,12 +785,10 @@ class TestBuildFromCrawler:
         class Pipeline(FilesPipeline):
             pass
 
-        with warnings.catch_warnings(record=True) as w:
+        pipe = Pipeline.from_crawler(self.crawler)
-            pipe = Pipeline.from_crawler(self.crawler)
+        assert pipe.crawler == self.crawler
-            assert pipe.crawler == self.crawler
+        assert pipe._fingerprinter
-            assert pipe._fingerprinter
+        assert pipe.store
-            assert len(w) == 0
-            assert pipe.store
 
     def test_has_from_crawler_and_init(self):
         class Pipeline(FilesPipeline):
@@ -805,13 +802,11 @@ class TestBuildFromCrawler:
                 o._from_crawler_called = True
                 return o
 
-        with warnings.catch_warnings(record=True) as w:
+        pipe = Pipeline.from_crawler(self.crawler)
-            pipe = Pipeline.from_crawler(self.crawler)
+        assert pipe.crawler == self.crawler
-            assert pipe.crawler == self.crawler
+        assert pipe._fingerprinter
-            assert pipe._fingerprinter
+        assert pipe.store
-            assert len(w) == 0
+        assert pipe._from_crawler_called
-            assert pipe.store
-            assert pipe._from_crawler_called
 
 
 @pytest.mark.parametrize("store", [None, ""])

tests/test_pipeline_media.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import warnings
 from unittest.mock import MagicMock
 
 import pytest
@@ -425,11 +424,9 @@ class TestBuildFromCrawler:
         class Pipeline(UserDefinedPipeline):
             pass
 
-        with warnings.catch_warnings(record=True) as w:
+        pipe = Pipeline.from_crawler(self.crawler)
-            pipe = Pipeline.from_crawler(self.crawler)
+        assert pipe.crawler == self.crawler
-            assert pipe.crawler == self.crawler
+        assert pipe._fingerprinter
-            assert pipe._fingerprinter
-            assert len(w) == 0
 
     def test_has_from_crawler_and_init(self):
         class Pipeline(UserDefinedPipeline):
@@ -447,13 +444,11 @@ class TestBuildFromCrawler:
                 o._from_crawler_called = True
                 return o
 
-        with warnings.catch_warnings(record=True) as w:
+        pipe = Pipeline.from_crawler(self.crawler)
-            pipe = Pipeline.from_crawler(self.crawler)
+        assert pipe.crawler == self.crawler
-            assert pipe.crawler == self.crawler
+        assert pipe._fingerprinter
-            assert pipe._fingerprinter
+        assert pipe._from_crawler_called
-            assert len(w) == 0
+        assert pipe._init_called
-            assert pipe._from_crawler_called
-            assert pipe._init_called
 
     def test_has_from_crawler(self):
         class Pipeline(UserDefinedPipeline):
@@ -467,13 +462,10 @@ class TestBuildFromCrawler:
                 o.store_uri = settings["FILES_STORE"]
                 return o
 
-        with warnings.catch_warnings(record=True) as w:
+        pipe = Pipeline.from_crawler(self.crawler)
-            pipe = Pipeline.from_crawler(self.crawler)
+        assert pipe.crawler == self.crawler
-            # this and the next assert will fail as MediaPipeline.from_crawler() wasn't called
+        assert pipe._fingerprinter
-            assert pipe.crawler == self.crawler
+        assert pipe._from_crawler_called
-            assert pipe._fingerprinter
-            assert len(w) == 0
-            assert pipe._from_crawler_called
 
 
 class MediaFailedFailurePipeline(MockedMediaPipeline):

tests/test_scheduler.py

@@ -12,6 +12,7 @@ import pytest
 from scrapy.core.downloader import Downloader
 from scrapy.core.scheduler import BaseScheduler, Scheduler
 from scrapy.crawler import Crawler
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.spiders import Spider
 from scrapy.utils.defer import ensure_awaitable
@@ -396,7 +397,11 @@ class TestIncompatibility:
 
     def test_incompatibility(self):
         with warnings.catch_warnings():
-            warnings.filterwarnings("ignore")
+            warnings.filterwarnings(
+                "ignore",
+                category=ScrapyDeprecationWarning,
+                message="The CONCURRENT_REQUESTS_PER_IP setting is deprecated",
+            )
             with pytest.raises(
                 ValueError, match="does not support CONCURRENT_REQUESTS_PER_IP"
             ):

tests/test_scrapy__getattr__.py

@@ -1,15 +1,18 @@
-import warnings
+from __future__ import annotations
+
+import pytest
+
+from scrapy.exceptions import ScrapyDeprecationWarning
 
 
-def test_deprecated_concurrent_requests_per_ip_attribute():
+def test_deprecated_concurrent_requests_per_ip_attribute() -> None:
-    with warnings.catch_warnings(record=True) as warns:
+    with pytest.warns(
+        ScrapyDeprecationWarning,
+        match=r"scrapy\.settings\.default_settings\.CONCURRENT_REQUESTS_PER_IP attribute is deprecated",
+    ):
         from scrapy.settings.default_settings import (  # noqa: PLC0415
             CONCURRENT_REQUESTS_PER_IP,
         )
 
-        assert CONCURRENT_REQUESTS_PER_IP is not None
+    assert CONCURRENT_REQUESTS_PER_IP is not None
-        assert isinstance(CONCURRENT_REQUESTS_PER_IP, int)
+    assert isinstance(CONCURRENT_REQUESTS_PER_IP, int)
-        assert (
-            "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead."
-            in warns[0].message.args
-        )

tests/test_settings/__init__.py

@@ -2,12 +2,12 @@
 # (too many false positives)
 
 import logging
-import warnings
 from unittest import mock
 
 import pytest
 
 from scrapy.core.downloader.handlers.file import FileDownloadHandler
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import (
     SETTINGS_PRIORITIES,
     BaseSettings,
@@ -711,15 +711,13 @@ def test_remove_from_list(before, name, item, after):
 
 
 def test_deprecated_concurrent_requests_per_ip_setting():
-    with warnings.catch_warnings(record=True) as warns:
+    settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1})
-        settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1})
+    with pytest.warns(
+        ScrapyDeprecationWarning,
+        match="The CONCURRENT_REQUESTS_PER_IP setting is deprecated",
+    ):
         settings.get("CONCURRENT_REQUESTS_PER_IP")
 
-    assert (
-        str(warns[0].message)
-        == "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead."
-    )
-
 
 class Component1:
     pass

tests/test_spider_crawl.py

@@ -8,6 +8,7 @@ import pytest
 from testfixtures import LogCapture
 from w3lib.url import safe_url_string
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import HtmlResponse, Request, TextResponse
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule, Spider
@@ -264,13 +265,16 @@ class TestCrawlSpider(TestSpider):
             start_urls = "https://www.example.com"
             _follow_links = False
 
-        with warnings.catch_warnings(record=True) as w:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", category=ScrapyDeprecationWarning)
             spider = _CrawlSpider()
-            assert len(w) == 0
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match=r"CrawlSpider\._parse_response method is deprecated",
+        ):
             spider._parse_response(
                 TextResponse(spider.start_urls, body=b""), None, None
             )
-            assert len(w) == 1
 
     def test_parse_response_override(self):
         class _CrawlSpider(CrawlSpider):
@@ -281,26 +285,28 @@ class TestCrawlSpider(TestSpider):
             start_urls = "https://www.example.com"
             _follow_links = False
 
-        with warnings.catch_warnings(record=True) as w:
+        with pytest.warns(
-            assert len(w) == 0
+            ScrapyDeprecationWarning,
+            match=r"CrawlSpider\._parse_response method, which the",
+        ):
             spider = _CrawlSpider()
-            assert len(w) == 1
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", category=ScrapyDeprecationWarning)
             spider._parse_response(
                 TextResponse(spider.start_urls, body=b""), None, None
             )
-            assert len(w) == 1
 
     def test_parse_with_rules(self):
         class _CrawlSpider(CrawlSpider):
             name = "test"
             start_urls = "https://www.example.com"
 
-        with warnings.catch_warnings(record=True) as w:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", category=ScrapyDeprecationWarning)
             spider = _CrawlSpider()
             spider.parse_with_rules(
                 TextResponse(spider.start_urls, body=b""), None, None
             )
-            assert len(w) == 0
 
 
 class TestDeprecation:

tests/test_spiderloader/__init__.py

@@ -1,7 +1,6 @@
 import contextlib
 import shutil
 import sys
-import warnings
 from pathlib import Path
 from unittest import mock
 
@@ -137,50 +136,38 @@ class TestSpiderLoader:
             SpiderLoader.from_settings(settings)
 
     def test_bad_spider_modules_warning(self):
-        with warnings.catch_warnings(record=True) as w:
+        module = "tests.test_spiderloader.test_spiders.doesnotexist"
-            module = "tests.test_spiderloader.test_spiders.doesnotexist"
+        settings = Settings(
-            settings = Settings(
+            {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True}
-                {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True}
+        )
-            )
+        with pytest.warns(RuntimeWarning, match="Could not load spiders from module"):
             spider_loader = SpiderLoader.from_settings(settings)
-            if str(w[0].message).startswith("_SixMetaPathImporter"):
-                # needed on 3.10 because of https://github.com/benjaminp/six/issues/349,
-                # at least until all six versions we can import (including botocore.vendored.six)
-                # are updated to 1.16.0+
-                w.pop(0)
-            assert "Could not load spiders from module" in str(w[0].message)
 
-            spiders = spider_loader.list()
+        spiders = spider_loader.list()
-            assert not spiders
+        assert not spiders
 
     def test_syntax_error_exception(self):
         module = "tests.test_spiderloader.test_spiders.spider1"
+        settings = Settings({"SPIDER_MODULES": [module]})
         with mock.patch.object(SpiderLoader, "_load_spiders") as m:
             m.side_effect = SyntaxError
-            settings = Settings({"SPIDER_MODULES": [module]})
             with pytest.raises(SyntaxError):
                 SpiderLoader.from_settings(settings)
 
     def test_syntax_error_warning(self):
-        with (
+        module = "tests.test_spiderloader.test_spiders.spider1"
-            warnings.catch_warnings(record=True) as w,
+        settings = Settings(
-            mock.patch.object(SpiderLoader, "_load_spiders") as m,
+            {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True}
-        ):
+        )
+        with mock.patch.object(SpiderLoader, "_load_spiders") as m:
             m.side_effect = SyntaxError
-            module = "tests.test_spiderloader.test_spiders.spider1"
+            with pytest.warns(
-            settings = Settings(
+                RuntimeWarning, match="Could not load spiders from module"
-                {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True}
+            ):
-            )
+                spider_loader = SpiderLoader.from_settings(settings)
-            spider_loader = SpiderLoader.from_settings(settings)
-            if str(w[0].message).startswith("_SixMetaPathImporter"):
-                # needed on 3.10 because of https://github.com/benjaminp/six/issues/349,
-                # at least until all six versions we can import (including botocore.vendored.six)
-                # are updated to 1.16.0+
-                w.pop(0)
-            assert "Could not load spiders from module" in str(w[0].message)
 
-            spiders = spider_loader.list()
+        spiders = spider_loader.list()
-            assert not spiders
+        assert not spiders
 
 
 class TestDuplicateSpiderNameLoader:
@@ -190,21 +177,17 @@ class TestDuplicateSpiderNameLoader:
         # copy 1 spider module so as to have duplicate spider name
         shutil.copyfile(spiders_dir / "spider3.py", spiders_dir / "spider3dupe.py")
 
-        with warnings.catch_warnings(record=True) as w:
+        msg = r"""There are several spiders with the same name:
+
+  Spider3 named 'spider3' \(in test_spiders_xxx\.spider3\)
+
+  Spider3 named 'spider3' \(in test_spiders_xxx\.spider3dupe\)
+
+  This can cause unexpected behavior\."""
+        with pytest.warns(UserWarning, match=msg):
             spider_loader = SpiderLoader.from_settings(settings)
-
+        spiders = set(spider_loader.list())
-            assert len(w) == 1
+        assert spiders == {"spider1", "spider2", "spider3", "spider4"}
-            msg = str(w[0].message)
-            assert "several spiders with the same name" in msg
-            assert "'spider3'" in msg
-            assert msg.count("'spider3'") == 2
-
-            assert "'spider1'" not in msg
-            assert "'spider2'" not in msg
-            assert "'spider4'" not in msg
-
-            spiders = set(spider_loader.list())
-            assert spiders == {"spider1", "spider2", "spider3", "spider4"}
 
     def test_multiple_dupename_warning(self, spider_loader_env):
         settings, spiders_dir = spider_loader_env
@@ -213,23 +196,21 @@ class TestDuplicateSpiderNameLoader:
         shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py")
         shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py")
 
-        with warnings.catch_warnings(record=True) as w:
+        msg = r"""There are several spiders with the same name:
+
+  Spider1 named 'spider1' \(in test_spiders_xxx\.spider1\)
+
+  Spider1 named 'spider1' \(in test_spiders_xxx\.spider1dupe\)
+
+  Spider2 named 'spider2' \(in test_spiders_xxx\.spider2\)
+
+  Spider2 named 'spider2' \(in test_spiders_xxx\.spider2dupe\)
+
+  This can cause unexpected behavior\."""
+        with pytest.warns(UserWarning, match=msg):
             spider_loader = SpiderLoader.from_settings(settings)
-
+        spiders = set(spider_loader.list())
-            assert len(w) == 1
+        assert spiders == {"spider1", "spider2", "spider3", "spider4"}
-            msg = str(w[0].message)
-            assert "several spiders with the same name" in msg
-            assert "'spider1'" in msg
-            assert msg.count("'spider1'") == 2
-
-            assert "'spider2'" in msg
-            assert msg.count("'spider2'") == 2
-
-            assert "'spider3'" not in msg
-            assert "'spider4'" not in msg
-
-            spiders = set(spider_loader.list())
-            assert spiders == {"spider1", "spider2", "spider3", "spider4"}
 
 
 class CustomSpiderLoader(SpiderLoader):

tests/test_spidermiddleware_referer.py

@@ -6,6 +6,7 @@ from urllib.parse import urlparse
 
 import pytest
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.spidermiddlewares.referer import (
@@ -842,13 +843,14 @@ class TestRequestMetaSettingFallback:
             response = Response(origin, headers=response_headers)
             request = Request(target, meta=request_meta)
 
-            with warnings.catch_warnings(record=True) as w:
+            if check_warning:
+                with pytest.warns(
+                    RuntimeWarning, match="Could not load referrer policy"
+                ):
+                    policy = mw.policy(response, request)
+            else:
                 policy = mw.policy(response, request)
-                assert isinstance(policy, policy_class)
+            assert isinstance(policy, policy_class)
-
-                if check_warning:
-                    assert len(w) == 1
-                    assert w[0].category is RuntimeWarning, w[0].message
 
 
 class TestSettingsPolicyByName:
@@ -973,49 +975,39 @@ class TestPolicyMethodResponseParamRename:
         self.response = Response("http://www.example.com")
 
     def test_pos_string(self):
-        with warnings.catch_warnings(record=True) as w:
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match=r"Passing a response URL to RefererMiddleware\.policy\(\)",
+        ):
             self.mw.policy("http://old.com", self.request)
-            found = False
-            for warning in w:
-                if "Passing a response URL" in str(warning.message):
-                    found = True
-                    break
-            assert found
 
     def test_pos_response(self):
-        with warnings.catch_warnings(record=True) as w:
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "error",
+                category=ScrapyDeprecationWarning,
+                message=r"Passing 'resp_or_url' is deprecated",
+            )
             self.mw.policy(self.response, self.request)
-            for warning in w:
-                assert "resp_or_url" not in str(warning.message)
 
     def test_key_resp_or_url(self):
-        with warnings.catch_warnings(record=True) as w:
+        with pytest.warns(
+            ScrapyDeprecationWarning, match=r"Passing 'resp_or_url' is deprecated"
+        ):
             self.mw.policy(resp_or_url=self.response, request=self.request)
-            found = False
-            for warning in w:
-                if "Passing 'resp_or_url' is deprecated, use 'response' instead" in str(
-                    warning.message
-                ):
-                    found = True
-                    break
-            assert found
 
     def test_key_response(self):
-        with warnings.catch_warnings(record=True) as w:
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "error",
+                category=ScrapyDeprecationWarning,
+                message=r"Passing 'resp_or_url' is deprecated",
+            )
             self.mw.policy(response=self.response, request=self.request)
-            for warning in w:
-                assert "resp_or_url" not in str(warning.message)
 
     def test_key_response_string(self):
-        with warnings.catch_warnings(record=True) as w:
+        with pytest.warns(ScrapyDeprecationWarning, match="Passing a response URL"):
-            warnings.simplefilter("always")
             self.mw.policy(response="http://old.com", request=self.request)
-            found = False
-            for warning in w:
-                if "Passing a response URL" in str(warning.message):
-                    found = True
-                    break
-            assert found
 
     def test_both_resp_or_url_and_response(self):
         with pytest.raises(

tests/test_utils_curl.py

@@ -213,10 +213,12 @@ class TestCurlToRequestKwargs:
 
     def test_ignore_unknown_options(self):
         # case 1: ignore_unknown_options=True:
+        curl_command = "curl --bar --baz http://www.example.com"
+        expected_result = {"method": "GET", "url": "http://www.example.com"}
         with warnings.catch_warnings():  # avoid warning when executing tests
-            warnings.simplefilter("ignore")
+            warnings.filterwarnings(
-            curl_command = "curl --bar --baz http://www.example.com"
+                "ignore", category=UserWarning, message="Unrecognized options:"
-            expected_result = {"method": "GET", "url": "http://www.example.com"}
+            )
             assert curl_to_request_kwargs(curl_command) == expected_result
 
         # case 2: ignore_unknown_options=False (raise exception):

tests/test_utils_datatypes.py

@@ -1,5 +1,4 @@
 import copy
-import warnings
 from abc import ABC, abstractmethod
 from collections.abc import Iterator, Mapping, MutableMapping
 from typing import Any
@@ -227,18 +226,12 @@ class TestCaselessDict(TestCaseInsensitiveDictBase):
     dict_class = CaselessDict
 
     def test_deprecation_message(self):
-        with warnings.catch_warnings(record=True) as caught:
+        with pytest.warns(
-            warnings.filterwarnings("always", category=ScrapyDeprecationWarning)
+            ScrapyDeprecationWarning,
+            match=r"scrapy.utils.datatypes.CaselessDict is deprecated",
+        ):
             self.dict_class({"foo": "bar"})
 
-            assert len(caught) == 1
-            assert issubclass(caught[0].category, ScrapyDeprecationWarning)
-            assert (
-                str(caught[0].message)
-                == "scrapy.utils.datatypes.CaselessDict is deprecated,"
-                " please use scrapy.utils.datatypes.CaseInsensitiveDict instead"
-            )
-
 
 class TestSequenceExclude:
     def test_list(self):

tests/test_utils_deprecate.py

@@ -1,7 +1,6 @@
 import inspect
 import warnings
 from unittest import mock
-from warnings import WarningMessage
 
 import pytest
 
@@ -22,34 +21,26 @@ class NewName(SomeBaseClass):
 
 
 class TestWarnWhenSubclassed:
-    def _mywarnings(self, w: list[WarningMessage]) -> list[WarningMessage]:
-        return [x for x in w if x.category is MyWarning]
-
     def test_no_warning_on_definition(self):
-        with warnings.catch_warnings(record=True) as w:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", category=ScrapyDeprecationWarning)
             create_deprecated_class("Deprecated", NewName)
 
-        w = self._mywarnings(w)
-        assert w == []
-
     def test_subclassing_warning_message(self):
+        msg = (
+            r"tests\.test_utils_deprecate\.UserClass inherits from "
+            r"deprecated class tests\.test_utils_deprecate\.Deprecated, "
+            r"please inherit from tests\.test_utils_deprecate\.NewName."
+            r" \(warning only on first subclass, there may be others\)"
+        )
         Deprecated = create_deprecated_class(
             "Deprecated", NewName, warn_category=MyWarning
         )
-
+        with pytest.warns(MyWarning, match=msg) as w:
-        with warnings.catch_warnings(record=True) as w:
 
             class UserClass(Deprecated):
                 pass
 
-        w = self._mywarnings(w)
-        assert len(w) == 1
-        assert (
-            str(w[0].message) == "tests.test_utils_deprecate.UserClass inherits from "
-            "deprecated class tests.test_utils_deprecate.Deprecated, "
-            "please inherit from tests.test_utils_deprecate.NewName."
-            " (warning only on first subclass, there may be others)"
-        )
         assert w[0].lineno == inspect.getsourcelines(UserClass)[1]
 
     def test_custom_class_paths(self):
@@ -61,62 +52,77 @@ class TestWarnWhenSubclassed:
             warn_category=MyWarning,
         )
 
-        with warnings.catch_warnings(record=True) as w:
+        with pytest.warns(
+            MyWarning,
+            match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass",
+        ):
 
             class UserClass(Deprecated):
                 pass
 
+        with pytest.warns(
+            MyWarning,
+            match=r"bar\.OldClass is deprecated, instantiate foo\.NewClass instead",
+        ):
             _ = Deprecated()
 
-        w = self._mywarnings(w)
-        assert len(w) == 2
-        assert "foo.NewClass" in str(w[0].message)
-        assert "bar.OldClass" in str(w[0].message)
-        assert "foo.NewClass" in str(w[1].message)
-        assert "bar.OldClass" in str(w[1].message)
-
     def test_subclassing_warns_only_on_direct_children(self):
         Deprecated = create_deprecated_class(
             "Deprecated", NewName, warn_once=False, warn_category=MyWarning
         )
 
-        with warnings.catch_warnings(record=True) as w:
+        with pytest.warns(
+            MyWarning,
+            match="UserClass inherits from deprecated class",
+        ):
 
             class UserClass(Deprecated):
                 pass
 
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", MyWarning)
+
             class NoWarnOnMe(UserClass):
                 pass
 
-        w = self._mywarnings(w)
-        assert len(w) == 1
-        assert "UserClass" in str(w[0].message)
-
     def test_subclassing_warns_once_by_default(self):
         Deprecated = create_deprecated_class(
             "Deprecated", NewName, warn_category=MyWarning
         )
 
-        with warnings.catch_warnings(record=True) as w:
+        with pytest.warns(
+            MyWarning,
+            match="UserClass inherits from deprecated class",
+        ):
 
             class UserClass(Deprecated):
                 pass
 
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", MyWarning)
+
             class FooClass(Deprecated):
                 pass
 
             class BarClass(Deprecated):
                 pass
 
-        w = self._mywarnings(w)
-        assert len(w) == 1
-        assert "UserClass" in str(w[0].message)
-
     def test_warning_on_instance(self):
         Deprecated = create_deprecated_class(
             "Deprecated", NewName, warn_category=MyWarning
         )
 
+        with pytest.warns(MyWarning) as w:
+            _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())
+
+        w = [x for x in w if x.category is MyWarning]
+        assert len(w) == 1
+        assert (
+            str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, "
+            "instantiate tests.test_utils_deprecate.NewName instead."
+        )
+        assert w[0].lineno == lineno
+
         # ignore subclassing warnings
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", MyWarning)
@@ -124,29 +130,20 @@ class TestWarnWhenSubclassed:
             class UserClass(Deprecated):
                 pass
 
-        with warnings.catch_warnings(record=True) as w:
+        with warnings.catch_warnings():
-            _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())
+            warnings.simplefilter("error", MyWarning)
-            _ = UserClass()  # subclass instances don't warn
+            UserClass()  # subclass instances don't warn
-
-        w = self._mywarnings(w)
-        assert len(w) == 1
-        assert (
-            str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, "
-            "instantiate tests.test_utils_deprecate.NewName instead."
-        )
-        assert w[0].lineno == lineno
 
     def test_warning_auto_message(self):
-        with warnings.catch_warnings(record=True) as w:
+        Deprecated = create_deprecated_class("Deprecated", NewName)
-            Deprecated = create_deprecated_class("Deprecated", NewName)
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName",
+        ):
 
             class UserClass2(Deprecated):
                 pass
 
-        msg = str(w[0].message)
-        assert "tests.test_utils_deprecate.NewName" in msg
-        assert "tests.test_utils_deprecate.Deprecated" in msg
-
     def test_issubclass(self):
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", ScrapyDeprecationWarning)
@@ -222,8 +219,8 @@ class TestWarnWhenSubclassed:
         create_deprecated_class("Deprecated", New)
 
     def test_deprecate_subclass_of_deprecated_class(self):
-        with warnings.catch_warnings(record=True) as w:
+        with warnings.catch_warnings():
-            warnings.simplefilter("always")
+            warnings.simplefilter("error", MyWarning)
             Deprecated = create_deprecated_class(
                 "Deprecated", NewName, warn_category=MyWarning
             )
@@ -234,33 +231,26 @@ class TestWarnWhenSubclassed:
                 warn_category=MyWarning,
             )
 
-        w = self._mywarnings(w)
+        with pytest.warns(
-        assert len(w) == 0, [str(warning) for warning in w]
+            MyWarning,
-
+            match=r"AlsoDeprecated is deprecated, instantiate foo\.Bar instead",
-        with warnings.catch_warnings(record=True) as w:
+        ):
             AlsoDeprecated()
 
+        with pytest.warns(
+            MyWarning,
+            match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar",
+        ):
+
             class UserClass(AlsoDeprecated):
                 pass
 
-        w = self._mywarnings(w)
-        assert len(w) == 2
-        assert "AlsoDeprecated" in str(w[0].message)
-        assert "foo.Bar" in str(w[0].message)
-        assert "AlsoDeprecated" in str(w[1].message)
-        assert "foo.Bar" in str(w[1].message)
-
     def test_inspect_stack(self):
         with (
             mock.patch("inspect.stack", side_effect=IndexError),
-            warnings.catch_warnings(record=True) as w,
+            pytest.warns(UserWarning, match="Error detecting parent module"),
         ):
-            DeprecatedName = create_deprecated_class("DeprecatedName", NewName)
+            create_deprecated_class("DeprecatedName", NewName)
-
-            class SubClass(DeprecatedName):
-                pass
-
-        assert "Error detecting parent module" in str(w[0].message)
 
 
 @mock.patch(
@@ -272,12 +262,12 @@ class TestWarnWhenSubclassed:
 )
 class TestUpdateClassPath:
     def test_old_path_gets_fixed(self):
-        with warnings.catch_warnings(record=True) as w:
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="`scrapy.contrib.debug.Debug` class is deprecated, use `scrapy.extensions.debug.Debug` instead",
+        ):
             output = update_classpath("scrapy.contrib.debug.Debug")
         assert output == "scrapy.extensions.debug.Debug"
-        assert len(w) == 1
-        assert "scrapy.contrib.debug.Debug" in str(w[0].message)
-        assert "scrapy.extensions.debug.Debug" in str(w[0].message)
 
     def test_sorted_replacement(self):
         with warnings.catch_warnings():
@@ -286,10 +276,10 @@ class TestUpdateClassPath:
         assert output == "scrapy.pipelines.Pipeline"
 
     def test_unmatched_path_stays_the_same(self):
-        with warnings.catch_warnings(record=True) as w:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", ScrapyDeprecationWarning)
             output = update_classpath("scrapy.unmatched.Path")
         assert output == "scrapy.unmatched.Path"
-        assert len(w) == 0
 
     def test_returns_nonstring(self):
         for notastring in [None, True, [1, 2, 3], object()]:

tests/test_utils_misc/test_return_with_argument_inside_generator.py

@@ -93,29 +93,27 @@ https://example.org
         assert is_generator_with_return_value(h1)
         assert is_generator_with_return_value(i1)
 
-        with warnings.catch_warnings(record=True) as w:
+        with pytest.warns(
+            UserWarning,
+            match='The "MockSpider.top_level_return_something" method is a generator',
+        ):
             warn_on_generator_with_return_value(mock_spider, top_level_return_something)
-            assert len(w) == 1
+        with pytest.warns(
-            assert (
+            UserWarning, match='The "MockSpider.f1" method is a generator'
-                'The "MockSpider.top_level_return_something" method is a generator'
+        ):
-                in str(w[0].message)
-            )
-        with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(mock_spider, f1)
-            assert len(w) == 1
+        with pytest.warns(
-            assert 'The "MockSpider.f1" method is a generator' in str(w[0].message)
+            UserWarning, match='The "MockSpider.g1" method is a generator'
-        with warnings.catch_warnings(record=True) as w:
+        ):
             warn_on_generator_with_return_value(mock_spider, g1)
-            assert len(w) == 1
+        with pytest.warns(
-            assert 'The "MockSpider.g1" method is a generator' in str(w[0].message)
+            UserWarning, match='The "MockSpider.h1" method is a generator'
-        with warnings.catch_warnings(record=True) as w:
+        ):
             warn_on_generator_with_return_value(mock_spider, h1)
-            assert len(w) == 1
+        with pytest.warns(
-            assert 'The "MockSpider.h1" method is a generator' in str(w[0].message)
+            UserWarning, match='The "MockSpider.i1" method is a generator'
-        with warnings.catch_warnings(record=True) as w:
+        ):
             warn_on_generator_with_return_value(mock_spider, i1)
-            assert len(w) == 1
-            assert 'The "MockSpider.i1" method is a generator' in str(w[0].message)
 
     def test_generators_return_none(self, mock_spider):
         def f2():
@@ -160,32 +158,18 @@ https://example.org
         assert not is_generator_with_return_value(k2)  # not recursive
         assert not is_generator_with_return_value(l2)
 
-        with warnings.catch_warnings(record=True) as w:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", UserWarning)
             warn_on_generator_with_return_value(mock_spider, top_level_return_none)
-            assert len(w) == 0
-        with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(mock_spider, f2)
-            assert len(w) == 0
-        with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(mock_spider, g2)
-            assert len(w) == 0
-        with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(mock_spider, h2)
-            assert len(w) == 0
-        with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(mock_spider, i2)
-            assert len(w) == 0
-        with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(mock_spider, j2)
-            assert len(w) == 0
-        with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(mock_spider, k2)
-            assert len(w) == 0
-        with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(mock_spider, l2)
-            assert len(w) == 0
 
-    def test_generators_return_none_with_decorator(self, mock_spider):  # noqa: PLR0915
+    def test_generators_return_none_with_decorator(self, mock_spider):
         def decorator(func):
             def inner_func():
                 func()
@@ -241,39 +225,23 @@ https://example.org
         assert not is_generator_with_return_value(k3)  # not recursive
         assert not is_generator_with_return_value(l3)
 
-        with warnings.catch_warnings(record=True) as w:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", UserWarning)
             warn_on_generator_with_return_value(mock_spider, top_level_return_none)
-            assert len(w) == 0
-        with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(mock_spider, f3)
-            assert len(w) == 0
-        with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(mock_spider, g3)
-            assert len(w) == 0
-        with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(mock_spider, h3)
-            assert len(w) == 0
-        with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(mock_spider, i3)
-            assert len(w) == 0
-        with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(mock_spider, j3)
-            assert len(w) == 0
-        with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(mock_spider, k3)
-            assert len(w) == 0
-        with warnings.catch_warnings(record=True) as w:
             warn_on_generator_with_return_value(mock_spider, l3)
-            assert len(w) == 0
 
     @mock.patch(
         "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error
     )
     def test_indentation_error(self, mock_spider):
-        with warnings.catch_warnings(record=True) as w:
+        with pytest.warns(UserWarning, match="Unable to determine"):
             warn_on_generator_with_return_value(mock_spider, top_level_return_none)
-            assert len(w) == 1
-            assert "Unable to determine" in str(w[0].message)
 
     def test_partial(self):
         def cb(arg1, arg2):
@@ -300,13 +268,11 @@ https://example.org
             yield 1
             return "value"
 
-        with warnings.catch_warnings(record=True) as w:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", UserWarning)
             warn_on_generator_with_return_value(spider, gen_with_return)
-            assert len(w) == 0
 
         spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True
 
-        with warnings.catch_warnings(record=True) as w:
+        with pytest.warns(UserWarning, match="is a generator"):
             warn_on_generator_with_return_value(spider, gen_with_return)
-            assert len(w) == 1
-            assert "is a generator" in str(w[0].message)

tests/test_utils_sitemap.py

@@ -1,5 +1,6 @@
-import warnings
+import pytest
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
 
 
@@ -204,7 +205,10 @@ Disallow: /forum/search/
 Disallow: /forum/active/
 """
 
-    with warnings.catch_warnings(record=True) as w:
+    with pytest.warns(
+        ScrapyDeprecationWarning,
+        match="Passing `str` type as `robots_text` is deprecated",
+    ):
         assert list(
             sitemap_urls_from_robots(robots, base_url="http://example.com")
         ) == [
@@ -213,9 +217,6 @@ Disallow: /forum/active/
             "http://example.com/sitemap-uppercase.xml",
             "http://example.com/sitemap-relative-url.xml",
         ]
-        assert "Passing `str` type as `robots_text` is deprecated, use `bytes`" in str(
-            w[0].message
-        )
 
 
 def test_sitemap_blanklines():