diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 6c5acf0de..2c9452174 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, cast from urllib.parse import urlparse from warnings import warn -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.spidermiddlewares.base import BaseSpiderMiddleware from scrapy.utils.misc import load_object @@ -349,7 +349,7 @@ class RefererMiddleware(BaseSpiderMiddleware): response = kwargs.pop("resp_or_url") warn( "Passing 'resp_or_url' is deprecated, use 'response' instead.", - DeprecationWarning, + ScrapyDeprecationWarning, stacklevel=2, ) if response is None: @@ -360,7 +360,7 @@ class RefererMiddleware(BaseSpiderMiddleware): warn( "Passing a response URL to RefererMiddleware.policy() instead " "of a Response object is deprecated.", - DeprecationWarning, + ScrapyDeprecationWarning, stacklevel=2, ) allow_import_path = True diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index d2da31f35..373c0b8b5 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -110,6 +110,7 @@ class CrawlSpider(Spider): "deprecated: it will be removed in future Scrapy releases. " "Please override the CrawlSpider.parse_with_rules method " "instead.", + ScrapyDeprecationWarning, stacklevel=2, ) @@ -199,6 +200,7 @@ class CrawlSpider(Spider): "The CrawlSpider._parse_response method is deprecated: " "it will be removed in future Scrapy releases. " "Please use the CrawlSpider.parse_with_rules method instead.", + ScrapyDeprecationWarning, stacklevel=2, ) return self.parse_with_rules(response, callback, cb_kwargs, follow) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 31d195c4b..c646aea4b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio import logging import re -import warnings from pathlib import Path from typing import Any, ClassVar @@ -88,9 +87,7 @@ class TestCrawler(TestBaseCrawler): self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") def test_crawler_accepts_None(self) -> None: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - crawler = Crawler(DefaultSpider) + crawler = Crawler(DefaultSpider) self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") def test_crawler_rejects_spider_objects(self) -> None: diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 412a59fcd..5d79691b2 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -3,8 +3,8 @@ import shutil import sys import tempfile from pathlib import Path -from warnings import catch_warnings +import pytest from testfixtures import LogCapture from scrapy.core.scheduler import Scheduler @@ -260,11 +260,8 @@ class TestBaseDupeFilter: dupefilter = _get_dupefilter( settings={"DUPEFILTER_CLASS": BaseDupeFilter}, ) - with catch_warnings(record=True) as warning_list: + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Calling BaseDupeFilter\.log\(\) is deprecated.", + ): dupefilter.log(None, None) - assert len(warning_list) == 1 - assert ( - str(warning_list[0].message) - == "Calling BaseDupeFilter.log() is deprecated." - ) - assert warning_list[0].category == ScrapyDeprecationWarning diff --git a/tests/test_http_request.py b/tests/test_http_request.py index fd494504d..e22689d49 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -460,11 +460,13 @@ class TestRequest: def test_from_curl_ignore_unknown_options(self): # By default: it works and ignores the unknown options: --foo and -z with warnings.catch_warnings(): # avoid warning when executing tests - warnings.simplefilter("ignore") + warnings.filterwarnings( + "ignore", category=UserWarning, message="Unrecognized options:" + ) r = self.request_class.from_curl( 'curl -X DELETE "http://example.org" --foo -z', ) - assert r.method == "DELETE" + assert r.method == "DELETE" # If `ignore_unknown_options` is set to `False` it raises an error with # the unknown options: --foo and -z diff --git a/tests/test_http_request_json.py b/tests/test_http_request_json.py index fcfe78365..65022afe1 100644 --- a/tests/test_http_request_json.py +++ b/tests/test_http_request_json.py @@ -4,6 +4,8 @@ import json import warnings from unittest import mock +import pytest + from scrapy.http import JsonRequest from scrapy.utils.python import to_bytes from tests.test_http_request import TestRequest @@ -63,40 +65,40 @@ class TestJsonRequest(TestRequest): data = { "name": "value", } - with warnings.catch_warnings(record=True) as _warnings: + with pytest.warns(UserWarning, match="data will be ignored"): r5 = self.request_class(url="http://www.example.com/", body=body, data=data) - assert r5.body == body - assert r5.method == "GET" - assert len(_warnings) == 1 - assert "data will be ignored" in str(_warnings[0].message) + assert r5.body == body + assert r5.method == "GET" def test_empty_body_data(self): """passing any body value and data should result a warning""" data = { "name": "value", } - with warnings.catch_warnings(record=True) as _warnings: + with pytest.warns(UserWarning, match="data will be ignored"): r6 = self.request_class(url="http://www.example.com/", body=b"", data=data) - assert r6.body == b"" - assert r6.method == "GET" - assert len(_warnings) == 1 - assert "data will be ignored" in str(_warnings[0].message) + assert r6.body == b"" + assert r6.method == "GET" def test_body_none_data(self): data = { "name": "value", } - with warnings.catch_warnings(record=True) as _warnings: + with warnings.catch_warnings(): + warnings.filterwarnings( + "error", category=UserWarning, message="Both body and data passed" + ) r7 = self.request_class(url="http://www.example.com/", body=None, data=data) - assert r7.body == to_bytes(json.dumps(data)) - assert r7.method == "POST" - assert len(_warnings) == 0 + assert r7.body == to_bytes(json.dumps(data)) + assert r7.method == "POST" def test_body_data_none(self): - with warnings.catch_warnings(record=True) as _warnings: + with warnings.catch_warnings(): + warnings.filterwarnings( + "error", category=UserWarning, message="Both body and data passed" + ) r8 = self.request_class(url="http://www.example.com/", body=None, data=None) - assert r8.method == "GET" - assert len(_warnings) == 0 + assert r8.method == "GET" def test_dumps_sort_keys(self): """Test that sort_keys=True is passed to json.dumps by default""" @@ -183,8 +185,5 @@ class TestJsonRequest(TestRequest): } r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1) - with warnings.catch_warnings(record=True) as _warnings: + with pytest.warns(UserWarning, match="data will be ignored"): r1.replace(data=data2, body=body2) - assert "Both body and data passed. data will be ignored" in str( - _warnings[0].message - ) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index c6a41fa6e..3829a35f5 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -2,7 +2,6 @@ import dataclasses import os import random import time -import warnings from abc import ABC, abstractmethod from datetime import datetime from ftplib import FTP @@ -786,12 +785,10 @@ class TestBuildFromCrawler: class Pipeline(FilesPipeline): pass - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 0 - assert pipe.store + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert pipe.store def test_has_from_crawler_and_init(self): class Pipeline(FilesPipeline): @@ -805,13 +802,11 @@ class TestBuildFromCrawler: o._from_crawler_called = True return o - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 0 - assert pipe.store - assert pipe._from_crawler_called + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert pipe.store + assert pipe._from_crawler_called @pytest.mark.parametrize("store", [None, ""]) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index da1bfa317..8ac949da7 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,6 +1,5 @@ from __future__ import annotations -import warnings from unittest.mock import MagicMock import pytest @@ -425,11 +424,9 @@ class TestBuildFromCrawler: class Pipeline(UserDefinedPipeline): pass - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 0 + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter def test_has_from_crawler_and_init(self): class Pipeline(UserDefinedPipeline): @@ -447,13 +444,11 @@ class TestBuildFromCrawler: o._from_crawler_called = True return o - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 0 - assert pipe._from_crawler_called - assert pipe._init_called + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert pipe._from_crawler_called + assert pipe._init_called def test_has_from_crawler(self): class Pipeline(UserDefinedPipeline): @@ -467,13 +462,10 @@ class TestBuildFromCrawler: o.store_uri = settings["FILES_STORE"] return o - with warnings.catch_warnings(record=True) as w: - pipe = Pipeline.from_crawler(self.crawler) - # this and the next assert will fail as MediaPipeline.from_crawler() wasn't called - assert pipe.crawler == self.crawler - assert pipe._fingerprinter - assert len(w) == 0 - assert pipe._from_crawler_called + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + assert pipe._from_crawler_called class MediaFailedFailurePipeline(MockedMediaPipeline): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index eb88baf01..8637de41e 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -12,6 +12,7 @@ import pytest from scrapy.core.downloader import Downloader from scrapy.core.scheduler import BaseScheduler, Scheduler from scrapy.crawler import Crawler +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.spiders import Spider from scrapy.utils.defer import ensure_awaitable @@ -396,7 +397,11 @@ class TestIncompatibility: def test_incompatibility(self): with warnings.catch_warnings(): - warnings.filterwarnings("ignore") + warnings.filterwarnings( + "ignore", + category=ScrapyDeprecationWarning, + message="The CONCURRENT_REQUESTS_PER_IP setting is deprecated", + ) with pytest.raises( ValueError, match="does not support CONCURRENT_REQUESTS_PER_IP" ): diff --git a/tests/test_scrapy__getattr__.py b/tests/test_scrapy__getattr__.py index 4c365859b..4647e1041 100644 --- a/tests/test_scrapy__getattr__.py +++ b/tests/test_scrapy__getattr__.py @@ -1,15 +1,18 @@ -import warnings +from __future__ import annotations + +import pytest + +from scrapy.exceptions import ScrapyDeprecationWarning -def test_deprecated_concurrent_requests_per_ip_attribute(): - with warnings.catch_warnings(record=True) as warns: +def test_deprecated_concurrent_requests_per_ip_attribute() -> None: + with pytest.warns( + ScrapyDeprecationWarning, + match=r"scrapy\.settings\.default_settings\.CONCURRENT_REQUESTS_PER_IP attribute is deprecated", + ): from scrapy.settings.default_settings import ( # noqa: PLC0415 CONCURRENT_REQUESTS_PER_IP, ) - assert CONCURRENT_REQUESTS_PER_IP is not None - assert isinstance(CONCURRENT_REQUESTS_PER_IP, int) - assert ( - "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead." - in warns[0].message.args - ) + assert CONCURRENT_REQUESTS_PER_IP is not None + assert isinstance(CONCURRENT_REQUESTS_PER_IP, int) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 9599ed6c9..908bb417c 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -2,12 +2,12 @@ # (too many false positives) import logging -import warnings from unittest import mock import pytest from scrapy.core.downloader.handlers.file import FileDownloadHandler +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import ( SETTINGS_PRIORITIES, BaseSettings, @@ -711,15 +711,13 @@ def test_remove_from_list(before, name, item, after): def test_deprecated_concurrent_requests_per_ip_setting(): - with warnings.catch_warnings(record=True) as warns: - settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1}) + settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1}) + with pytest.warns( + ScrapyDeprecationWarning, + match="The CONCURRENT_REQUESTS_PER_IP setting is deprecated", + ): settings.get("CONCURRENT_REQUESTS_PER_IP") - assert ( - str(warns[0].message) - == "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead." - ) - class Component1: pass diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py index c97934b74..2eeae110b 100644 --- a/tests/test_spider_crawl.py +++ b/tests/test_spider_crawl.py @@ -8,6 +8,7 @@ import pytest from testfixtures import LogCapture from w3lib.url import safe_url_string +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Request, TextResponse from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule, Spider @@ -264,13 +265,16 @@ class TestCrawlSpider(TestSpider): start_urls = "https://www.example.com" _follow_links = False - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", category=ScrapyDeprecationWarning) spider = _CrawlSpider() - assert len(w) == 0 + with pytest.warns( + ScrapyDeprecationWarning, + match=r"CrawlSpider\._parse_response method is deprecated", + ): spider._parse_response( TextResponse(spider.start_urls, body=b""), None, None ) - assert len(w) == 1 def test_parse_response_override(self): class _CrawlSpider(CrawlSpider): @@ -281,26 +285,28 @@ class TestCrawlSpider(TestSpider): start_urls = "https://www.example.com" _follow_links = False - with warnings.catch_warnings(record=True) as w: - assert len(w) == 0 + with pytest.warns( + ScrapyDeprecationWarning, + match=r"CrawlSpider\._parse_response method, which the", + ): spider = _CrawlSpider() - assert len(w) == 1 + with warnings.catch_warnings(): + warnings.simplefilter("error", category=ScrapyDeprecationWarning) spider._parse_response( TextResponse(spider.start_urls, body=b""), None, None ) - assert len(w) == 1 def test_parse_with_rules(self): class _CrawlSpider(CrawlSpider): name = "test" start_urls = "https://www.example.com" - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", category=ScrapyDeprecationWarning) spider = _CrawlSpider() spider.parse_with_rules( TextResponse(spider.start_urls, body=b""), None, None ) - assert len(w) == 0 class TestDeprecation: diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index de27ad519..d85942f90 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -1,7 +1,6 @@ import contextlib import shutil import sys -import warnings from pathlib import Path from unittest import mock @@ -137,50 +136,38 @@ class TestSpiderLoader: SpiderLoader.from_settings(settings) def test_bad_spider_modules_warning(self): - with warnings.catch_warnings(record=True) as w: - module = "tests.test_spiderloader.test_spiders.doesnotexist" - settings = Settings( - {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} - ) + module = "tests.test_spiderloader.test_spiders.doesnotexist" + settings = Settings( + {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} + ) + with pytest.warns(RuntimeWarning, match="Could not load spiders from module"): spider_loader = SpiderLoader.from_settings(settings) - if str(w[0].message).startswith("_SixMetaPathImporter"): - # needed on 3.10 because of https://github.com/benjaminp/six/issues/349, - # at least until all six versions we can import (including botocore.vendored.six) - # are updated to 1.16.0+ - w.pop(0) - assert "Could not load spiders from module" in str(w[0].message) - spiders = spider_loader.list() - assert not spiders + spiders = spider_loader.list() + assert not spiders def test_syntax_error_exception(self): module = "tests.test_spiderloader.test_spiders.spider1" + settings = Settings({"SPIDER_MODULES": [module]}) with mock.patch.object(SpiderLoader, "_load_spiders") as m: m.side_effect = SyntaxError - settings = Settings({"SPIDER_MODULES": [module]}) with pytest.raises(SyntaxError): SpiderLoader.from_settings(settings) def test_syntax_error_warning(self): - with ( - warnings.catch_warnings(record=True) as w, - mock.patch.object(SpiderLoader, "_load_spiders") as m, - ): + module = "tests.test_spiderloader.test_spiders.spider1" + settings = Settings( + {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} + ) + with mock.patch.object(SpiderLoader, "_load_spiders") as m: m.side_effect = SyntaxError - module = "tests.test_spiderloader.test_spiders.spider1" - settings = Settings( - {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} - ) - spider_loader = SpiderLoader.from_settings(settings) - if str(w[0].message).startswith("_SixMetaPathImporter"): - # needed on 3.10 because of https://github.com/benjaminp/six/issues/349, - # at least until all six versions we can import (including botocore.vendored.six) - # are updated to 1.16.0+ - w.pop(0) - assert "Could not load spiders from module" in str(w[0].message) + with pytest.warns( + RuntimeWarning, match="Could not load spiders from module" + ): + spider_loader = SpiderLoader.from_settings(settings) - spiders = spider_loader.list() - assert not spiders + spiders = spider_loader.list() + assert not spiders class TestDuplicateSpiderNameLoader: @@ -190,21 +177,17 @@ class TestDuplicateSpiderNameLoader: # copy 1 spider module so as to have duplicate spider name shutil.copyfile(spiders_dir / "spider3.py", spiders_dir / "spider3dupe.py") - with warnings.catch_warnings(record=True) as w: + msg = r"""There are several spiders with the same name: + + Spider3 named 'spider3' \(in test_spiders_xxx\.spider3\) + + Spider3 named 'spider3' \(in test_spiders_xxx\.spider3dupe\) + + This can cause unexpected behavior\.""" + with pytest.warns(UserWarning, match=msg): spider_loader = SpiderLoader.from_settings(settings) - - assert len(w) == 1 - msg = str(w[0].message) - assert "several spiders with the same name" in msg - assert "'spider3'" in msg - assert msg.count("'spider3'") == 2 - - assert "'spider1'" not in msg - assert "'spider2'" not in msg - assert "'spider4'" not in msg - - spiders = set(spider_loader.list()) - assert spiders == {"spider1", "spider2", "spider3", "spider4"} + spiders = set(spider_loader.list()) + assert spiders == {"spider1", "spider2", "spider3", "spider4"} def test_multiple_dupename_warning(self, spider_loader_env): settings, spiders_dir = spider_loader_env @@ -213,23 +196,21 @@ class TestDuplicateSpiderNameLoader: shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py") shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py") - with warnings.catch_warnings(record=True) as w: + msg = r"""There are several spiders with the same name: + + Spider1 named 'spider1' \(in test_spiders_xxx\.spider1\) + + Spider1 named 'spider1' \(in test_spiders_xxx\.spider1dupe\) + + Spider2 named 'spider2' \(in test_spiders_xxx\.spider2\) + + Spider2 named 'spider2' \(in test_spiders_xxx\.spider2dupe\) + + This can cause unexpected behavior\.""" + with pytest.warns(UserWarning, match=msg): spider_loader = SpiderLoader.from_settings(settings) - - assert len(w) == 1 - msg = str(w[0].message) - assert "several spiders with the same name" in msg - assert "'spider1'" in msg - assert msg.count("'spider1'") == 2 - - assert "'spider2'" in msg - assert msg.count("'spider2'") == 2 - - assert "'spider3'" not in msg - assert "'spider4'" not in msg - - spiders = set(spider_loader.list()) - assert spiders == {"spider1", "spider2", "spider3", "spider4"} + spiders = set(spider_loader.list()) + assert spiders == {"spider1", "spider2", "spider3", "spider4"} class CustomSpiderLoader(SpiderLoader): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index a9089419a..b3a434c3f 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -6,6 +6,7 @@ from urllib.parse import urlparse import pytest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spidermiddlewares.referer import ( @@ -842,13 +843,14 @@ class TestRequestMetaSettingFallback: response = Response(origin, headers=response_headers) request = Request(target, meta=request_meta) - with warnings.catch_warnings(record=True) as w: + if check_warning: + with pytest.warns( + RuntimeWarning, match="Could not load referrer policy" + ): + policy = mw.policy(response, request) + else: policy = mw.policy(response, request) - assert isinstance(policy, policy_class) - - if check_warning: - assert len(w) == 1 - assert w[0].category is RuntimeWarning, w[0].message + assert isinstance(policy, policy_class) class TestSettingsPolicyByName: @@ -973,49 +975,39 @@ class TestPolicyMethodResponseParamRename: self.response = Response("http://www.example.com") def test_pos_string(self): - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a response URL to RefererMiddleware\.policy\(\)", + ): self.mw.policy("http://old.com", self.request) - found = False - for warning in w: - if "Passing a response URL" in str(warning.message): - found = True - break - assert found def test_pos_response(self): - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.filterwarnings( + "error", + category=ScrapyDeprecationWarning, + message=r"Passing 'resp_or_url' is deprecated", + ) self.mw.policy(self.response, self.request) - for warning in w: - assert "resp_or_url" not in str(warning.message) def test_key_resp_or_url(self): - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + ScrapyDeprecationWarning, match=r"Passing 'resp_or_url' is deprecated" + ): self.mw.policy(resp_or_url=self.response, request=self.request) - found = False - for warning in w: - if "Passing 'resp_or_url' is deprecated, use 'response' instead" in str( - warning.message - ): - found = True - break - assert found def test_key_response(self): - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.filterwarnings( + "error", + category=ScrapyDeprecationWarning, + message=r"Passing 'resp_or_url' is deprecated", + ) self.mw.policy(response=self.response, request=self.request) - for warning in w: - assert "resp_or_url" not in str(warning.message) def test_key_response_string(self): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") + with pytest.warns(ScrapyDeprecationWarning, match="Passing a response URL"): self.mw.policy(response="http://old.com", request=self.request) - found = False - for warning in w: - if "Passing a response URL" in str(warning.message): - found = True - break - assert found def test_both_resp_or_url_and_response(self): with pytest.raises( diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index b1532ca77..0627a4b93 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -213,10 +213,12 @@ class TestCurlToRequestKwargs: def test_ignore_unknown_options(self): # case 1: ignore_unknown_options=True: + curl_command = "curl --bar --baz http://www.example.com" + expected_result = {"method": "GET", "url": "http://www.example.com"} with warnings.catch_warnings(): # avoid warning when executing tests - warnings.simplefilter("ignore") - curl_command = "curl --bar --baz http://www.example.com" - expected_result = {"method": "GET", "url": "http://www.example.com"} + warnings.filterwarnings( + "ignore", category=UserWarning, message="Unrecognized options:" + ) assert curl_to_request_kwargs(curl_command) == expected_result # case 2: ignore_unknown_options=False (raise exception): diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index f573993a1..ba6b82503 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,5 +1,4 @@ import copy -import warnings from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, MutableMapping from typing import Any @@ -227,18 +226,12 @@ class TestCaselessDict(TestCaseInsensitiveDictBase): dict_class = CaselessDict def test_deprecation_message(self): - with warnings.catch_warnings(record=True) as caught: - warnings.filterwarnings("always", category=ScrapyDeprecationWarning) + with pytest.warns( + ScrapyDeprecationWarning, + match=r"scrapy.utils.datatypes.CaselessDict is deprecated", + ): self.dict_class({"foo": "bar"}) - assert len(caught) == 1 - assert issubclass(caught[0].category, ScrapyDeprecationWarning) - assert ( - str(caught[0].message) - == "scrapy.utils.datatypes.CaselessDict is deprecated," - " please use scrapy.utils.datatypes.CaseInsensitiveDict instead" - ) - class TestSequenceExclude: def test_list(self): diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 5ea6f678e..0706fec99 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,7 +1,6 @@ import inspect import warnings from unittest import mock -from warnings import WarningMessage import pytest @@ -22,34 +21,26 @@ class NewName(SomeBaseClass): class TestWarnWhenSubclassed: - def _mywarnings(self, w: list[WarningMessage]) -> list[WarningMessage]: - return [x for x in w if x.category is MyWarning] - def test_no_warning_on_definition(self): - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", category=ScrapyDeprecationWarning) create_deprecated_class("Deprecated", NewName) - w = self._mywarnings(w) - assert w == [] - def test_subclassing_warning_message(self): + msg = ( + r"tests\.test_utils_deprecate\.UserClass inherits from " + r"deprecated class tests\.test_utils_deprecate\.Deprecated, " + r"please inherit from tests\.test_utils_deprecate\.NewName." + r" \(warning only on first subclass, there may be others\)" + ) Deprecated = create_deprecated_class( "Deprecated", NewName, warn_category=MyWarning ) - - with warnings.catch_warnings(record=True) as w: + with pytest.warns(MyWarning, match=msg) as w: class UserClass(Deprecated): pass - w = self._mywarnings(w) - assert len(w) == 1 - assert ( - str(w[0].message) == "tests.test_utils_deprecate.UserClass inherits from " - "deprecated class tests.test_utils_deprecate.Deprecated, " - "please inherit from tests.test_utils_deprecate.NewName." - " (warning only on first subclass, there may be others)" - ) assert w[0].lineno == inspect.getsourcelines(UserClass)[1] def test_custom_class_paths(self): @@ -61,62 +52,77 @@ class TestWarnWhenSubclassed: warn_category=MyWarning, ) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + MyWarning, + match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass", + ): class UserClass(Deprecated): pass + with pytest.warns( + MyWarning, + match=r"bar\.OldClass is deprecated, instantiate foo\.NewClass instead", + ): _ = Deprecated() - w = self._mywarnings(w) - assert len(w) == 2 - assert "foo.NewClass" in str(w[0].message) - assert "bar.OldClass" in str(w[0].message) - assert "foo.NewClass" in str(w[1].message) - assert "bar.OldClass" in str(w[1].message) - def test_subclassing_warns_only_on_direct_children(self): Deprecated = create_deprecated_class( "Deprecated", NewName, warn_once=False, warn_category=MyWarning ) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + MyWarning, + match="UserClass inherits from deprecated class", + ): class UserClass(Deprecated): pass + with warnings.catch_warnings(): + warnings.simplefilter("error", MyWarning) + class NoWarnOnMe(UserClass): pass - w = self._mywarnings(w) - assert len(w) == 1 - assert "UserClass" in str(w[0].message) - def test_subclassing_warns_once_by_default(self): Deprecated = create_deprecated_class( "Deprecated", NewName, warn_category=MyWarning ) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + MyWarning, + match="UserClass inherits from deprecated class", + ): class UserClass(Deprecated): pass + with warnings.catch_warnings(): + warnings.simplefilter("error", MyWarning) + class FooClass(Deprecated): pass class BarClass(Deprecated): pass - w = self._mywarnings(w) - assert len(w) == 1 - assert "UserClass" in str(w[0].message) - def test_warning_on_instance(self): Deprecated = create_deprecated_class( "Deprecated", NewName, warn_category=MyWarning ) + with pytest.warns(MyWarning) as w: + _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) + + w = [x for x in w if x.category is MyWarning] + assert len(w) == 1 + assert ( + str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, " + "instantiate tests.test_utils_deprecate.NewName instead." + ) + assert w[0].lineno == lineno + # ignore subclassing warnings with warnings.catch_warnings(): warnings.simplefilter("ignore", MyWarning) @@ -124,29 +130,20 @@ class TestWarnWhenSubclassed: class UserClass(Deprecated): pass - with warnings.catch_warnings(record=True) as w: - _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) - _ = UserClass() # subclass instances don't warn - - w = self._mywarnings(w) - assert len(w) == 1 - assert ( - str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, " - "instantiate tests.test_utils_deprecate.NewName instead." - ) - assert w[0].lineno == lineno + with warnings.catch_warnings(): + warnings.simplefilter("error", MyWarning) + UserClass() # subclass instances don't warn def test_warning_auto_message(self): - with warnings.catch_warnings(record=True) as w: - Deprecated = create_deprecated_class("Deprecated", NewName) + Deprecated = create_deprecated_class("Deprecated", NewName) + with pytest.warns( + ScrapyDeprecationWarning, + match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName", + ): class UserClass2(Deprecated): pass - msg = str(w[0].message) - assert "tests.test_utils_deprecate.NewName" in msg - assert "tests.test_utils_deprecate.Deprecated" in msg - def test_issubclass(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", ScrapyDeprecationWarning) @@ -222,8 +219,8 @@ class TestWarnWhenSubclassed: create_deprecated_class("Deprecated", New) def test_deprecate_subclass_of_deprecated_class(self): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") + with warnings.catch_warnings(): + warnings.simplefilter("error", MyWarning) Deprecated = create_deprecated_class( "Deprecated", NewName, warn_category=MyWarning ) @@ -234,33 +231,26 @@ class TestWarnWhenSubclassed: warn_category=MyWarning, ) - w = self._mywarnings(w) - assert len(w) == 0, [str(warning) for warning in w] - - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + MyWarning, + match=r"AlsoDeprecated is deprecated, instantiate foo\.Bar instead", + ): AlsoDeprecated() + with pytest.warns( + MyWarning, + match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar", + ): + class UserClass(AlsoDeprecated): pass - w = self._mywarnings(w) - assert len(w) == 2 - assert "AlsoDeprecated" in str(w[0].message) - assert "foo.Bar" in str(w[0].message) - assert "AlsoDeprecated" in str(w[1].message) - assert "foo.Bar" in str(w[1].message) - def test_inspect_stack(self): with ( mock.patch("inspect.stack", side_effect=IndexError), - warnings.catch_warnings(record=True) as w, + pytest.warns(UserWarning, match="Error detecting parent module"), ): - DeprecatedName = create_deprecated_class("DeprecatedName", NewName) - - class SubClass(DeprecatedName): - pass - - assert "Error detecting parent module" in str(w[0].message) + create_deprecated_class("DeprecatedName", NewName) @mock.patch( @@ -272,12 +262,12 @@ class TestWarnWhenSubclassed: ) class TestUpdateClassPath: def test_old_path_gets_fixed(self): - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + ScrapyDeprecationWarning, + match="`scrapy.contrib.debug.Debug` class is deprecated, use `scrapy.extensions.debug.Debug` instead", + ): output = update_classpath("scrapy.contrib.debug.Debug") assert output == "scrapy.extensions.debug.Debug" - assert len(w) == 1 - assert "scrapy.contrib.debug.Debug" in str(w[0].message) - assert "scrapy.extensions.debug.Debug" in str(w[0].message) def test_sorted_replacement(self): with warnings.catch_warnings(): @@ -286,10 +276,10 @@ class TestUpdateClassPath: assert output == "scrapy.pipelines.Pipeline" def test_unmatched_path_stays_the_same(self): - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) output = update_classpath("scrapy.unmatched.Path") assert output == "scrapy.unmatched.Path" - assert len(w) == 0 def test_returns_nonstring(self): for notastring in [None, True, [1, 2, 3], object()]: diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py index 3783416b9..1acc3aac2 100644 --- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -93,29 +93,27 @@ https://example.org assert is_generator_with_return_value(h1) assert is_generator_with_return_value(i1) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + UserWarning, + match='The "MockSpider.top_level_return_something" method is a generator', + ): warn_on_generator_with_return_value(mock_spider, top_level_return_something) - assert len(w) == 1 - assert ( - 'The "MockSpider.top_level_return_something" method is a generator' - in str(w[0].message) - ) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + UserWarning, match='The "MockSpider.f1" method is a generator' + ): warn_on_generator_with_return_value(mock_spider, f1) - assert len(w) == 1 - assert 'The "MockSpider.f1" method is a generator' in str(w[0].message) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + UserWarning, match='The "MockSpider.g1" method is a generator' + ): warn_on_generator_with_return_value(mock_spider, g1) - assert len(w) == 1 - assert 'The "MockSpider.g1" method is a generator' in str(w[0].message) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + UserWarning, match='The "MockSpider.h1" method is a generator' + ): warn_on_generator_with_return_value(mock_spider, h1) - assert len(w) == 1 - assert 'The "MockSpider.h1" method is a generator' in str(w[0].message) - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + UserWarning, match='The "MockSpider.i1" method is a generator' + ): warn_on_generator_with_return_value(mock_spider, i1) - assert len(w) == 1 - assert 'The "MockSpider.i1" method is a generator' in str(w[0].message) def test_generators_return_none(self, mock_spider): def f2(): @@ -160,32 +158,18 @@ https://example.org assert not is_generator_with_return_value(k2) # not recursive assert not is_generator_with_return_value(l2) - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) warn_on_generator_with_return_value(mock_spider, top_level_return_none) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, f2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, g2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, h2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, i2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, j2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, k2) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, l2) - assert len(w) == 0 - def test_generators_return_none_with_decorator(self, mock_spider): # noqa: PLR0915 + def test_generators_return_none_with_decorator(self, mock_spider): def decorator(func): def inner_func(): func() @@ -241,39 +225,23 @@ https://example.org assert not is_generator_with_return_value(k3) # not recursive assert not is_generator_with_return_value(l3) - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) warn_on_generator_with_return_value(mock_spider, top_level_return_none) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, f3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, g3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, h3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, i3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, j3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, k3) - assert len(w) == 0 - with warnings.catch_warnings(record=True) as w: warn_on_generator_with_return_value(mock_spider, l3) - assert len(w) == 0 @mock.patch( "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error ) def test_indentation_error(self, mock_spider): - with warnings.catch_warnings(record=True) as w: + with pytest.warns(UserWarning, match="Unable to determine"): warn_on_generator_with_return_value(mock_spider, top_level_return_none) - assert len(w) == 1 - assert "Unable to determine" in str(w[0].message) def test_partial(self): def cb(arg1, arg2): @@ -300,13 +268,11 @@ https://example.org yield 1 return "value" - with warnings.catch_warnings(record=True) as w: + with warnings.catch_warnings(): + warnings.simplefilter("error", UserWarning) warn_on_generator_with_return_value(spider, gen_with_return) - assert len(w) == 0 spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True - with warnings.catch_warnings(record=True) as w: + with pytest.warns(UserWarning, match="is a generator"): warn_on_generator_with_return_value(spider, gen_with_return) - assert len(w) == 1 - assert "is a generator" in str(w[0].message) diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index f2bcd7541..3599c4824 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -1,5 +1,6 @@ -import warnings +import pytest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots @@ -204,7 +205,10 @@ Disallow: /forum/search/ Disallow: /forum/active/ """ - with warnings.catch_warnings(record=True) as w: + with pytest.warns( + ScrapyDeprecationWarning, + match="Passing `str` type as `robots_text` is deprecated", + ): assert list( sitemap_urls_from_robots(robots, base_url="http://example.com") ) == [ @@ -213,9 +217,6 @@ Disallow: /forum/active/ "http://example.com/sitemap-uppercase.xml", "http://example.com/sitemap-relative-url.xml", ] - assert "Passing `str` type as `robots_text` is deprecated, use `bytes`" in str( - w[0].message - ) def test_sitemap_blanklines():