RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor and improve catching warnings in tests. (#7643)

This commit is contained in:
Andrey Rakhmatullin 2026-06-20 00:04:34 +05:00 committed by GitHub
parent 6393858c7e
commit c9f952c258
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 275 additions and 354 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/spidermiddlewares/referer.py
View File

@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, cast
from urllib.parse import urlparse
from warnings import warn
from scrapy.exceptions import NotConfigured
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
from scrapy.utils.misc import load_object
@ -349,7 +349,7 @@ class RefererMiddleware(BaseSpiderMiddleware):
response = kwargs.pop("resp_or_url")
warn(
"Passing 'resp_or_url' is deprecated, use 'response' instead.",
DeprecationWarning,
ScrapyDeprecationWarning,
stacklevel=2,
)
if response is None:
@ -360,7 +360,7 @@ class RefererMiddleware(BaseSpiderMiddleware):
warn(
"Passing a response URL to RefererMiddleware.policy() instead "
"of a Response object is deprecated.",
DeprecationWarning,
ScrapyDeprecationWarning,
stacklevel=2,
)
allow_import_path = True

2
scrapy/spiders/crawl.py
View File

@ -110,6 +110,7 @@ class CrawlSpider(Spider):
"deprecated: it will be removed in future Scrapy releases. "
"Please override the CrawlSpider.parse_with_rules method "
"instead.",
ScrapyDeprecationWarning,
stacklevel=2,
)
@ -199,6 +200,7 @@ class CrawlSpider(Spider):
"The CrawlSpider._parse_response method is deprecated: "
"it will be removed in future Scrapy releases. "
"Please use the CrawlSpider.parse_with_rules method instead.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return self.parse_with_rules(response, callback, cb_kwargs, follow)

5
tests/test_crawler.py
View File

@ -3,7 +3,6 @@ from __future__ import annotations
import asyncio
import logging
import re
import warnings
from pathlib import Path
from typing import Any, ClassVar
@ -88,9 +87,7 @@ class TestCrawler(TestBaseCrawler):
self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
def test_crawler_accepts_None(self) -> None:
with warnings.catch_warnings():
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
crawler = Crawler(DefaultSpider)
crawler = Crawler(DefaultSpider)
self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
def test_crawler_rejects_spider_objects(self) -> None:

13
tests/test_dupefilters.py
View File

@ -3,8 +3,8 @@ import shutil
import sys
import tempfile
from pathlib import Path
from warnings import catch_warnings
import pytest
from testfixtures import LogCapture
from scrapy.core.scheduler import Scheduler
@ -260,11 +260,8 @@ class TestBaseDupeFilter:
dupefilter = _get_dupefilter(
settings={"DUPEFILTER_CLASS": BaseDupeFilter},
)
with catch_warnings(record=True) as warning_list:
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Calling BaseDupeFilter\.log\(\) is deprecated.",
):
dupefilter.log(None, None)
assert len(warning_list) == 1
assert (
str(warning_list[0].message)
== "Calling BaseDupeFilter.log() is deprecated."
)
assert warning_list[0].category == ScrapyDeprecationWarning

6
tests/test_http_request.py
View File

@ -460,11 +460,13 @@ class TestRequest:
def test_from_curl_ignore_unknown_options(self):
# By default: it works and ignores the unknown options: --foo and -z
with warnings.catch_warnings(): # avoid warning when executing tests
warnings.simplefilter("ignore")
warnings.filterwarnings(
"ignore", category=UserWarning, message="Unrecognized options:"
)
r = self.request_class.from_curl(
'curl -X DELETE "http://example.org" --foo -z',
)
assert r.method == "DELETE"
assert r.method == "DELETE"
# If `ignore_unknown_options` is set to `False` it raises an error with
# the unknown options: --foo and -z

41
tests/test_http_request_json.py
View File

@ -4,6 +4,8 @@ import json
import warnings
from unittest import mock
import pytest
from scrapy.http import JsonRequest
from scrapy.utils.python import to_bytes
from tests.test_http_request import TestRequest
@ -63,40 +65,40 @@ class TestJsonRequest(TestRequest):
data = {
"name": "value",
}
with warnings.catch_warnings(record=True) as _warnings:
with pytest.warns(UserWarning, match="data will be ignored"):
r5 = self.request_class(url="http://www.example.com/", body=body, data=data)
assert r5.body == body
assert r5.method == "GET"
assert len(_warnings) == 1
assert "data will be ignored" in str(_warnings[0].message)
assert r5.body == body
assert r5.method == "GET"
def test_empty_body_data(self):
"""passing any body value and data should result a warning"""
data = {
"name": "value",
}
with warnings.catch_warnings(record=True) as _warnings:
with pytest.warns(UserWarning, match="data will be ignored"):
r6 = self.request_class(url="http://www.example.com/", body=b"", data=data)
assert r6.body == b""
assert r6.method == "GET"
assert len(_warnings) == 1
assert "data will be ignored" in str(_warnings[0].message)
assert r6.body == b""
assert r6.method == "GET"
def test_body_none_data(self):
data = {
"name": "value",
}
with warnings.catch_warnings(record=True) as _warnings:
with warnings.catch_warnings():
warnings.filterwarnings(
"error", category=UserWarning, message="Both body and data passed"
)
r7 = self.request_class(url="http://www.example.com/", body=None, data=data)
assert r7.body == to_bytes(json.dumps(data))
assert r7.method == "POST"
assert len(_warnings) == 0
assert r7.body == to_bytes(json.dumps(data))
assert r7.method == "POST"
def test_body_data_none(self):
with warnings.catch_warnings(record=True) as _warnings:
with warnings.catch_warnings():
warnings.filterwarnings(
"error", category=UserWarning, message="Both body and data passed"
)
r8 = self.request_class(url="http://www.example.com/", body=None, data=None)
assert r8.method == "GET"
assert len(_warnings) == 0
assert r8.method == "GET"
def test_dumps_sort_keys(self):
"""Test that sort_keys=True is passed to json.dumps by default"""
@ -183,8 +185,5 @@ class TestJsonRequest(TestRequest):
}
r1 = self.request_class(url="http://www.example.com/", data=data1, body=body1)
with warnings.catch_warnings(record=True) as _warnings:
with pytest.warns(UserWarning, match="data will be ignored"):
r1.replace(data=data2, body=body2)
assert "Both body and data passed. data will be ignored" in str(
_warnings[0].message
)

23
tests/test_pipeline_files.py
View File

@ -2,7 +2,6 @@ import dataclasses
import os
import random
import time
import warnings
from abc import ABC, abstractmethod
from datetime import datetime
from ftplib import FTP
@ -786,12 +785,10 @@ class TestBuildFromCrawler:
class Pipeline(FilesPipeline):
pass
with warnings.catch_warnings(record=True) as w:
pipe = Pipeline.from_crawler(self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert len(w) == 0
assert pipe.store
pipe = Pipeline.from_crawler(self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert pipe.store
def test_has_from_crawler_and_init(self):
class Pipeline(FilesPipeline):
@ -805,13 +802,11 @@ class TestBuildFromCrawler:
o._from_crawler_called = True
return o
with warnings.catch_warnings(record=True) as w:
pipe = Pipeline.from_crawler(self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert len(w) == 0
assert pipe.store
assert pipe._from_crawler_called
pipe = Pipeline.from_crawler(self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert pipe.store
assert pipe._from_crawler_called
@pytest.mark.parametrize("store", [None, ""])

32
tests/test_pipeline_media.py
View File

@ -1,6 +1,5 @@
from __future__ import annotations
import warnings
from unittest.mock import MagicMock
import pytest
@ -425,11 +424,9 @@ class TestBuildFromCrawler:
class Pipeline(UserDefinedPipeline):
pass
with warnings.catch_warnings(record=True) as w:
pipe = Pipeline.from_crawler(self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert len(w) == 0
pipe = Pipeline.from_crawler(self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
def test_has_from_crawler_and_init(self):
class Pipeline(UserDefinedPipeline):
@ -447,13 +444,11 @@ class TestBuildFromCrawler:
o._from_crawler_called = True
return o
with warnings.catch_warnings(record=True) as w:
pipe = Pipeline.from_crawler(self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert len(w) == 0
assert pipe._from_crawler_called
assert pipe._init_called
pipe = Pipeline.from_crawler(self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert pipe._from_crawler_called
assert pipe._init_called
def test_has_from_crawler(self):
class Pipeline(UserDefinedPipeline):
@ -467,13 +462,10 @@ class TestBuildFromCrawler:
o.store_uri = settings["FILES_STORE"]
return o
with warnings.catch_warnings(record=True) as w:
pipe = Pipeline.from_crawler(self.crawler)
# this and the next assert will fail as MediaPipeline.from_crawler() wasn't called
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert len(w) == 0
assert pipe._from_crawler_called
pipe = Pipeline.from_crawler(self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert pipe._from_crawler_called
class MediaFailedFailurePipeline(MockedMediaPipeline):

7
tests/test_scheduler.py
View File

@ -12,6 +12,7 @@ import pytest
from scrapy.core.downloader import Downloader
from scrapy.core.scheduler import BaseScheduler, Scheduler
from scrapy.crawler import Crawler
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.defer import ensure_awaitable
@ -396,7 +397,11 @@ class TestIncompatibility:
def test_incompatibility(self):
with warnings.catch_warnings():
warnings.filterwarnings("ignore")
warnings.filterwarnings(
"ignore",
category=ScrapyDeprecationWarning,
message="The CONCURRENT_REQUESTS_PER_IP setting is deprecated",
)
with pytest.raises(
ValueError, match="does not support CONCURRENT_REQUESTS_PER_IP"
):

21
tests/test_scrapy__getattr__.py
View File

@ -1,15 +1,18 @@
import warnings
from __future__ import annotations
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
def test_deprecated_concurrent_requests_per_ip_attribute():
with warnings.catch_warnings(record=True) as warns:
def test_deprecated_concurrent_requests_per_ip_attribute() -> None:
with pytest.warns(
ScrapyDeprecationWarning,
match=r"scrapy\.settings\.default_settings\.CONCURRENT_REQUESTS_PER_IP attribute is deprecated",
):
from scrapy.settings.default_settings import ( # noqa: PLC0415
CONCURRENT_REQUESTS_PER_IP,
)
assert CONCURRENT_REQUESTS_PER_IP is not None
assert isinstance(CONCURRENT_REQUESTS_PER_IP, int)
assert (
"The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead."
in warns[0].message.args
)
assert CONCURRENT_REQUESTS_PER_IP is not None
assert isinstance(CONCURRENT_REQUESTS_PER_IP, int)

14
tests/test_settings/__init__.py
View File

@ -2,12 +2,12 @@
# (too many false positives)
import logging
import warnings
from unittest import mock
import pytest
from scrapy.core.downloader.handlers.file import FileDownloadHandler
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import (
SETTINGS_PRIORITIES,
BaseSettings,
@ -711,15 +711,13 @@ def test_remove_from_list(before, name, item, after):
def test_deprecated_concurrent_requests_per_ip_setting():
with warnings.catch_warnings(record=True) as warns:
settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1})
settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1})
with pytest.warns(
ScrapyDeprecationWarning,
match="The CONCURRENT_REQUESTS_PER_IP setting is deprecated",
):
settings.get("CONCURRENT_REQUESTS_PER_IP")
assert (
str(warns[0].message)
== "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead."
)
class Component1:
pass

24
tests/test_spider_crawl.py
View File

@ -8,6 +8,7 @@ import pytest
from testfixtures import LogCapture
from w3lib.url import safe_url_string
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Request, TextResponse
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule, Spider
@ -264,13 +265,16 @@ class TestCrawlSpider(TestSpider):
start_urls = "https://www.example.com"
_follow_links = False
with warnings.catch_warnings(record=True) as w:
with warnings.catch_warnings():
warnings.simplefilter("error", category=ScrapyDeprecationWarning)
spider = _CrawlSpider()
assert len(w) == 0
with pytest.warns(
ScrapyDeprecationWarning,
match=r"CrawlSpider\._parse_response method is deprecated",
):
spider._parse_response(
TextResponse(spider.start_urls, body=b""), None, None
)
assert len(w) == 1
def test_parse_response_override(self):
class _CrawlSpider(CrawlSpider):
@ -281,26 +285,28 @@ class TestCrawlSpider(TestSpider):
start_urls = "https://www.example.com"
_follow_links = False
with warnings.catch_warnings(record=True) as w:
assert len(w) == 0
with pytest.warns(
ScrapyDeprecationWarning,
match=r"CrawlSpider\._parse_response method, which the",
):
spider = _CrawlSpider()
assert len(w) == 1
with warnings.catch_warnings():
warnings.simplefilter("error", category=ScrapyDeprecationWarning)
spider._parse_response(
TextResponse(spider.start_urls, body=b""), None, None
)
assert len(w) == 1
def test_parse_with_rules(self):
class _CrawlSpider(CrawlSpider):
name = "test"
start_urls = "https://www.example.com"
with warnings.catch_warnings(record=True) as w:
with warnings.catch_warnings():
warnings.simplefilter("error", category=ScrapyDeprecationWarning)
spider = _CrawlSpider()
spider.parse_with_rules(
TextResponse(spider.start_urls, body=b""), None, None
)
assert len(w) == 0
class TestDeprecation:

105
tests/test_spiderloader/__init__.py
View File

@ -1,7 +1,6 @@
import contextlib
import shutil
import sys
import warnings
from pathlib import Path
from unittest import mock
@ -137,50 +136,38 @@ class TestSpiderLoader:
SpiderLoader.from_settings(settings)
def test_bad_spider_modules_warning(self):
with warnings.catch_warnings(record=True) as w:
module = "tests.test_spiderloader.test_spiders.doesnotexist"
settings = Settings(
{"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True}
)
module = "tests.test_spiderloader.test_spiders.doesnotexist"
settings = Settings(
{"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True}
)
with pytest.warns(RuntimeWarning, match="Could not load spiders from module"):
spider_loader = SpiderLoader.from_settings(settings)
if str(w[0].message).startswith("_SixMetaPathImporter"):
# needed on 3.10 because of https://github.com/benjaminp/six/issues/349,
# at least until all six versions we can import (including botocore.vendored.six)
# are updated to 1.16.0+
w.pop(0)
assert "Could not load spiders from module" in str(w[0].message)
spiders = spider_loader.list()
assert not spiders
spiders = spider_loader.list()
assert not spiders
def test_syntax_error_exception(self):
module = "tests.test_spiderloader.test_spiders.spider1"
settings = Settings({"SPIDER_MODULES": [module]})
with mock.patch.object(SpiderLoader, "_load_spiders") as m:
m.side_effect = SyntaxError
settings = Settings({"SPIDER_MODULES": [module]})
with pytest.raises(SyntaxError):
SpiderLoader.from_settings(settings)
def test_syntax_error_warning(self):
with (
warnings.catch_warnings(record=True) as w,
mock.patch.object(SpiderLoader, "_load_spiders") as m,
):
module = "tests.test_spiderloader.test_spiders.spider1"
settings = Settings(
{"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True}
)
with mock.patch.object(SpiderLoader, "_load_spiders") as m:
m.side_effect = SyntaxError
module = "tests.test_spiderloader.test_spiders.spider1"
settings = Settings(
{"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True}
)
spider_loader = SpiderLoader.from_settings(settings)
if str(w[0].message).startswith("_SixMetaPathImporter"):
# needed on 3.10 because of https://github.com/benjaminp/six/issues/349,
# at least until all six versions we can import (including botocore.vendored.six)
# are updated to 1.16.0+
w.pop(0)
assert "Could not load spiders from module" in str(w[0].message)
with pytest.warns(
RuntimeWarning, match="Could not load spiders from module"
):
spider_loader = SpiderLoader.from_settings(settings)
spiders = spider_loader.list()
assert not spiders
spiders = spider_loader.list()
assert not spiders
class TestDuplicateSpiderNameLoader:
@ -190,21 +177,17 @@ class TestDuplicateSpiderNameLoader:
# copy 1 spider module so as to have duplicate spider name
shutil.copyfile(spiders_dir / "spider3.py", spiders_dir / "spider3dupe.py")
with warnings.catch_warnings(record=True) as w:
msg = r"""There are several spiders with the same name:
Spider3 named 'spider3' \(in test_spiders_xxx\.spider3\)
Spider3 named 'spider3' \(in test_spiders_xxx\.spider3dupe\)
This can cause unexpected behavior\."""
with pytest.warns(UserWarning, match=msg):
spider_loader = SpiderLoader.from_settings(settings)
assert len(w) == 1
msg = str(w[0].message)
assert "several spiders with the same name" in msg
assert "'spider3'" in msg
assert msg.count("'spider3'") == 2
assert "'spider1'" not in msg
assert "'spider2'" not in msg
assert "'spider4'" not in msg
spiders = set(spider_loader.list())
assert spiders == {"spider1", "spider2", "spider3", "spider4"}
spiders = set(spider_loader.list())
assert spiders == {"spider1", "spider2", "spider3", "spider4"}
def test_multiple_dupename_warning(self, spider_loader_env):
settings, spiders_dir = spider_loader_env
@ -213,23 +196,21 @@ class TestDuplicateSpiderNameLoader:
shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py")
shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py")
with warnings.catch_warnings(record=True) as w:
msg = r"""There are several spiders with the same name:
Spider1 named 'spider1' \(in test_spiders_xxx\.spider1\)
Spider1 named 'spider1' \(in test_spiders_xxx\.spider1dupe\)
Spider2 named 'spider2' \(in test_spiders_xxx\.spider2\)
Spider2 named 'spider2' \(in test_spiders_xxx\.spider2dupe\)
This can cause unexpected behavior\."""
with pytest.warns(UserWarning, match=msg):
spider_loader = SpiderLoader.from_settings(settings)
assert len(w) == 1
msg = str(w[0].message)
assert "several spiders with the same name" in msg
assert "'spider1'" in msg
assert msg.count("'spider1'") == 2
assert "'spider2'" in msg
assert msg.count("'spider2'") == 2
assert "'spider3'" not in msg
assert "'spider4'" not in msg
spiders = set(spider_loader.list())
assert spiders == {"spider1", "spider2", "spider3", "spider4"}
spiders = set(spider_loader.list())
assert spiders == {"spider1", "spider2", "spider3", "spider4"}
class CustomSpiderLoader(SpiderLoader):

64
tests/test_spidermiddleware_referer.py
View File

@ -6,6 +6,7 @@ from urllib.parse import urlparse
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spidermiddlewares.referer import (
@ -842,13 +843,14 @@ class TestRequestMetaSettingFallback:
response = Response(origin, headers=response_headers)
request = Request(target, meta=request_meta)
with warnings.catch_warnings(record=True) as w:
if check_warning:
with pytest.warns(
RuntimeWarning, match="Could not load referrer policy"
):
policy = mw.policy(response, request)
else:
policy = mw.policy(response, request)
assert isinstance(policy, policy_class)
if check_warning:
assert len(w) == 1
assert w[0].category is RuntimeWarning, w[0].message
assert isinstance(policy, policy_class)
class TestSettingsPolicyByName:
@ -973,49 +975,39 @@ class TestPolicyMethodResponseParamRename:
self.response = Response("http://www.example.com")
def test_pos_string(self):
with warnings.catch_warnings(record=True) as w:
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a response URL to RefererMiddleware\.policy\(\)",
):
self.mw.policy("http://old.com", self.request)
found = False
for warning in w:
if "Passing a response URL" in str(warning.message):
found = True
break
assert found
def test_pos_response(self):
with warnings.catch_warnings(record=True) as w:
with warnings.catch_warnings():
warnings.filterwarnings(
"error",
category=ScrapyDeprecationWarning,
message=r"Passing 'resp_or_url' is deprecated",
)
self.mw.policy(self.response, self.request)
for warning in w:
assert "resp_or_url" not in str(warning.message)
def test_key_resp_or_url(self):
with warnings.catch_warnings(record=True) as w:
with pytest.warns(
ScrapyDeprecationWarning, match=r"Passing 'resp_or_url' is deprecated"
):
self.mw.policy(resp_or_url=self.response, request=self.request)
found = False
for warning in w:
if "Passing 'resp_or_url' is deprecated, use 'response' instead" in str(
warning.message
):
found = True
break
assert found
def test_key_response(self):
with warnings.catch_warnings(record=True) as w:
with warnings.catch_warnings():
warnings.filterwarnings(
"error",
category=ScrapyDeprecationWarning,
message=r"Passing 'resp_or_url' is deprecated",
)
self.mw.policy(response=self.response, request=self.request)
for warning in w:
assert "resp_or_url" not in str(warning.message)
def test_key_response_string(self):
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
with pytest.warns(ScrapyDeprecationWarning, match="Passing a response URL"):
self.mw.policy(response="http://old.com", request=self.request)
found = False
for warning in w:
if "Passing a response URL" in str(warning.message):
found = True
break
assert found
def test_both_resp_or_url_and_response(self):
with pytest.raises(

8
tests/test_utils_curl.py
View File

@ -213,10 +213,12 @@ class TestCurlToRequestKwargs:
def test_ignore_unknown_options(self):
# case 1: ignore_unknown_options=True:
curl_command = "curl --bar --baz http://www.example.com"
expected_result = {"method": "GET", "url": "http://www.example.com"}
with warnings.catch_warnings(): # avoid warning when executing tests
warnings.simplefilter("ignore")
curl_command = "curl --bar --baz http://www.example.com"
expected_result = {"method": "GET", "url": "http://www.example.com"}
warnings.filterwarnings(
"ignore", category=UserWarning, message="Unrecognized options:"
)
assert curl_to_request_kwargs(curl_command) == expected_result
# case 2: ignore_unknown_options=False (raise exception):

15
tests/test_utils_datatypes.py
View File

@ -1,5 +1,4 @@
import copy
import warnings
from abc import ABC, abstractmethod
from collections.abc import Iterator, Mapping, MutableMapping
from typing import Any
@ -227,18 +226,12 @@ class TestCaselessDict(TestCaseInsensitiveDictBase):
dict_class = CaselessDict
def test_deprecation_message(self):
with warnings.catch_warnings(record=True) as caught:
warnings.filterwarnings("always", category=ScrapyDeprecationWarning)
with pytest.warns(
ScrapyDeprecationWarning,
match=r"scrapy.utils.datatypes.CaselessDict is deprecated",
):
self.dict_class({"foo": "bar"})
assert len(caught) == 1
assert issubclass(caught[0].category, ScrapyDeprecationWarning)
assert (
str(caught[0].message)
== "scrapy.utils.datatypes.CaselessDict is deprecated,"
" please use scrapy.utils.datatypes.CaseInsensitiveDict instead"
)
class TestSequenceExclude:
def test_list(self):

148
tests/test_utils_deprecate.py
View File

@ -1,7 +1,6 @@
import inspect
import warnings
from unittest import mock
from warnings import WarningMessage
import pytest
@ -22,34 +21,26 @@ class NewName(SomeBaseClass):
class TestWarnWhenSubclassed:
def _mywarnings(self, w: list[WarningMessage]) -> list[WarningMessage]:
return [x for x in w if x.category is MyWarning]
def test_no_warning_on_definition(self):
with warnings.catch_warnings(record=True) as w:
with warnings.catch_warnings():
warnings.simplefilter("error", category=ScrapyDeprecationWarning)
create_deprecated_class("Deprecated", NewName)
w = self._mywarnings(w)
assert w == []
def test_subclassing_warning_message(self):
msg = (
r"tests\.test_utils_deprecate\.UserClass inherits from "
r"deprecated class tests\.test_utils_deprecate\.Deprecated, "
r"please inherit from tests\.test_utils_deprecate\.NewName."
r" \(warning only on first subclass, there may be others\)"
)
Deprecated = create_deprecated_class(
"Deprecated", NewName, warn_category=MyWarning
)
with warnings.catch_warnings(record=True) as w:
with pytest.warns(MyWarning, match=msg) as w:
class UserClass(Deprecated):
pass
w = self._mywarnings(w)
assert len(w) == 1
assert (
str(w[0].message) == "tests.test_utils_deprecate.UserClass inherits from "
"deprecated class tests.test_utils_deprecate.Deprecated, "
"please inherit from tests.test_utils_deprecate.NewName."
" (warning only on first subclass, there may be others)"
)
assert w[0].lineno == inspect.getsourcelines(UserClass)[1]
def test_custom_class_paths(self):
@ -61,62 +52,77 @@ class TestWarnWhenSubclassed:
warn_category=MyWarning,
)
with warnings.catch_warnings(record=True) as w:
with pytest.warns(
MyWarning,
match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass",
):
class UserClass(Deprecated):
pass
with pytest.warns(
MyWarning,
match=r"bar\.OldClass is deprecated, instantiate foo\.NewClass instead",
):
_ = Deprecated()
w = self._mywarnings(w)
assert len(w) == 2
assert "foo.NewClass" in str(w[0].message)
assert "bar.OldClass" in str(w[0].message)
assert "foo.NewClass" in str(w[1].message)
assert "bar.OldClass" in str(w[1].message)
def test_subclassing_warns_only_on_direct_children(self):
Deprecated = create_deprecated_class(
"Deprecated", NewName, warn_once=False, warn_category=MyWarning
)
with warnings.catch_warnings(record=True) as w:
with pytest.warns(
MyWarning,
match="UserClass inherits from deprecated class",
):
class UserClass(Deprecated):
pass
with warnings.catch_warnings():
warnings.simplefilter("error", MyWarning)
class NoWarnOnMe(UserClass):
pass
w = self._mywarnings(w)
assert len(w) == 1
assert "UserClass" in str(w[0].message)
def test_subclassing_warns_once_by_default(self):
Deprecated = create_deprecated_class(
"Deprecated", NewName, warn_category=MyWarning
)
with warnings.catch_warnings(record=True) as w:
with pytest.warns(
MyWarning,
match="UserClass inherits from deprecated class",
):
class UserClass(Deprecated):
pass
with warnings.catch_warnings():
warnings.simplefilter("error", MyWarning)
class FooClass(Deprecated):
pass
class BarClass(Deprecated):
pass
w = self._mywarnings(w)
assert len(w) == 1
assert "UserClass" in str(w[0].message)
def test_warning_on_instance(self):
Deprecated = create_deprecated_class(
"Deprecated", NewName, warn_category=MyWarning
)
with pytest.warns(MyWarning) as w:
_, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())
w = [x for x in w if x.category is MyWarning]
assert len(w) == 1
assert (
str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, "
"instantiate tests.test_utils_deprecate.NewName instead."
)
assert w[0].lineno == lineno
# ignore subclassing warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore", MyWarning)
@ -124,29 +130,20 @@ class TestWarnWhenSubclassed:
class UserClass(Deprecated):
pass
with warnings.catch_warnings(record=True) as w:
_, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())
_ = UserClass() # subclass instances don't warn
w = self._mywarnings(w)
assert len(w) == 1
assert (
str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, "
"instantiate tests.test_utils_deprecate.NewName instead."
)
assert w[0].lineno == lineno
with warnings.catch_warnings():
warnings.simplefilter("error", MyWarning)
UserClass() # subclass instances don't warn
def test_warning_auto_message(self):
with warnings.catch_warnings(record=True) as w:
Deprecated = create_deprecated_class("Deprecated", NewName)
Deprecated = create_deprecated_class("Deprecated", NewName)
with pytest.warns(
ScrapyDeprecationWarning,
match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName",
):
class UserClass2(Deprecated):
pass
msg = str(w[0].message)
assert "tests.test_utils_deprecate.NewName" in msg
assert "tests.test_utils_deprecate.Deprecated" in msg
def test_issubclass(self):
with warnings.catch_warnings():
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
@ -222,8 +219,8 @@ class TestWarnWhenSubclassed:
create_deprecated_class("Deprecated", New)
def test_deprecate_subclass_of_deprecated_class(self):
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
with warnings.catch_warnings():
warnings.simplefilter("error", MyWarning)
Deprecated = create_deprecated_class(
"Deprecated", NewName, warn_category=MyWarning
)
@ -234,33 +231,26 @@ class TestWarnWhenSubclassed:
warn_category=MyWarning,
)
w = self._mywarnings(w)
assert len(w) == 0, [str(warning) for warning in w]
with warnings.catch_warnings(record=True) as w:
with pytest.warns(
MyWarning,
match=r"AlsoDeprecated is deprecated, instantiate foo\.Bar instead",
):
AlsoDeprecated()
with pytest.warns(
MyWarning,
match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar",
):
class UserClass(AlsoDeprecated):
pass
w = self._mywarnings(w)
assert len(w) == 2
assert "AlsoDeprecated" in str(w[0].message)
assert "foo.Bar" in str(w[0].message)
assert "AlsoDeprecated" in str(w[1].message)
assert "foo.Bar" in str(w[1].message)
def test_inspect_stack(self):
with (
mock.patch("inspect.stack", side_effect=IndexError),
warnings.catch_warnings(record=True) as w,
pytest.warns(UserWarning, match="Error detecting parent module"),
):
DeprecatedName = create_deprecated_class("DeprecatedName", NewName)
class SubClass(DeprecatedName):
pass
assert "Error detecting parent module" in str(w[0].message)
create_deprecated_class("DeprecatedName", NewName)
@mock.patch(
@ -272,12 +262,12 @@ class TestWarnWhenSubclassed:
)
class TestUpdateClassPath:
def test_old_path_gets_fixed(self):
with warnings.catch_warnings(record=True) as w:
with pytest.warns(
ScrapyDeprecationWarning,
match="`scrapy.contrib.debug.Debug` class is deprecated, use `scrapy.extensions.debug.Debug` instead",
):
output = update_classpath("scrapy.contrib.debug.Debug")
assert output == "scrapy.extensions.debug.Debug"
assert len(w) == 1
assert "scrapy.contrib.debug.Debug" in str(w[0].message)
assert "scrapy.extensions.debug.Debug" in str(w[0].message)
def test_sorted_replacement(self):
with warnings.catch_warnings():
@ -286,10 +276,10 @@ class TestUpdateClassPath:
assert output == "scrapy.pipelines.Pipeline"
def test_unmatched_path_stays_the_same(self):
with warnings.catch_warnings(record=True) as w:
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
output = update_classpath("scrapy.unmatched.Path")
assert output == "scrapy.unmatched.Path"
assert len(w) == 0
def test_returns_nonstring(self):
for notastring in [None, True, [1, 2, 3], object()]:

84
tests/test_utils_misc/test_return_with_argument_inside_generator.py
View File

@ -93,29 +93,27 @@ https://example.org
assert is_generator_with_return_value(h1)
assert is_generator_with_return_value(i1)
with warnings.catch_warnings(record=True) as w:
with pytest.warns(
UserWarning,
match='The "MockSpider.top_level_return_something" method is a generator',
):
warn_on_generator_with_return_value(mock_spider, top_level_return_something)
assert len(w) == 1
assert (
'The "MockSpider.top_level_return_something" method is a generator'
in str(w[0].message)
)
with warnings.catch_warnings(record=True) as w:
with pytest.warns(
UserWarning, match='The "MockSpider.f1" method is a generator'
):
warn_on_generator_with_return_value(mock_spider, f1)
assert len(w) == 1
assert 'The "MockSpider.f1" method is a generator' in str(w[0].message)
with warnings.catch_warnings(record=True) as w:
with pytest.warns(
UserWarning, match='The "MockSpider.g1" method is a generator'
):
warn_on_generator_with_return_value(mock_spider, g1)
assert len(w) == 1
assert 'The "MockSpider.g1" method is a generator' in str(w[0].message)
with warnings.catch_warnings(record=True) as w:
with pytest.warns(
UserWarning, match='The "MockSpider.h1" method is a generator'
):
warn_on_generator_with_return_value(mock_spider, h1)
assert len(w) == 1
assert 'The "MockSpider.h1" method is a generator' in str(w[0].message)
with warnings.catch_warnings(record=True) as w:
with pytest.warns(
UserWarning, match='The "MockSpider.i1" method is a generator'
):
warn_on_generator_with_return_value(mock_spider, i1)
assert len(w) == 1
assert 'The "MockSpider.i1" method is a generator' in str(w[0].message)
def test_generators_return_none(self, mock_spider):
def f2():
@ -160,32 +158,18 @@ https://example.org
assert not is_generator_with_return_value(k2) # not recursive
assert not is_generator_with_return_value(l2)
with warnings.catch_warnings(record=True) as w:
with warnings.catch_warnings():
warnings.simplefilter("error", UserWarning)
warn_on_generator_with_return_value(mock_spider, top_level_return_none)
assert len(w) == 0
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(mock_spider, f2)
assert len(w) == 0
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(mock_spider, g2)
assert len(w) == 0
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(mock_spider, h2)
assert len(w) == 0
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(mock_spider, i2)
assert len(w) == 0
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(mock_spider, j2)
assert len(w) == 0
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(mock_spider, k2)
assert len(w) == 0
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(mock_spider, l2)
assert len(w) == 0
def test_generators_return_none_with_decorator(self, mock_spider): # noqa: PLR0915
def test_generators_return_none_with_decorator(self, mock_spider):
def decorator(func):
def inner_func():
func()
@ -241,39 +225,23 @@ https://example.org
assert not is_generator_with_return_value(k3) # not recursive
assert not is_generator_with_return_value(l3)
with warnings.catch_warnings(record=True) as w:
with warnings.catch_warnings():
warnings.simplefilter("error", UserWarning)
warn_on_generator_with_return_value(mock_spider, top_level_return_none)
assert len(w) == 0
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(mock_spider, f3)
assert len(w) == 0
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(mock_spider, g3)
assert len(w) == 0
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(mock_spider, h3)
assert len(w) == 0
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(mock_spider, i3)
assert len(w) == 0
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(mock_spider, j3)
assert len(w) == 0
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(mock_spider, k3)
assert len(w) == 0
with warnings.catch_warnings(record=True) as w:
warn_on_generator_with_return_value(mock_spider, l3)
assert len(w) == 0
@mock.patch(
"scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error
)
def test_indentation_error(self, mock_spider):
with warnings.catch_warnings(record=True) as w:
with pytest.warns(UserWarning, match="Unable to determine"):
warn_on_generator_with_return_value(mock_spider, top_level_return_none)
assert len(w) == 1
assert "Unable to determine" in str(w[0].message)
def test_partial(self):
def cb(arg1, arg2):
@ -300,13 +268,11 @@ https://example.org
yield 1
return "value"
with warnings.catch_warnings(record=True) as w:
with warnings.catch_warnings():
warnings.simplefilter("error", UserWarning)
warn_on_generator_with_return_value(spider, gen_with_return)
assert len(w) == 0
spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True
with warnings.catch_warnings(record=True) as w:
with pytest.warns(UserWarning, match="is a generator"):
warn_on_generator_with_return_value(spider, gen_with_return)
assert len(w) == 1
assert "is a generator" in str(w[0].message)

11
tests/test_utils_sitemap.py
View File

@ -1,5 +1,6 @@
import warnings
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
@ -204,7 +205,10 @@ Disallow: /forum/search/
Disallow: /forum/active/
"""
with warnings.catch_warnings(record=True) as w:
with pytest.warns(
ScrapyDeprecationWarning,
match="Passing `str` type as `robots_text` is deprecated",
):
assert list(
sitemap_urls_from_robots(robots, base_url="http://example.com")
) == [
@ -213,9 +217,6 @@ Disallow: /forum/active/
"http://example.com/sitemap-uppercase.xml",
"http://example.com/sitemap-relative-url.xml",
]
assert "Passing `str` type as `robots_text` is deprecated, use `bytes`" in str(
w[0].message
)
def test_sitemap_blanklines():