diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 2c9452174..264f685c1 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -308,6 +308,12 @@ class RefererMiddleware(BaseSpiderMiddleware): # Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string self.policies[""] = NoReferrerWhenDowngradePolicy if settings is None: + warn( + "Instantiating RefererMiddleware without a 'settings' argument is " + "deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) return setting_policies = settings.getdict("REFERRER_POLICIES") for policy_name, policy_class_import_path in setting_policies.items(): diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py index 70326f3f1..4de57fda9 100644 --- a/tests/test_spidermiddleware_base.py +++ b/tests/test_spidermiddleware_base.py @@ -56,6 +56,11 @@ async def test_processed_request(crawler: Crawler) -> None: spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3] for processed in [ list(mw.process_spider_output(Response("data:,"), spider_output)), + await collect_asyncgen( + mw.process_spider_output_async( + Response("data:,"), as_async_generator(spider_output) + ) + ), await collect_asyncgen(mw.process_start(as_async_generator(spider_output))), ]: assert len(processed) == 3 diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index 00e547305..307ed2440 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -7,6 +7,7 @@ import pytest from scrapy.http import Request, Response from scrapy.spidermiddlewares.depth import DepthMiddleware from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler if TYPE_CHECKING: @@ -55,3 +56,27 @@ def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> No rdm = stats.get_value("request_depth_max") assert rdm == 1 + + +def test_priority_and_non_verbose_stats() -> None: + crawler = get_crawler( + Spider, + {"DEPTH_LIMIT": 0, "DEPTH_STATS_VERBOSE": False, "DEPTH_PRIORITY": 10}, + ) + assert crawler.stats is not None + crawler.stats.open_spider() + try: + mw = build_from_crawler(DepthMiddleware, crawler) + resp = Response("http://toscrape.com") + resp.request = Request("http://toscrape.com") + resp.request.meta["depth"] = 2 + out = list(mw.process_spider_output(resp, [Request("http://toscrape.com")])) + assert len(out) == 1 + # priority is decremented by depth * DEPTH_PRIORITY + assert out[0].priority == -30 + assert out[0].meta["depth"] == 3 + # non-verbose stats don't track per-depth counts but still track the max + assert crawler.stats.get_value("request_depth_count/3") is None + assert crawler.stats.get_value("request_depth_max") == 3 + finally: + crawler.stats.close_spider() diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index b3a434c3f..32e83f990 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -6,7 +6,7 @@ from urllib.parse import urlparse import pytest -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spidermiddlewares.referer import ( @@ -1017,6 +1017,14 @@ class TestPolicyMethodResponseParamRename: response=self.response, resp_or_url=self.response, request=self.request ) + def test_missing_response(self): + with pytest.raises(TypeError, match="Missing required argument: 'response'"): + self.mw.policy(request=self.request) + + def test_missing_request(self): + with pytest.raises(TypeError, match="Missing required argument: 'request'"): + self.mw.policy(response=self.response) + @coroutine_test async def test_response_policy_only_supports_policy_names(): @@ -1115,3 +1123,36 @@ async def test_referer_policies_setting(): ] assert len(output) == 1 assert output[0].headers == {b"Referer": [b"https://python.org/"]} + + +class TestReferrerPolicyHelpers: + def test_origin_referrer_local_scheme(self): + # A local scheme yields no referrer. + assert UnsafeUrlPolicy().origin_referrer("data:,foo") is None + + def test_strip_url_empty(self): + assert UnsafeUrlPolicy().strip_url("") is None + + def test_potentially_trustworthy_data_scheme(self): + assert UnsafeUrlPolicy().potentially_trustworthy("data:,foo") is False + + +def test_default_policy(): + crawler = get_crawler() + mw = build_from_crawler(RefererMiddleware, crawler) + assert mw.default_policy is DefaultReferrerPolicy + + +def test_no_settings_constructor(): + with pytest.warns( + ScrapyDeprecationWarning, + match="Instantiating RefererMiddleware without a 'settings' argument", + ): + mw = RefererMiddleware() + assert mw.default_policy is DefaultReferrerPolicy + + +def test_not_configured_when_disabled(): + crawler = get_crawler(settings_dict={"REFERER_ENABLED": False}) + with pytest.raises(NotConfigured): + build_from_crawler(RefererMiddleware, crawler) diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py index 26397d37c..def3a3df3 100644 --- a/tests/test_spidermiddleware_start.py +++ b/tests/test_spidermiddleware_start.py @@ -1,4 +1,4 @@ -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.spidermiddlewares.start import StartSpiderMiddleware from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler @@ -23,3 +23,13 @@ class TestMiddleware: async for request in mw.process_start(start()) ] assert result == [True, True, False, "foo"] + + def test_spider_output_not_marked(self): + # Requests from a non-None response (spider output) are not flagged. + crawler = get_crawler(Spider) + mw = build_from_crawler(StartSpiderMiddleware, crawler) + response = Response("data:,") + request = Request("data:,1") + out = list(mw.process_spider_output(response, [request])) + assert out == [request] + assert "is_start_request" not in request.meta diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 750ae3b07..1ed3a5637 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -5,9 +5,11 @@ from typing import TYPE_CHECKING import pytest +from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler if TYPE_CHECKING: @@ -46,6 +48,12 @@ def test_middleware_works(mw: UrlLengthMiddleware) -> None: assert process_spider_output(mw) == [short_url_req] +def test_not_configured_without_limit() -> None: + crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": 0}) + with pytest.raises(NotConfigured): + build_from_crawler(UrlLengthMiddleware, crawler) + + def test_logging( stats: StatsCollector, mw: UrlLengthMiddleware, caplog: pytest.LogCaptureFixture ) -> None: