Improve test coverage for spider middlewares (#7664)

scrapy/spidermiddlewares/referer.py

@@ -308,6 +308,12 @@ class RefererMiddleware(BaseSpiderMiddleware):
         # Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string
         self.policies[""] = NoReferrerWhenDowngradePolicy
         if settings is None:
+            warn(
+                "Instantiating RefererMiddleware without a 'settings' argument is "
+                "deprecated.",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
             return
         setting_policies = settings.getdict("REFERRER_POLICIES")
         for policy_name, policy_class_import_path in setting_policies.items():

tests/test_spidermiddleware_base.py

@@ -56,6 +56,11 @@ async def test_processed_request(crawler: Crawler) -> None:
     spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3]
     for processed in [
         list(mw.process_spider_output(Response("data:,"), spider_output)),
+        await collect_asyncgen(
+            mw.process_spider_output_async(
+                Response("data:,"), as_async_generator(spider_output)
+            )
+        ),
         await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
     ]:
         assert len(processed) == 3

tests/test_spidermiddleware_depth.py

@@ -7,6 +7,7 @@ import pytest
 from scrapy.http import Request, Response
 from scrapy.spidermiddlewares.depth import DepthMiddleware
 from scrapy.spiders import Spider
+from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.test import get_crawler
 
 if TYPE_CHECKING:
@@ -55,3 +56,27 @@ def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> No
 
     rdm = stats.get_value("request_depth_max")
     assert rdm == 1
+
+
+def test_priority_and_non_verbose_stats() -> None:
+    crawler = get_crawler(
+        Spider,
+        {"DEPTH_LIMIT": 0, "DEPTH_STATS_VERBOSE": False, "DEPTH_PRIORITY": 10},
+    )
+    assert crawler.stats is not None
+    crawler.stats.open_spider()
+    try:
+        mw = build_from_crawler(DepthMiddleware, crawler)
+        resp = Response("http://toscrape.com")
+        resp.request = Request("http://toscrape.com")
+        resp.request.meta["depth"] = 2
+        out = list(mw.process_spider_output(resp, [Request("http://toscrape.com")]))
+        assert len(out) == 1
+        # priority is decremented by depth * DEPTH_PRIORITY
+        assert out[0].priority == -30
+        assert out[0].meta["depth"] == 3
+        # non-verbose stats don't track per-depth counts but still track the max
+        assert crawler.stats.get_value("request_depth_count/3") is None
+        assert crawler.stats.get_value("request_depth_max") == 3
+    finally:
+        crawler.stats.close_spider()

tests/test_spidermiddleware_referer.py

@@ -6,7 +6,7 @@ from urllib.parse import urlparse
 
 import pytest
 
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.spidermiddlewares.referer import (
@@ -1017,6 +1017,14 @@ class TestPolicyMethodResponseParamRename:
                 response=self.response, resp_or_url=self.response, request=self.request
             )
 
+    def test_missing_response(self):
+        with pytest.raises(TypeError, match="Missing required argument: 'response'"):
+            self.mw.policy(request=self.request)
+
+    def test_missing_request(self):
+        with pytest.raises(TypeError, match="Missing required argument: 'request'"):
+            self.mw.policy(response=self.response)
+
 
 @coroutine_test
 async def test_response_policy_only_supports_policy_names():
@@ -1115,3 +1123,36 @@ async def test_referer_policies_setting():
     ]
     assert len(output) == 1
     assert output[0].headers == {b"Referer": [b"https://python.org/"]}
+
+
+class TestReferrerPolicyHelpers:
+    def test_origin_referrer_local_scheme(self):
+        # A local scheme yields no referrer.
+        assert UnsafeUrlPolicy().origin_referrer("data:,foo") is None
+
+    def test_strip_url_empty(self):
+        assert UnsafeUrlPolicy().strip_url("") is None
+
+    def test_potentially_trustworthy_data_scheme(self):
+        assert UnsafeUrlPolicy().potentially_trustworthy("data:,foo") is False
+
+
+def test_default_policy():
+    crawler = get_crawler()
+    mw = build_from_crawler(RefererMiddleware, crawler)
+    assert mw.default_policy is DefaultReferrerPolicy
+
+
+def test_no_settings_constructor():
+    with pytest.warns(
+        ScrapyDeprecationWarning,
+        match="Instantiating RefererMiddleware without a 'settings' argument",
+    ):
+        mw = RefererMiddleware()
+    assert mw.default_policy is DefaultReferrerPolicy
+
+
+def test_not_configured_when_disabled():
+    crawler = get_crawler(settings_dict={"REFERER_ENABLED": False})
+    with pytest.raises(NotConfigured):
+        build_from_crawler(RefererMiddleware, crawler)

tests/test_spidermiddleware_start.py

@@ -1,4 +1,4 @@
-from scrapy.http import Request
+from scrapy.http import Request, Response
 from scrapy.spidermiddlewares.start import StartSpiderMiddleware
 from scrapy.spiders import Spider
 from scrapy.utils.misc import build_from_crawler
@@ -23,3 +23,13 @@ class TestMiddleware:
             async for request in mw.process_start(start())
         ]
         assert result == [True, True, False, "foo"]
+
+    def test_spider_output_not_marked(self):
+        # Requests from a non-None response (spider output) are not flagged.
+        crawler = get_crawler(Spider)
+        mw = build_from_crawler(StartSpiderMiddleware, crawler)
+        response = Response("data:,")
+        request = Request("data:,1")
+        out = list(mw.process_spider_output(response, [request]))
+        assert out == [request]
+        assert "is_start_request" not in request.meta

tests/test_spidermiddleware_urllength.py

@@ -5,9 +5,11 @@ from typing import TYPE_CHECKING
 
 import pytest
 
+from scrapy.exceptions import NotConfigured
 from scrapy.http import Request, Response
 from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
 from scrapy.spiders import Spider
+from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.test import get_crawler
 
 if TYPE_CHECKING:
@@ -46,6 +48,12 @@ def test_middleware_works(mw: UrlLengthMiddleware) -> None:
     assert process_spider_output(mw) == [short_url_req]
 
 
+def test_not_configured_without_limit() -> None:
+    crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": 0})
+    with pytest.raises(NotConfigured):
+        build_from_crawler(UrlLengthMiddleware, crawler)
+
+
 def test_logging(
     stats: StatsCollector, mw: UrlLengthMiddleware, caplog: pytest.LogCaptureFixture
 ) -> None: