diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a0b6851e5..7706c8c15 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -323,15 +323,17 @@ def _load_policy_class( try: return cast(Type[ReferrerPolicy], load_object(policy)) except ValueError: - try: - return _policy_classes[policy.lower()] - except KeyError: - msg = f"Could not load referrer policy {policy!r}" - if not warning_only: - raise RuntimeError(msg) - else: - warnings.warn(msg, RuntimeWarning) - return None + tokens = [token.strip() for token in policy.lower().split(",")] + for token in tokens[::-1]: + if token in _policy_classes: + return _policy_classes[token] + + msg = f"Could not load referrer policy {policy!r}" + if not warning_only: + raise RuntimeError(msg) + else: + warnings.warn(msg, RuntimeWarning) + return None class RefererMiddleware: diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index afffa87fb..5797edfbd 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -884,6 +884,53 @@ class TestSettingsPolicyByName(TestCase): with self.assertRaises(RuntimeError): RefererMiddleware(settings) + def test_multiple_policy_tokens(self): + # test parsing without space(s) after the comma + settings1 = Settings( + { + "REFERRER_POLICY": ",".join( + [ + "some-custom-unknown-policy", + POLICY_SAME_ORIGIN, + POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, + "another-custom-unknown-policy", + ] + ) + } + ) + mw1 = RefererMiddleware(settings1) + self.assertEqual(mw1.default_policy, StrictOriginWhenCrossOriginPolicy) + + # test parsing with space(s) after the comma + settings2 = Settings( + { + "REFERRER_POLICY": ", ".join( + [ + POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, + "another-custom-unknown-policy", + POLICY_UNSAFE_URL, + ] + ) + } + ) + mw2 = RefererMiddleware(settings2) + self.assertEqual(mw2.default_policy, UnsafeUrlPolicy) + + def test_multiple_policy_tokens_all_invalid(self): + settings = Settings( + { + "REFERRER_POLICY": ",".join( + [ + "some-custom-unknown-policy", + "another-custom-unknown-policy", + "yet-another-custom-unknown-policy", + ] + ) + } + ) + with self.assertRaises(RuntimeError): + RefererMiddleware(settings) + class TestPolicyHeaderPrecedence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"}