From 0d58af86971ba54bfc1feffa88fc564ffba650f4 Mon Sep 17 00:00:00 2001
From: Fabian Schneebauer <fabian@fs-it.org>
Date: Wed, 29 May 2024 10:59:32 +0200
Subject: [PATCH 1/2] Add support for multiple referer policy tokens.

---
 scrapy/spidermiddlewares/referer.py    | 20 ++++++-----
 tests/test_spidermiddleware_referer.py | 47 ++++++++++++++++++++++++++
 2 files changed, 58 insertions(+), 9 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index a0b6851e5..7706c8c15 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -323,15 +323,17 @@ def _load_policy_class(
     try:
         return cast(Type[ReferrerPolicy], load_object(policy))
     except ValueError:
-        try:
-            return _policy_classes[policy.lower()]
-        except KeyError:
-            msg = f"Could not load referrer policy {policy!r}"
-            if not warning_only:
-                raise RuntimeError(msg)
-            else:
-                warnings.warn(msg, RuntimeWarning)
-                return None
+        tokens = [token.strip() for token in policy.lower().split(",")]
+        for token in tokens[::-1]:
+            if token in _policy_classes:
+                return _policy_classes[token]
+
+        msg = f"Could not load referrer policy {policy!r}"
+        if not warning_only:
+            raise RuntimeError(msg)
+        else:
+            warnings.warn(msg, RuntimeWarning)
+            return None
 
 
 class RefererMiddleware:
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index afffa87fb..5797edfbd 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -884,6 +884,53 @@ class TestSettingsPolicyByName(TestCase):
         with self.assertRaises(RuntimeError):
             RefererMiddleware(settings)
 
+    def test_multiple_policy_tokens(self):
+        # test parsing without space(s) after the comma
+        settings1 = Settings(
+            {
+                "REFERRER_POLICY": ",".join(
+                    [
+                        "some-custom-unknown-policy",
+                        POLICY_SAME_ORIGIN,
+                        POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN,
+                        "another-custom-unknown-policy",
+                    ]
+                )
+            }
+        )
+        mw1 = RefererMiddleware(settings1)
+        self.assertEqual(mw1.default_policy, StrictOriginWhenCrossOriginPolicy)
+
+        # test parsing with space(s) after the comma
+        settings2 = Settings(
+            {
+                "REFERRER_POLICY": ",    ".join(
+                    [
+                        POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN,
+                        "another-custom-unknown-policy",
+                        POLICY_UNSAFE_URL,
+                    ]
+                )
+            }
+        )
+        mw2 = RefererMiddleware(settings2)
+        self.assertEqual(mw2.default_policy, UnsafeUrlPolicy)
+
+    def test_multiple_policy_tokens_all_invalid(self):
+        settings = Settings(
+            {
+                "REFERRER_POLICY": ",".join(
+                    [
+                        "some-custom-unknown-policy",
+                        "another-custom-unknown-policy",
+                        "yet-another-custom-unknown-policy",
+                    ]
+                )
+            }
+        )
+        with self.assertRaises(RuntimeError):
+            RefererMiddleware(settings)
+
 
 class TestPolicyHeaderPrecedence001(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"}

From 62a028b99dc73b8ddddd37f986780a5a070f2938 Mon Sep 17 00:00:00 2001
From: Fabian Schneebauer <67049088+0xdeb@users.noreply.github.com>
Date: Wed, 29 May 2024 13:19:27 +0200
Subject: [PATCH 2/2] Add spec link to scrapy/spidermiddlewares/referer.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/spidermiddlewares/referer.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 7706c8c15..8af0bdf5b 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -324,6 +324,7 @@ def _load_policy_class(
         return cast(Type[ReferrerPolicy], load_object(policy))
     except ValueError:
         tokens = [token.strip() for token in policy.lower().split(",")]
+        # https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header
         for token in tokens[::-1]:
             if token in _policy_classes:
                 return _policy_classes[token]