From e50e670eff2fe1b109f6b5dd026c4c706a93585c Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 12 Oct 2016 16:16:53 +0200
Subject: [PATCH] Add test for custom referrer policy via settings

---
 scrapy/spidermiddlewares/referer.py    |  4 ++++
 tests/test_spidermiddleware_referer.py | 30 ++++++++++++++++++++++++--
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 2a3790bde..0bba63cb7 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -227,9 +227,12 @@ class RefererMiddleware(object):
         if settings is not None:
             policy = settings.get('REFERER_POLICY')
             if policy is not None:
+                # expect a string for the path to the policy class
                 try:
                     self.default_policy = load_object(policy)
                 except ValueError:
+                    # otherwise try to interpret the string as standard
+                    # https://www.w3.org/TR/referrer-policy/#referrer-policies
                     try:
                         self.default_policy = _policy_classes[policy.lower()]
                     except:
@@ -239,6 +242,7 @@ class RefererMiddleware(object):
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('REFERER_ENABLED'):
             raise NotConfigured
+
         return cls(crawler.settings)
 
     def policy(self, response, request):
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index b1ab366a7..cfc4b5296 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -11,7 +11,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware, \
     DefaultReferrerPolicy, \
     NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \
     OriginWhenCrossOriginPolicy, OriginPolicy, \
-    SameOriginPolicy, UnsafeUrlPolicy
+    SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy
 
 
 class TestRefererMiddleware(TestCase):
@@ -249,6 +249,33 @@ class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddlewa
     settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
 
 
+class CustomPythonOrgPolicy(ReferrerPolicy):
+    """
+    A dummy policy that returns referrer as http(s)://python.org
+    depending on the scheme of the target URL.
+    """
+    def referrer(self, response, request):
+        from scrapy.utils.httpobj import urlparse_cached
+
+        scheme = urlparse_cached(request).scheme
+        if scheme == 'https':
+            return b'https://python.org/'
+        elif scheme == 'http':
+            return b'http://python.org/'
+
+
+class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}
+    scenarii = [
+        ('https://example.com/',    'https://scrapy.org/',  b'https://python.org/'),
+        ('http://example.com/',     'http://scrapy.org/',   b'http://python.org/'),
+        ('http://example.com/',     'https://scrapy.org/',  b'https://python.org/'),
+        ('https://example.com/',    'http://scrapy.org/',   b'http://python.org/'),
+        ('file:///home/path/to/somefile.html',  'https://scrapy.org/', b'https://python.org/'),
+        ('file:///home/path/to/somefile.html',  'http://scrapy.org/',  b'http://python.org/'),
+
+    ]
+
 # --- Tests using Request meta dict to set policy
 class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT}
@@ -278,7 +305,6 @@ class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
 
 
-
 class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
     req_meta = {'referrer_policy': POLICY_UNSAFE_URL}