Add test for custom referrer policy via settings

scrapy/spidermiddlewares/referer.py

@@ -227,9 +227,12 @@ class RefererMiddleware(object):
         if settings is not None:
             policy = settings.get('REFERER_POLICY')
             if policy is not None:
+                # expect a string for the path to the policy class
                 try:
                     self.default_policy = load_object(policy)
                 except ValueError:
+                    # otherwise try to interpret the string as standard
+                    # https://www.w3.org/TR/referrer-policy/#referrer-policies
                     try:
                         self.default_policy = _policy_classes[policy.lower()]
                     except:
@@ -239,6 +242,7 @@ class RefererMiddleware(object):
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('REFERER_ENABLED'):
             raise NotConfigured
+
         return cls(crawler.settings)
 
     def policy(self, response, request):

tests/test_spidermiddleware_referer.py

@@ -11,7 +11,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware, \
     DefaultReferrerPolicy, \
     NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \
     OriginWhenCrossOriginPolicy, OriginPolicy, \
-    SameOriginPolicy, UnsafeUrlPolicy
+    SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy
 
 
 class TestRefererMiddleware(TestCase):
@@ -249,6 +249,33 @@ class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddlewa
     settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
 
 
+class CustomPythonOrgPolicy(ReferrerPolicy):
+    """
+    A dummy policy that returns referrer as http(s)://python.org
+    depending on the scheme of the target URL.
+    """
+    def referrer(self, response, request):
+        from scrapy.utils.httpobj import urlparse_cached
+
+        scheme = urlparse_cached(request).scheme
+        if scheme == 'https':
+            return b'https://python.org/'
+        elif scheme == 'http':
+            return b'http://python.org/'
+
+
+class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware):
+    settings = {'REFERER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}
+    scenarii = [
+        ('https://example.com/',    'https://scrapy.org/',  b'https://python.org/'),
+        ('http://example.com/',     'http://scrapy.org/',   b'http://python.org/'),
+        ('http://example.com/',     'https://scrapy.org/',  b'https://python.org/'),
+        ('https://example.com/',    'http://scrapy.org/',   b'http://python.org/'),
+        ('file:///home/path/to/somefile.html',  'https://scrapy.org/', b'https://python.org/'),
+        ('file:///home/path/to/somefile.html',  'http://scrapy.org/',  b'http://python.org/'),
+
+    ]
+
 # --- Tests using Request meta dict to set policy
 class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT}
@@ -278,7 +305,6 @@ class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
     req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
 
 
-
 class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
     req_meta = {'referrer_policy': POLICY_UNSAFE_URL}