From bc200d1155e0f93f37897ac206aa32945daf89bd Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 21 Feb 2017 17:08:39 +0100 Subject: [PATCH] Rename setting to REFERRER_POLICY (with 2 Rs) --- docs/topics/spider-middleware.rst | 12 ++++----- scrapy/settings/default_settings.py | 2 +- scrapy/spidermiddlewares/referer.py | 2 +- tests/test_spidermiddleware_referer.py | 36 +++++++++++++------------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 349dbb3a1..9a0ccd0c1 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -328,10 +328,10 @@ Default: ``True`` Whether to enable referer middleware. -.. setting:: REFERER_POLICY +.. setting:: REFERRER_POLICY -REFERER_POLICY -^^^^^^^^^^^^^^ +REFERRER_POLICY +^^^^^^^^^^^^^^^ .. versionadded:: 1.4 @@ -344,10 +344,10 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` .. note:: You can also set the Referrer Policy per request, using the special ``"referrer_policy"`` :ref:`Request.meta ` key, - with the same acceptable values as for the ``REFERER_POLICY`` setting. + with the same acceptable values as for the ``REFERRER_POLICY`` setting. -Acceptable values for REFERER_POLICY -************************************ +Acceptable values for REFERRER_POLICY +************************************* - either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass — a custom policy or one of the built-in ones (see classes below), diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 15a134dd3..35d9844a7 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -234,7 +234,7 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True -REFERER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' +REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index c015e13c8..24c163089 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -266,7 +266,7 @@ class RefererMiddleware(object): def __init__(self, settings=None): self.default_policy = DefaultReferrerPolicy if settings is not None: - policy = settings.get('REFERER_POLICY') + policy = settings.get('REFERRER_POLICY') if policy is not None: # expect a string for the path to the policy class try: diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 4779b0ed1..81868efab 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -289,35 +289,35 @@ class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware): # --- Tests using settings to set policy using class path class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} class CustomPythonOrgPolicy(ReferrerPolicy): @@ -336,7 +336,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware): - settings = {'REFERER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} + settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} scenarii = [ ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'), @@ -385,17 +385,17 @@ class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} req_meta = {'referrer_policy': POLICY_NO_REFERRER} class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} @@ -411,7 +411,7 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: - settings = Settings({'REFERER_POLICY': s}) + settings = Settings({'REFERRER_POLICY': s}) mw = RefererMiddleware(settings) self.assertEquals(mw.default_policy, p) @@ -425,24 +425,24 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: - settings = Settings({'REFERER_POLICY': s.upper()}) + settings = Settings({'REFERRER_POLICY': s.upper()}) mw = RefererMiddleware(settings) self.assertEquals(mw.default_policy, p) def test_invalid_name(self): - settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'}) + settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'}) with self.assertRaises(NotConfigured): mw = RefererMiddleware(settings) class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}