RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix referrer policy from response headers and support explicit empty string

This commit is contained in:
Paul Tremberth 2017-03-06 16:20:37 +01:00
parent d7b26edf6b
commit 768f3155e5
2 changed files with 13 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/spidermiddlewares/referer.py
View File

@ -261,6 +261,9 @@ _policy_classes = {p.name: p for p in (
DefaultReferrerPolicy,
)}
# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string
_policy_classes[''] = NoReferrerWhenDowngradePolicy
def _load_policy_class(policy, warning_only=False):
"""
@ -317,8 +320,9 @@ class RefererMiddleware(object):
policy_name = request.meta.get('referrer_policy')
if policy_name is None:
if isinstance(resp_or_url, Response):
policy_name = to_native_str(
resp_or_url.headers.get('Referrer-Policy', '').decode('latin1'))
policy_header = resp_or_url.headers.get('Referrer-Policy')
if policy_header is not None:
policy_name = to_native_str(policy_header.decode('latin1'))
if policy_name is None:
return self.default_policy()

7
tests/test_spidermiddleware_referer.py
View File

@ -526,6 +526,13 @@ class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMid
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
"""
The empty string means "no-referrer-when-downgrade"
"""
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
resp_headers = {'Referrer-Policy': ''}
class TestReferrerOnRedirect(TestRefererMiddleware):