From baed7c436f8c6e52a1aab62fbaf900c0a0f3bbda Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 11:18:26 +0200 Subject: [PATCH 01/41] WIP Add Referrer policies --- scrapy/spidermiddlewares/referer.py | 206 ++++++++++++++++++++++++- tests/test_spidermiddleware_referer.py | 74 +++++++++ 2 files changed, 279 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 6a8c46543..21b340f22 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,22 +2,226 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ +from six.moves.urllib.parse import urlsplit, urlunsplit from scrapy.http import Request from scrapy.exceptions import NotConfigured +from scrapy.utils.python import to_native_str + + +LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) + +class ReferrerPolicy(object): + + NOREFERRER_SCHEMES = LOCAL_SCHEMES + + def referrer(self, response, request): + raise NotImplementedError() + + def strip_url(self, url, origin_only=False): + """ + https://www.w3.org/TR/referrer-policy/#strip-url + + If url is null, return no referrer. + If url's scheme is a local scheme, then return no referrer. + Set url's username to the empty string. + Set url's password to null. + Set url's fragment to null. + If the origin-only flag is true, then: + Set url's path to null. + Set url's query to null. + Return url. + """ + if url is None or not url: + return None + parsed = urlsplit(url, allow_fragments=True) + + if parsed.scheme in self.NOREFERRER_SCHEMES: + return None + if parsed.username or parsed.password: + netloc = parsed.netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') + else: + netloc = parsed.netloc + return urlunsplit(( + parsed.scheme, + netloc, + '' if origin_only else parsed.path, + '' if origin_only else parsed.query, + '')) + + +class NoReferrerPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer + + The simplest policy is "no-referrer", which specifies that no referrer information + is to be sent along with requests made from a particular request client to any origin. + The header will be omitted entirely. + """ + name = "no-referrer" + + def referrer(self, response, request): + return None + + +class NoReferrerWhenDowngradePolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + + The "no-referrer-when-downgrade" policy sends a full URL + along with requests from a TLS-protected environment settings object + to a a priori authenticated URL, + and requests from request clients which are not TLS-protected to any origin. + + Requests from TLS-protected request clients to non-a priori authenticated URLs, + on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + + This is a user agent's default behavior, if no policy is otherwise specified. + """ + name = "no-referrer-when-downgrade" + + def referrer(self, response, request): + target_url = request.url + + referrer_source = response.url + referrer_url = self.strip_url(referrer_source) + + # https://www.w3.org/TR/referrer-policy/#determine-requests-referrer: + # + # If environment is TLS-protected + # and the origin of request's current URL is not an a priori authenticated URL, + # then return no referrer. + if urlsplit(referrer_source).scheme in ('https', 'ftps') and \ + urlsplit(target_url).scheme in ('http',): + return None + return referrer_url + + +class SameOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin + + The "same-origin" policy specifies that a full URL, stripped for use as a referrer, + is sent as referrer information when making same-origin requests from a particular request client. + + Cross-origin requests, on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + """ + name = "same-origin" + + def referrer(self, response, request): + target_url = request.url + referrer_source = response.url + if urlsplit(referrer_source).netloc == urlsplit(target_url).netloc: + return self.strip_url(referrer_source) + else: + return None + + +class OriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-origin + + The "origin" policy specifies that only the ASCII serialization + of the origin of the request client is sent as referrer information + when making both same-origin requests and cross-origin requests + from a particular request client. + """ + name = "origin" + + def referrer(self, response, request): + return self.strip_url(referrer_source, origin_only=True) + + +class OriginWhenCrossOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin + + The "origin-when-cross-origin" policy specifies that a full URL, + stripped for use as a referrer, is sent as referrer information + when making same-origin requests from a particular request client, + and only the ASCII serialization of the origin of the request client + is sent as referrer information when making cross-origin requests + from a particular request client. + """ + name = "origin-when-cross-origin" + + def referrer(self, response, request): + target_url = request.url + referrer_source = response.url + + # same origin --> send full referrer + # different origin --> send only "origin" as referrer + if urlsplit(referrer_source).netloc != urlsplit(target_url).netloc: + origin_only = True + return self.strip_url(referrer_source, origin_only=origin_only) + + +class UnsafeUrlPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url + + The "unsafe-url" policy specifies that a full URL, stripped for use as a referrer, + is sent along with both cross-origin requests + and same-origin requests made from a particular request client. + + Note: The policy's name doesn't lie; it is unsafe. + This policy will leak origins and paths from TLS-protected resources + to insecure origins. + Carefully consider the impact of setting such a policy for potentially sensitive documents. + """ + name = "unsafe-url" + + def referrer(self, response, request): + referrer_source = response.url + return self.strip_url(referrer_source) + + +class LegacyPolicy(ReferrerPolicy): + def referrer(self, response, request): + return response.url + + +class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): + + NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') + + +_policies = {p.name: p for p in ( + NoReferrerPolicy, + NoReferrerWhenDowngradePolicy, + SameOriginPolicy, + OriginPolicy, + OriginWhenCrossOriginPolicy, + UnsafeUrlPolicy, +)} class RefererMiddleware(object): + def __init__(self, policy_class=DefaultReferrerPolicy): + self.default_policy = policy_class + @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured return cls() + def policy(self, response, request): + policy_name = request.meta.get('referrer_policy') + if policy_name is None: + policy_name = to_native_str(response.headers.get('Referrer-Policy', '').decode('latin1')) + + policy_class = _policies.get(policy_name.lower(), self.default_policy) + return policy_class() + def process_spider_output(self, response, result, spider): def _set_referer(r): if isinstance(r, Request): - r.headers.setdefault('Referer', response.url) + referrer = self.policy(response, r).referrer(response, r) + if referrer is not None: + r.headers.setdefault('Referer', referrer) return r return (_set_referer(r) for r in result or ()) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index bd7673efb..f109bb248 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -19,3 +19,77 @@ class TestRefererMiddleware(TestCase): self.assertEquals(out[0].headers.get('Referer'), b'http://scrapytest.org') + def test_policy_default(self): + """ + Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + + with some additional filtering of s3:// + """ + # a) https:// --> https:// -- include Referer header + origin = Response('https://example.com/') + target = Request('https://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), + b'https://example.com/') + + # b.1) http:// --> http:// -- include Referer header + origin = Response('http://example.com/') + target = Request('http://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), + b'http://example.com/') + + # b.2) http:// --> https:// -- include Referer header + origin = Response('http://example.com/') + target = Request('https://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), + b'http://example.com/') + + # c) https:// --> http:// -- Referer header NOT sent + origin = Response('https://example.com/') + target = Request('http://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), None) + + def test_policy_default_no_credentials_leak(self): + origin = Response('http://user:password@example.com/') + target = Request('https://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), + b'http://example.com/') + + def test_policy_default_file_no_referrer_leak(self): + # file:// --> https:// -- Referrer NOT sent + origin = Response('file:///home/path/to/somefile.html') + target = Request('https://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), None) + + # file:// --> http:// -- Referrer NOT sent + origin = Response('file:///home/path/to/somefile.html') + target = Request('http://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), None) + + def test_policy_default_s3_no_referrer_leak(self): + # s3:// --> https:// -- Referrer NOT sent + origin = Response('s3://mybucket/path/to/data.csv') + target = Request('https://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), None) + + # s3:// --> http:// -- Referrer NOT sent + origin = Response('s3://mybucket/path/to/data.csv') + target = Request('http://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), None) From 7ec1b5f6c316f6c251821441f513e3f71cf63da0 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 17:12:44 +0200 Subject: [PATCH 02/41] Add tests for the different referrer policies --- scrapy/spidermiddlewares/referer.py | 66 ++++--- tests/test_spidermiddleware_referer.py | 242 +++++++++++++++++++------ 2 files changed, 235 insertions(+), 73 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 21b340f22..1895aa95d 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -11,6 +11,15 @@ from scrapy.utils.python import to_native_str LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) +POLICY_NO_REFERRER = "no-referrer" +POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade" +POLICY_SAME_ORIGIN = "same-origin" +POLICY_ORIGIN = "origin" +POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin" +POLICY_UNSAFE_URL = "unsafe-url" +POLICY_SCRAPY_DEFAULT = "scrapy-default" + + class ReferrerPolicy(object): NOREFERRER_SCHEMES = LOCAL_SCHEMES @@ -38,17 +47,29 @@ class ReferrerPolicy(object): if parsed.scheme in self.NOREFERRER_SCHEMES: return None + + netloc = parsed.netloc + # strip username and password if present if parsed.username or parsed.password: - netloc = parsed.netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') - else: - netloc = parsed.netloc + netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') + + # strip standard protocol numbers + # Note: strictly speaking, standard port numbers should only be + # stripped when comparing origins + if parsed.port: + if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)): + netloc = netloc.replace(':{p.port}'.format(p=parsed), '') + return urlunsplit(( parsed.scheme, netloc, - '' if origin_only else parsed.path, + '/' if origin_only else parsed.path, '' if origin_only else parsed.query, '')) + def origin(self, url): + return self.strip_url(url, origin_only=True) + class NoReferrerPolicy(ReferrerPolicy): """ @@ -58,7 +79,7 @@ class NoReferrerPolicy(ReferrerPolicy): is to be sent along with requests made from a particular request client to any origin. The header will be omitted entirely. """ - name = "no-referrer" + name = POLICY_NO_REFERRER def referrer(self, response, request): return None @@ -79,7 +100,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): This is a user agent's default behavior, if no policy is otherwise specified. """ - name = "no-referrer-when-downgrade" + name = POLICY_NO_REFERRER_WHEN_DOWNGRADE def referrer(self, response, request): target_url = request.url @@ -108,12 +129,12 @@ class SameOriginPolicy(ReferrerPolicy): Cross-origin requests, on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = "same-origin" + name = POLICY_SAME_ORIGIN def referrer(self, response, request): target_url = request.url referrer_source = response.url - if urlsplit(referrer_source).netloc == urlsplit(target_url).netloc: + if self.origin(referrer_source) == self.origin(target_url): return self.strip_url(referrer_source) else: return None @@ -128,10 +149,10 @@ class OriginPolicy(ReferrerPolicy): when making both same-origin requests and cross-origin requests from a particular request client. """ - name = "origin" + name = POLICY_ORIGIN def referrer(self, response, request): - return self.strip_url(referrer_source, origin_only=True) + return self.strip_url(response.url, origin_only=True) class OriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -145,17 +166,17 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): is sent as referrer information when making cross-origin requests from a particular request client. """ - name = "origin-when-cross-origin" + name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response, request): target_url = request.url referrer_source = response.url + source_origin = self.origin(referrer_source) + if source_origin == self.origin(target_url): + return self.strip_url(referrer_source, origin_only=False) + else: + return source_origin - # same origin --> send full referrer - # different origin --> send only "origin" as referrer - if urlsplit(referrer_source).netloc != urlsplit(target_url).netloc: - origin_only = True - return self.strip_url(referrer_source, origin_only=origin_only) class UnsafeUrlPolicy(ReferrerPolicy): @@ -171,7 +192,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): to insecure origins. Carefully consider the impact of setting such a policy for potentially sensitive documents. """ - name = "unsafe-url" + name = POLICY_UNSAFE_URL def referrer(self, response, request): referrer_source = response.url @@ -186,15 +207,17 @@ class LegacyPolicy(ReferrerPolicy): class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') + name = POLICY_SCRAPY_DEFAULT -_policies = {p.name: p for p in ( +_policy_classes = {p.name: p for p in ( NoReferrerPolicy, NoReferrerWhenDowngradePolicy, SameOriginPolicy, OriginPolicy, OriginWhenCrossOriginPolicy, UnsafeUrlPolicy, + DefaultReferrerPolicy, )} class RefererMiddleware(object): @@ -211,10 +234,11 @@ class RefererMiddleware(object): def policy(self, response, request): policy_name = request.meta.get('referrer_policy') if policy_name is None: - policy_name = to_native_str(response.headers.get('Referrer-Policy', '').decode('latin1')) + policy_name = to_native_str( + response.headers.get('Referrer-Policy', '').decode('latin1')) - policy_class = _policies.get(policy_name.lower(), self.default_policy) - return policy_class() + cls = _policy_classes.get(policy_name.lower(), self.default_policy) + return cls() def process_spider_output(self, response, result, spider): def _set_referer(r): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index f109bb248..8458fe90b 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -2,7 +2,10 @@ from unittest import TestCase from scrapy.http import Response, Request from scrapy.spiders import Spider -from scrapy.spidermiddlewares.referer import RefererMiddleware +from scrapy.spidermiddlewares.referer import RefererMiddleware, \ + POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ + POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ + POLICY_UNSAFE_URL class TestRefererMiddleware(TestCase): @@ -25,71 +28,206 @@ class TestRefererMiddleware(TestCase): with some additional filtering of s3:// """ - # a) https:// --> https:// -- include Referer header - origin = Response('https://example.com/') - target = Request('https://scrapy.org/') + for origin, target, referrer in [ + ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'), + ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'), + ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'), + ('https://example.com/', 'http://scrapy.org/', None), - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'https://example.com/') + # no credentials leak + ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'), - # b.1) http:// --> http:// -- include Referer header - origin = Response('http://example.com/') - target = Request('http://scrapy.org/') + # no referrer leak for local schemes + ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://example.com/') + # no referrer leak for s3 origins + ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None), + ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None), + ]: + response = Response(origin) + request = Request(target) - # b.2) http:// --> https:// -- include Referer header - origin = Response('http://example.com/') - target = Request('https://scrapy.org/') + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://example.com/') + def test_policy_no_referrer(self): - # c) https:// --> http:// -- Referer header NOT sent - origin = Response('https://example.com/') - target = Request('http://scrapy.org/') + for origin, target, referrer in [ + ('https://example.com/page.html', 'https://example.com/', None), + ('http://www.example.com/', 'https://scrapy.org/', None), + ('http://www.example.com/', 'http://scrapy.org/', None), + ('https://www.example.com/', 'http://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER}) - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) - def test_policy_default_no_credentials_leak(self): - origin = Response('http://user:password@example.com/') - target = Request('https://scrapy.org/') + def test_policy_no_referrer_when_downgrade(self): - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://example.com/') + for origin, target, referrer in [ + # TLS to TLS: send non-empty referrer + ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), + ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'), + ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'), - def test_policy_default_file_no_referrer_leak(self): - # file:// --> https:// -- Referrer NOT sent - origin = Response('file:///home/path/to/somefile.html') - target = Request('https://scrapy.org/') + # TLS to non-TLS: do not send referrer + ('https://example.com/page.html', 'http://not.example.com/', None), + ('https://example.com/page.html', 'http://scrapy.org/', None), + ('ftps://example.com/urls.zip', 'http://scrapy.org/', None), - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + # non-TLS to TLS or non-TLS: send referrer + ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'), + ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'), + ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'), + ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'), - # file:// --> http:// -- Referrer NOT sent - origin = Response('file:///home/path/to/somefile.html') - target = Request('http://scrapy.org/') + # test for user/password stripping + ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE}) - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) - def test_policy_default_s3_no_referrer_leak(self): - # s3:// --> https:// -- Referrer NOT sent - origin = Response('s3://mybucket/path/to/data.csv') - target = Request('https://scrapy.org/') + def test_policy_same_origin(self): - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + for origin, target, referrer in [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), - # s3:// --> http:// -- Referrer NOT sent - origin = Response('s3://mybucket/path/to/data.csv') - target = Request('http://scrapy.org/') + # Different host: do NOT send referrer + ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None), - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + # Different port: do NOT send referrer + ('https://example.com:444/page.html', 'https://example.com/not-page.html', None), + ('http://example.com:81/page.html', 'http://example.com/not-page.html', None), + ('http://example.com/page.html', 'http://example.com:81/not-page.html', None), + + # Different protocols: do NOT send refferer + ('https://example.com/page.html', 'http://example.com/not-page.html', None), + ('https://example.com/page.html', 'http://not.example.com/', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_SAME_ORIGIN}) + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) + + def test_policy_origin(self): + + for origin, target, referrer in [ + # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), + ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_ORIGIN}) + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) + + def test_policy_origin_when_cross_origin(self): + + for origin, target, referrer in [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: send origin as referrer + ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), + ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), + ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + # exact match required + ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), + + # Different port: send origin as referrer + ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), + ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), + + # Different protocols: send origin as referrer + ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'), + ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + + # test for user/password stripping + ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + # TLS to non-TLS downgrade: send origin + ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}) + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) + + def test_policy_unsafe_url(self): + + for origin, target, referrer in [ + # TLS to TLS: send referrer + ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), + ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'), + ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'), + ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'), + + # TLS to non-TLS: send referrer (yes, it's unsafe) + ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'), + ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'), + ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'), + + # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe) + ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'), + ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'), + ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'), + ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), + + # test for user/password stripping + ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), + ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_UNSAFE_URL}) + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) From 3af88a2877f947c74c6d9003da620629b27d5a17 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 18:27:18 +0200 Subject: [PATCH 03/41] Use urlparse_cached() on request and responses --- scrapy/spidermiddlewares/referer.py | 79 ++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 19 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 1895aa95d..cbbcd3e97 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,11 +2,12 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ -from six.moves.urllib.parse import urlsplit, urlunsplit +from six.moves.urllib.parse import urlparse, urlunparse, ParseResult from scrapy.http import Request from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_native_str +from scrapy.utils.httpobj import urlparse_cached LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) @@ -27,7 +28,7 @@ class ReferrerPolicy(object): def referrer(self, response, request): raise NotImplementedError() - def strip_url(self, url, origin_only=False): + def strip_url_parsed(self, req_or_resp, origin_only=False): """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -41,9 +42,9 @@ class ReferrerPolicy(object): Set url's query to null. Return url. """ - if url is None or not url: + if req_or_resp.url is None or not req_or_resp.url: return None - parsed = urlsplit(url, allow_fragments=True) + parsed = urlparse_cached(req_or_resp) if parsed.scheme in self.NOREFERRER_SCHEMES: return None @@ -60,16 +61,61 @@ class ReferrerPolicy(object): if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)): netloc = netloc.replace(':{p.port}'.format(p=parsed), '') - return urlunsplit(( + return ParseResult(parsed.scheme, + netloc, + '/' if origin_only else parsed.path, + '' if origin_only else parsed.params, + '' if origin_only else parsed.query, + '') + + def strip_url(self, url, origin_only=False): + """ + https://www.w3.org/TR/referrer-policy/#strip-url + + If url is null, return no referrer. + If url's scheme is a local scheme, then return no referrer. + Set url's username to the empty string. + Set url's password to null. + Set url's fragment to null. + If the origin-only flag is true, then: + Set url's path to null. + Set url's query to null. + Return url. + """ + if url is None or not url: + return None + parsed = urlparse(url, allow_fragments=True) + + if parsed.scheme in self.NOREFERRER_SCHEMES: + return None + + netloc = parsed.netloc + # strip username and password if present + if parsed.username or parsed.password: + netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') + + # strip standard protocol numbers + # Note: strictly speaking, standard port numbers should only be + # stripped when comparing origins + if parsed.port: + if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)): + netloc = netloc.replace(':{p.port}'.format(p=parsed), '') + + return urlunparse(( parsed.scheme, netloc, '/' if origin_only else parsed.path, + '' if origin_only else parsed.params, '' if origin_only else parsed.query, '')) def origin(self, url): return self.strip_url(url, origin_only=True) + def origin_parsed(self, req_or_resp): + """Return (scheme, host, path) tuple for a request or response URL.""" + return tuple(self.strip_url_parsed(req_or_resp, origin_only=True)[:3]) + class NoReferrerPolicy(ReferrerPolicy): """ @@ -103,20 +149,17 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): name = POLICY_NO_REFERRER_WHEN_DOWNGRADE def referrer(self, response, request): - target_url = request.url - - referrer_source = response.url - referrer_url = self.strip_url(referrer_source) - # https://www.w3.org/TR/referrer-policy/#determine-requests-referrer: # # If environment is TLS-protected # and the origin of request's current URL is not an a priori authenticated URL, # then return no referrer. - if urlsplit(referrer_source).scheme in ('https', 'ftps') and \ - urlsplit(target_url).scheme in ('http',): + if urlparse_cached(response).scheme in ('https', 'ftps') and \ + urlparse_cached(request).scheme in ('http',): return None - return referrer_url + stripped = self.strip_url_parsed(response) + if stripped is not None: + return urlunparse(stripped) class SameOriginPolicy(ReferrerPolicy): @@ -132,12 +175,10 @@ class SameOriginPolicy(ReferrerPolicy): name = POLICY_SAME_ORIGIN def referrer(self, response, request): - target_url = request.url - referrer_source = response.url - if self.origin(referrer_source) == self.origin(target_url): - return self.strip_url(referrer_source) - else: - return None + if self.origin_parsed(response) == self.origin_parsed(request): + stripped = self.strip_url_parsed(response) + if stripped is not None: + return urlunparse(stripped) class OriginPolicy(ReferrerPolicy): From f2ee6be3bb311bb8f89ccd929bc0ebd8412d0183 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 18:33:31 +0200 Subject: [PATCH 04/41] Use urlparse_cached() for OriginPolicy --- scrapy/spidermiddlewares/referer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index cbbcd3e97..bf2a3c037 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -193,7 +193,9 @@ class OriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN def referrer(self, response, request): - return self.strip_url(response.url, origin_only=True) + stripped = self.strip_url_parsed(response, origin_only=True) + if stripped is not None: + return urlunparse(stripped) class OriginWhenCrossOriginPolicy(ReferrerPolicy): From 59cb884ace1cb1c3339f1d0f05895501dd4d0447 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 18:44:46 +0200 Subject: [PATCH 05/41] Use urlparse_cached() for OriginWhenCrossOriginPolicy --- scrapy/spidermiddlewares/referer.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index bf2a3c037..60f21ae84 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -193,9 +193,9 @@ class OriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN def referrer(self, response, request): - stripped = self.strip_url_parsed(response, origin_only=True) - if stripped is not None: - return urlunparse(stripped) + origin = self.strip_url_parsed(response, origin_only=True) + if origin is not None: + return urlunparse(origin) class OriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -212,14 +212,13 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response, request): - target_url = request.url - referrer_source = response.url - source_origin = self.origin(referrer_source) - if source_origin == self.origin(target_url): - return self.strip_url(referrer_source, origin_only=False) + origin = self.origin_parsed(response) + if origin == self.origin_parsed(request): + stripped = self.strip_url_parsed(response) + if stripped is not None: + return urlunparse(stripped) else: - return source_origin - + return urlunparse(origin + ('', '', '')) class UnsafeUrlPolicy(ReferrerPolicy): From f6a800fde67447d94480d4271de62f5e9a60bfb0 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 18:55:35 +0200 Subject: [PATCH 06/41] Remove all non-cached urlparsing references --- scrapy/spidermiddlewares/referer.py | 71 ++++++----------------------- 1 file changed, 14 insertions(+), 57 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 60f21ae84..46351576f 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,7 +2,7 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ -from six.moves.urllib.parse import urlparse, urlunparse, ParseResult +from six.moves.urllib.parse import ParseResult, urlunparse from scrapy.http import Request from scrapy.exceptions import NotConfigured @@ -28,7 +28,7 @@ class ReferrerPolicy(object): def referrer(self, response, request): raise NotImplementedError() - def strip_url_parsed(self, req_or_resp, origin_only=False): + def strip_url(self, req_or_resp, origin_only=False): """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -68,53 +68,9 @@ class ReferrerPolicy(object): '' if origin_only else parsed.query, '') - def strip_url(self, url, origin_only=False): - """ - https://www.w3.org/TR/referrer-policy/#strip-url - - If url is null, return no referrer. - If url's scheme is a local scheme, then return no referrer. - Set url's username to the empty string. - Set url's password to null. - Set url's fragment to null. - If the origin-only flag is true, then: - Set url's path to null. - Set url's query to null. - Return url. - """ - if url is None or not url: - return None - parsed = urlparse(url, allow_fragments=True) - - if parsed.scheme in self.NOREFERRER_SCHEMES: - return None - - netloc = parsed.netloc - # strip username and password if present - if parsed.username or parsed.password: - netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') - - # strip standard protocol numbers - # Note: strictly speaking, standard port numbers should only be - # stripped when comparing origins - if parsed.port: - if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)): - netloc = netloc.replace(':{p.port}'.format(p=parsed), '') - - return urlunparse(( - parsed.scheme, - netloc, - '/' if origin_only else parsed.path, - '' if origin_only else parsed.params, - '' if origin_only else parsed.query, - '')) - - def origin(self, url): - return self.strip_url(url, origin_only=True) - - def origin_parsed(self, req_or_resp): + def origin(self, req_or_resp): """Return (scheme, host, path) tuple for a request or response URL.""" - return tuple(self.strip_url_parsed(req_or_resp, origin_only=True)[:3]) + return tuple(self.strip_url(req_or_resp, origin_only=True)[:3]) class NoReferrerPolicy(ReferrerPolicy): @@ -157,7 +113,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): if urlparse_cached(response).scheme in ('https', 'ftps') and \ urlparse_cached(request).scheme in ('http',): return None - stripped = self.strip_url_parsed(response) + stripped = self.strip_url(response) if stripped is not None: return urlunparse(stripped) @@ -175,8 +131,8 @@ class SameOriginPolicy(ReferrerPolicy): name = POLICY_SAME_ORIGIN def referrer(self, response, request): - if self.origin_parsed(response) == self.origin_parsed(request): - stripped = self.strip_url_parsed(response) + if self.origin(response) == self.origin(request): + stripped = self.strip_url(response) if stripped is not None: return urlunparse(stripped) @@ -193,7 +149,7 @@ class OriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN def referrer(self, response, request): - origin = self.strip_url_parsed(response, origin_only=True) + origin = self.strip_url(response, origin_only=True) if origin is not None: return urlunparse(origin) @@ -212,9 +168,9 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response, request): - origin = self.origin_parsed(response) - if origin == self.origin_parsed(request): - stripped = self.strip_url_parsed(response) + origin = self.origin(response) + if origin == self.origin(request): + stripped = self.strip_url(response) if stripped is not None: return urlunparse(stripped) else: @@ -237,8 +193,9 @@ class UnsafeUrlPolicy(ReferrerPolicy): name = POLICY_UNSAFE_URL def referrer(self, response, request): - referrer_source = response.url - return self.strip_url(referrer_source) + stripped = self.strip_url(response) + if stripped is not None: + return urlunparse(stripped) class LegacyPolicy(ReferrerPolicy): From f6205778f31dae3ac0f60528a7aa4ef3ef6a181a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 19:06:26 +0200 Subject: [PATCH 07/41] Refactor ReferrerPolicy methods --- scrapy/spidermiddlewares/referer.py | 30 ++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 46351576f..44a599433 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -28,6 +28,16 @@ class ReferrerPolicy(object): def referrer(self, response, request): raise NotImplementedError() + def stripped_referrer(self, req_or_resp): + stripped = self.strip_url(req_or_resp) + if stripped is not None: + return urlunparse(stripped) + + def origin_referrer(self, req_or_resp): + stripped = self.strip_url(req_or_resp, origin_only=True) + if stripped is not None: + return urlunparse(stripped) + def strip_url(self, req_or_resp, origin_only=False): """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -113,9 +123,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): if urlparse_cached(response).scheme in ('https', 'ftps') and \ urlparse_cached(request).scheme in ('http',): return None - stripped = self.strip_url(response) - if stripped is not None: - return urlunparse(stripped) + return self.stripped_referrer(response) class SameOriginPolicy(ReferrerPolicy): @@ -132,9 +140,7 @@ class SameOriginPolicy(ReferrerPolicy): def referrer(self, response, request): if self.origin(response) == self.origin(request): - stripped = self.strip_url(response) - if stripped is not None: - return urlunparse(stripped) + return self.stripped_referrer(response) class OriginPolicy(ReferrerPolicy): @@ -149,9 +155,7 @@ class OriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN def referrer(self, response, request): - origin = self.strip_url(response, origin_only=True) - if origin is not None: - return urlunparse(origin) + return self.origin_referrer(response) class OriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -170,9 +174,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): def referrer(self, response, request): origin = self.origin(response) if origin == self.origin(request): - stripped = self.strip_url(response) - if stripped is not None: - return urlunparse(stripped) + return self.stripped_referrer(response) else: return urlunparse(origin + ('', '', '')) @@ -193,9 +195,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): name = POLICY_UNSAFE_URL def referrer(self, response, request): - stripped = self.strip_url(response) - if stripped is not None: - return urlunparse(stripped) + return self.stripped_referrer(response) class LegacyPolicy(ReferrerPolicy): From 842ce131aa666dce0db61aeee701c420050a5a9c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 11 Oct 2016 18:27:31 +0200 Subject: [PATCH 08/41] Make default referrer policy customizable via settings --- scrapy/settings/default_settings.py | 1 + scrapy/spidermiddlewares/referer.py | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a5931a3d5..15a134dd3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -234,6 +234,7 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True +REFERER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 44a599433..88041d8f0 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -8,6 +8,7 @@ from scrapy.http import Request from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_native_str from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import load_object LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) @@ -221,16 +222,27 @@ _policy_classes = {p.name: p for p in ( class RefererMiddleware(object): - def __init__(self, policy_class=DefaultReferrerPolicy): - self.default_policy = policy_class + def __init__(self, settings={}): + policy = settings.get('REFERER_POLICY') + if policy is not None: + try: + self.default_policy = load_object(policy) + except ValueError: + try: + self.default_policy = _policy_classes[policy] + except: + raise NotConfigured("Unknown referrer policy name %r" % policy) + else: + self.default_policy = DefaultReferrerPolicy @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured - return cls() + return cls(crawler.settings) def policy(self, response, request): + # policy set in request's meta dict takes precedence over default policy policy_name = request.meta.get('referrer_policy') if policy_name is None: policy_name = to_native_str( From e72b6e33611f61dbb358f1ff97c5c195a7db696f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 11 Oct 2016 18:27:56 +0200 Subject: [PATCH 09/41] Add tests for referrer policy via settings and via Request meta --- tests/test_spidermiddleware_referer.py | 501 ++++++++++++++----------- 1 file changed, 292 insertions(+), 209 deletions(-) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 8458fe90b..b724d7999 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,233 +1,316 @@ from unittest import TestCase +from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request +from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.spidermiddlewares.referer import RefererMiddleware, \ POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ - POLICY_UNSAFE_URL + POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \ + DefaultReferrerPolicy, \ + NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ + OriginWhenCrossOriginPolicy, OriginPolicy, \ + SameOriginPolicy, UnsafeUrlPolicy class TestRefererMiddleware(TestCase): + req_meta = {} + resp_headers = {} + settings = {} + scenarii = [ + ('http://scrapytest.org', 'http://scrapytest.org/', b'http://scrapytest.org'), + ] + def setUp(self): self.spider = Spider('foo') - self.mw = RefererMiddleware() + settings = Settings(self.settings) + self.mw = RefererMiddleware(settings) - def test_process_spider_output(self): - res = Response('http://scrapytest.org') - reqs = [Request('http://scrapytest.org/')] + def get_request(self, target): + return Request(target, meta=self.req_meta) - out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://scrapytest.org') + def get_response(self, origin): + return Response(origin, headers=self.resp_headers) - def test_policy_default(self): - """ - Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + def test(self): - with some additional filtering of s3:// - """ - for origin, target, referrer in [ - ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'), - ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'), - ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'), - ('https://example.com/', 'http://scrapy.org/', None), - - # no credentials leak - ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'), - - # no referrer leak for local schemes - ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None), - ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), - - # no referrer leak for s3 origins - ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None), - ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None), - ]: - response = Response(origin) - request = Request(target) + for origin, target, referrer in self.scenarii: + response = self.get_response(origin) + request = self.get_request(target) out = list(self.mw.process_spider_output(response, [request], self.spider)) self.assertEquals(out[0].headers.get('Referer'), referrer) - def test_policy_no_referrer(self): - for origin, target, referrer in [ - ('https://example.com/page.html', 'https://example.com/', None), - ('http://www.example.com/', 'https://scrapy.org/', None), - ('http://www.example.com/', 'http://scrapy.org/', None), - ('https://www.example.com/', 'http://scrapy.org/', None), - ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), +class MixinDefault(object): + """ + Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + + with some additional filtering of s3:// + """ + scenarii = [ + ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'), + ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'), + ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'), + ('https://example.com/', 'http://scrapy.org/', None), + + # no credentials leak + ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'), + + # no referrer leak for local schemes + ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + + # no referrer leak for s3 origins + ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None), + ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None), + ] + + +class MixinNoReferrer(object): + scenarii = [ + ('https://example.com/page.html', 'https://example.com/', None), + ('http://www.example.com/', 'https://scrapy.org/', None), + ('http://www.example.com/', 'http://scrapy.org/', None), + ('https://www.example.com/', 'http://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + ] + + +class MixinNoReferrerWhenDowngrade(object): + scenarii = [ + # TLS to TLS: send non-empty referrer + ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), + ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'), + ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'), + + # TLS to non-TLS: do not send referrer + ('https://example.com/page.html', 'http://not.example.com/', None), + ('https://example.com/page.html', 'http://scrapy.org/', None), + ('ftps://example.com/urls.zip', 'http://scrapy.org/', None), + + # non-TLS to TLS or non-TLS: send referrer + ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'), + ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'), + ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'), + ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'), + + # test for user/password stripping + ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ] + + +class MixinSameOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: do NOT send referrer + ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None), + + # Different port: do NOT send referrer + ('https://example.com:444/page.html', 'https://example.com/not-page.html', None), + ('http://example.com:81/page.html', 'http://example.com/not-page.html', None), + ('http://example.com/page.html', 'http://example.com:81/not-page.html', None), + + # Different protocols: do NOT send refferer + ('https://example.com/page.html', 'http://example.com/not-page.html', None), + ('https://example.com/page.html', 'http://not.example.com/', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), + ] + + +class MixinOrigin(object): + scenarii = [ + # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), + ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ] + + +class MixinOriginWhenCrossOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: send origin as referrer + ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), + ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), + ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + # exact match required + ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), + + # Different port: send origin as referrer + ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), + ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), + + # Different protocols: send origin as referrer + ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'), + ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + + # test for user/password stripping + ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + # TLS to non-TLS downgrade: send origin + ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), + ] + + +class MixinUnsafeUrl(object): + scenarii = [ + # TLS to TLS: send referrer + ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), + ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'), + ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'), + ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'), + + # TLS to non-TLS: send referrer (yes, it's unsafe) + ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'), + ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'), + ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'), + + # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe) + ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'), + ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'), + ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'), + ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), + + # test for user/password stripping + ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), + ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), + ] + + +class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware): + pass + + +# --- Tests using settings to set policy using class path +class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} + + +class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + + +class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + + +class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} + + +class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + + +class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + + +# --- Tests using Request meta dict to set policy +class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT} + + +class TestRefererMiddlewareNoReferrer(MixinNoReferrer, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_NO_REFERRER} + + +class TestRefererMiddlewareNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE} + + +class TestRefererMiddlewareSameOrigin(MixinSameOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_SAME_ORIGIN} + + +class TestRefererMiddlewareOrigin(MixinOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_ORIGIN} + + +class TestRefererMiddlewareOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} + + +class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + + +class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + +class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + req_meta = {'referrer_policy': POLICY_NO_REFERRER} + + +class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + +class TestRefererMiddlewareSettingsPolicyByName(TestCase): + + def test_valid_name(self): + for s, p in [ + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER}) + settings = Settings({'REFERER_POLICY': s}) + mw = RefererMiddleware(settings) + self.assertEquals(mw.default_policy, p) - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_no_referrer_when_downgrade(self): - - for origin, target, referrer in [ - # TLS to TLS: send non-empty referrer - ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), - ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), - ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), - ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'), - ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'), - - # TLS to non-TLS: do not send referrer - ('https://example.com/page.html', 'http://not.example.com/', None), - ('https://example.com/page.html', 'http://scrapy.org/', None), - ('ftps://example.com/urls.zip', 'http://scrapy.org/', None), - - # non-TLS to TLS or non-TLS: send referrer - ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), - ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'), - ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'), - ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'), - ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'), - ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'), - ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'), - - # test for user/password stripping - ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_same_origin(self): - - for origin, target, referrer in [ - # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), - - # Different host: do NOT send referrer - ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), - ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None), - ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None), - - # Different port: do NOT send referrer - ('https://example.com:444/page.html', 'https://example.com/not-page.html', None), - ('http://example.com:81/page.html', 'http://example.com/not-page.html', None), - ('http://example.com/page.html', 'http://example.com:81/not-page.html', None), - - # Different protocols: do NOT send refferer - ('https://example.com/page.html', 'http://example.com/not-page.html', None), - ('https://example.com/page.html', 'http://not.example.com/', None), - ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), - ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None), - ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), - - # test for user/password stripping - ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_SAME_ORIGIN}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_origin(self): - - for origin, target, referrer in [ - # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), - ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), - ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'), - ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), - - # test for user/password stripping - ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_ORIGIN}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_origin_when_cross_origin(self): - - for origin, target, referrer in [ - # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), - - # Different host: send origin as referrer - ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), - ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), - ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), - # exact match required - ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), - - # Different port: send origin as referrer - ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), - ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), - - # Different protocols: send origin as referrer - ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'), - ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'), - ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), - ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), - ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), - - # test for user/password stripping - ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), - # TLS to non-TLS downgrade: send origin - ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_unsafe_url(self): - - for origin, target, referrer in [ - # TLS to TLS: send referrer - ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), - ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'), - ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), - ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), - ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'), - ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'), - - # TLS to non-TLS: send referrer (yes, it's unsafe) - ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'), - ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'), - ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'), - - # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe) - ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'), - ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'), - ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'), - ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'), - ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'), - ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'), - ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'), - ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), - - # test for user/password stripping - ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), - ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_UNSAFE_URL}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) + def test_invalid_name(self): + settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'}) + with self.assertRaises(NotConfigured): + mw = RefererMiddleware(settings) From 0344f57fefc0877bf9048a084002fc719335e31c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 11 Oct 2016 19:53:15 +0200 Subject: [PATCH 10/41] Support case-insensitive policy names in settings --- scrapy/spidermiddlewares/referer.py | 2 +- tests/test_spidermiddleware_referer.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 88041d8f0..deda7b284 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -229,7 +229,7 @@ class RefererMiddleware(object): self.default_policy = load_object(policy) except ValueError: try: - self.default_policy = _policy_classes[policy] + self.default_policy = _policy_classes[policy.lower()] except: raise NotConfigured("Unknown referrer policy name %r" % policy) else: diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index b724d7999..b1ab366a7 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -310,6 +310,20 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): mw = RefererMiddleware(settings) self.assertEquals(mw.default_policy, p) + def test_valid_name_casevariants(self): + for s, p in [ + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), + ]: + settings = Settings({'REFERER_POLICY': s.upper()}) + mw = RefererMiddleware(settings) + self.assertEquals(mw.default_policy, p) + def test_invalid_name(self): settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'}) with self.assertRaises(NotConfigured): From ec8b4c1a9bfd004960d730b41078b9f0633627d8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 11 Oct 2016 20:00:34 +0200 Subject: [PATCH 11/41] Change __init__ default "settings" arg handling --- scrapy/spidermiddlewares/referer.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index deda7b284..2a3790bde 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -222,18 +222,18 @@ _policy_classes = {p.name: p for p in ( class RefererMiddleware(object): - def __init__(self, settings={}): - policy = settings.get('REFERER_POLICY') - if policy is not None: - try: - self.default_policy = load_object(policy) - except ValueError: + def __init__(self, settings=None): + self.default_policy = DefaultReferrerPolicy + if settings is not None: + policy = settings.get('REFERER_POLICY') + if policy is not None: try: - self.default_policy = _policy_classes[policy.lower()] - except: - raise NotConfigured("Unknown referrer policy name %r" % policy) - else: - self.default_policy = DefaultReferrerPolicy + self.default_policy = load_object(policy) + except ValueError: + try: + self.default_policy = _policy_classes[policy.lower()] + except: + raise NotConfigured("Unknown referrer policy name %r" % policy) @classmethod def from_crawler(cls, crawler): From e50e670eff2fe1b109f6b5dd026c4c706a93585c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 12 Oct 2016 16:16:53 +0200 Subject: [PATCH 12/41] Add test for custom referrer policy via settings --- scrapy/spidermiddlewares/referer.py | 4 ++++ tests/test_spidermiddleware_referer.py | 30 ++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 2a3790bde..0bba63cb7 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -227,9 +227,12 @@ class RefererMiddleware(object): if settings is not None: policy = settings.get('REFERER_POLICY') if policy is not None: + # expect a string for the path to the policy class try: self.default_policy = load_object(policy) except ValueError: + # otherwise try to interpret the string as standard + # https://www.w3.org/TR/referrer-policy/#referrer-policies try: self.default_policy = _policy_classes[policy.lower()] except: @@ -239,6 +242,7 @@ class RefererMiddleware(object): def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured + return cls(crawler.settings) def policy(self, response, request): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index b1ab366a7..cfc4b5296 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -11,7 +11,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware, \ DefaultReferrerPolicy, \ NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ OriginWhenCrossOriginPolicy, OriginPolicy, \ - SameOriginPolicy, UnsafeUrlPolicy + SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy class TestRefererMiddleware(TestCase): @@ -249,6 +249,33 @@ class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddlewa settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} +class CustomPythonOrgPolicy(ReferrerPolicy): + """ + A dummy policy that returns referrer as http(s)://python.org + depending on the scheme of the target URL. + """ + def referrer(self, response, request): + from scrapy.utils.httpobj import urlparse_cached + + scheme = urlparse_cached(request).scheme + if scheme == 'https': + return b'https://python.org/' + elif scheme == 'http': + return b'http://python.org/' + + +class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware): + settings = {'REFERER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} + scenarii = [ + ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), + ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'), + ('http://example.com/', 'https://scrapy.org/', b'https://python.org/'), + ('https://example.com/', 'http://scrapy.org/', b'http://python.org/'), + ('file:///home/path/to/somefile.html', 'https://scrapy.org/', b'https://python.org/'), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', b'http://python.org/'), + + ] + # --- Tests using Request meta dict to set policy class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT} @@ -278,7 +305,6 @@ class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_UNSAFE_URL} - class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} From d3d4d66ce8e5d01aa1fd6013a6d63337bb931460 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 12 Oct 2016 16:30:25 +0200 Subject: [PATCH 13/41] Add tests for referrer-policy set in response HTTP headers --- tests/test_spidermiddleware_referer.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index cfc4b5296..9555817d9 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -354,3 +354,16 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'}) with self.assertRaises(NotConfigured): mw = RefererMiddleware(settings) + + +class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} + +class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} + +class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} From 285d5bc03a7ebe8eaa558a5c24ff0693353c3e87 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 12 Oct 2016 17:34:12 +0200 Subject: [PATCH 14/41] Patch "Referer" header on HTTP redirects if necessary --- scrapy/spidermiddlewares/referer.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 0bba63cb7..01f1fdf85 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -4,8 +4,9 @@ originated it. """ from six.moves.urllib.parse import ParseResult, urlunparse -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured +from scrapy import signals from scrapy.utils.python import to_native_str from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object @@ -242,8 +243,9 @@ class RefererMiddleware(object): def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured - - return cls(crawler.settings) + mw = cls(crawler.settings) + crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled) + return mw def policy(self, response, request): # policy set in request's meta dict takes precedence over default policy @@ -264,3 +266,18 @@ class RefererMiddleware(object): return r return (_set_referer(r) for r in result or ()) + def request_scheduled(self, request, spider): + # check redirected request to patch "Referer" header if necessary + redirected_urls = request.meta.get('redirect_urls', []) + if redirected_urls: + request_referrer = request.headers.get('Referer') + # we don't patch the referrer value if there is none + if request_referrer is not None: + faked_response = Response(redirected_urls[0]) + policy_referrer = self.policy(faked_response, + request).referrer(faked_response, request) + if policy_referrer != request_referrer: + if policy_referrer is None: + request.headers.pop('Referer') + else: + request.headers['Referer'] = policy_referrer From c9c59db489575131d573e130015fd1fb6b133882 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 12 Oct 2016 18:29:47 +0200 Subject: [PATCH 15/41] Update documentation about REFERER_POLICY setting --- docs/topics/spider-middleware.rst | 38 ++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 8360827e8..a9d3d4568 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -95,7 +95,7 @@ following methods: it has processed the response. :meth:`process_spider_output` must return an iterable of - :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` + :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` objects. :param response: the response which generated this output from the @@ -328,6 +328,42 @@ Default: ``True`` Whether to enable referer middleware. +.. setting:: REFERER_POLICY + +REFERER_POLICY +^^^^^^^^^^^^^^ + +.. versionadded:: 1.3 + +Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` + +`Referrer Policy`_ to apply when populating Request "Referer" header. + +This setting accepts: + +- a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass, + either a custom one or one of the built-in ones + (see ``scrapy.spidermiddlewares.referer``), +- or one of the standard W3C-defined string values, i.e. ``"no-referrer"``, + ``"no-referrer-when-downgrade"``, ``"same-origin"``, ``"origin"``, + ``"origin-when-cross-origin"`` or ``"unsafe-url"``. + (It can also be the non-standard value ``"scrapy-default"`` to use + Scrapy's default referrer policy.) + +Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_, +with the addition that "Referrer" is not sent if the parent request was +using ``file://`` or ``s3://`` scheme. + +.. warning:: + By default, Scrapy's default referrer policy, just like `"no-referrer-when-downgrade"`_, + will send a non-empty "Referer" header from any ``https://`` to any ``https://`` URL, + even if the domain is different. + ``same-origin`` may be a better choice if you want to remove referrer + information for cross-domain requests. + +.. _Referrer Policy: https://www.w3.org/TR/referrer-policy +.. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + UrlLengthMiddleware ------------------- From 5dd7311cd48e676147138746229d7ab2b429b8a9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 19 Oct 2016 14:45:33 +0200 Subject: [PATCH 16/41] Move URL credentials stripping to a helper function --- scrapy/spidermiddlewares/referer.py | 43 +++----------- scrapy/utils/url.py | 34 ++++++++++- tests/test_utils_url.py | 92 ++++++++++++++++++++++++++++- 3 files changed, 133 insertions(+), 36 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 01f1fdf85..e40e798b8 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,14 +2,13 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ -from six.moves.urllib.parse import ParseResult, urlunparse - from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals from scrapy.utils.python import to_native_str from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object +from scrapy.utils.url import strip_url_credentials LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) @@ -31,14 +30,10 @@ class ReferrerPolicy(object): raise NotImplementedError() def stripped_referrer(self, req_or_resp): - stripped = self.strip_url(req_or_resp) - if stripped is not None: - return urlunparse(stripped) + return self.strip_url(req_or_resp) def origin_referrer(self, req_or_resp): - stripped = self.strip_url(req_or_resp, origin_only=True) - if stripped is not None: - return urlunparse(stripped) + return self.strip_url(req_or_resp, origin_only=True) def strip_url(self, req_or_resp, origin_only=False): """ @@ -56,33 +51,13 @@ class ReferrerPolicy(object): """ if req_or_resp.url is None or not req_or_resp.url: return None - parsed = urlparse_cached(req_or_resp) - - if parsed.scheme in self.NOREFERRER_SCHEMES: - return None - - netloc = parsed.netloc - # strip username and password if present - if parsed.username or parsed.password: - netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') - - # strip standard protocol numbers - # Note: strictly speaking, standard port numbers should only be - # stripped when comparing origins - if parsed.port: - if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)): - netloc = netloc.replace(':{p.port}'.format(p=parsed), '') - - return ParseResult(parsed.scheme, - netloc, - '/' if origin_only else parsed.path, - '' if origin_only else parsed.params, - '' if origin_only else parsed.query, - '') + parsed_url = urlparse_cached(req_or_resp) + if parsed_url.scheme not in self.NOREFERRER_SCHEMES: + return strip_url_credentials(parsed_url, origin_only=origin_only) def origin(self, req_or_resp): - """Return (scheme, host, path) tuple for a request or response URL.""" - return tuple(self.strip_url(req_or_resp, origin_only=True)[:3]) + """Return serialized origin (scheme, host, path) for a request or response URL.""" + return self.strip_url(req_or_resp, origin_only=True) class NoReferrerPolicy(ReferrerPolicy): @@ -178,7 +153,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): if origin == self.origin(request): return self.stripped_referrer(response) else: - return urlunparse(origin + ('', '', '')) + return origin class UnsafeUrlPolicy(ReferrerPolicy): diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index dc1cce4ac..f3ccfb0e8 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead. """ import posixpath import re -from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse) +from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse) # scrapy.utils.url was moved to w3lib.url and import * ensures this # move doesn't break old code @@ -103,3 +103,35 @@ def guess_scheme(url): return any_to_uri(url) else: return add_http_if_no_scheme(url) + + +def strip_url_credentials(url, origin_only=False, keep_fragments=False): + + if url is None: + return None + + if not isinstance(url, ParseResult): + parsed_url = urlparse(url) + else: + parsed_url = url + + netloc = parsed_url.netloc + # strip username and password if present + if parsed_url.username or parsed_url.password: + netloc = netloc.split('@')[-1] + + # strip standard protocol numbers + # Note: strictly speaking, standard port numbers should only be + # stripped when comparing origins + if parsed_url.port: + if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443)): + netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '') + + return urlunparse(( + parsed_url.scheme, + netloc, + '/' if origin_only else parsed_url.path, + '' if origin_only else parsed_url.params, + '' if origin_only else parsed_url.query, + '' if not keep_fragments else parsed_url.fragment + )) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index f46d1d927..f1a5c3196 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -6,7 +6,8 @@ from six.moves.urllib.parse import urlparse from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, - add_http_if_no_scheme, guess_scheme, parse_url) + add_http_if_no_scheme, guess_scheme, + parse_url, strip_url_credentials) __doctests__ = ['scrapy.utils.url'] @@ -241,5 +242,94 @@ for k, args in enumerate ([ setattr (GuessSchemeTest, t_method.__name__, t_method) +class StripUrlCredentials(unittest.TestCase): + + def test_noop(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com/index.html'), + 'http://www.example.com/index.html') + + def test_noop_query_string(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com/index.html?somekey=somevalue'), + 'http://www.example.com/index.html?somekey=somevalue') + + def test_fragments(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com/index.html?somekey=somevalue#section', keep_fragments=True), + 'http://www.example.com/index.html?somekey=somevalue#section') + + def test_noop_trailing_path(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com/'), + 'http://www.example.com/') + + def test_noop_trailing_path2(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com'), + 'http://www.example.com') + + def test_trailing_path_origin(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com', origin_only=True), + 'http://www.example.com/') + + def test_username(self): + # username is stripped (and fragment too) + self.assertEqual(strip_url_credentials( + 'http://username@www.example.com/index.html?somekey=somevalue#section'), + 'http://www.example.com/index.html?somekey=somevalue') + + def test_username_empty_pass(self): + # same as above + self.assertEqual(strip_url_credentials( + 'https://username:@www.example.com/index.html?somekey=somevalue#section'), + 'https://www.example.com/index.html?somekey=somevalue') + + def test_username_password(self): + self.assertEqual(strip_url_credentials( + 'ftp://username:password@www.example.com/index.html?somekey=somevalue#section'), + 'ftp://www.example.com/index.html?somekey=somevalue') + + def test_default_http_port(self): + self.assertEqual(strip_url_credentials( + 'http://username:password@www.example.com:80/index.html'), + 'http://www.example.com/index.html') + + def test_non_default_http_port(self): + self.assertEqual(strip_url_credentials( + 'http://username:password@www.example.com:8080/index.html'), + 'http://www.example.com:8080/index.html') + + def test_default_https_port(self): + self.assertEqual(strip_url_credentials( + 'https://username:password@www.example.com:443/index.html'), + 'https://www.example.com/index.html') + + def test_non_default_https_port(self): + self.assertEqual(strip_url_credentials( + 'https://username:password@www.example.com:442/index.html'), + 'https://www.example.com:442/index.html') + + def test_origin_only(self): + self.assertEqual(strip_url_credentials( + 'http://username:password@www.example.com/index.html', origin_only=True), + 'http://www.example.com/') + + def test_default_http_port_origin_only(self): + self.assertEqual(strip_url_credentials( + 'http://username:password@www.example.com:80/index.html', origin_only=True), + 'http://www.example.com/') + + def test_non_default_http_port_origin_only(self): + self.assertEqual(strip_url_credentials( + 'http://username:password@www.example.com:8008/index.html', origin_only=True), + 'http://www.example.com:8008/') + + def test_default_https_port_origin_only(self): + self.assertEqual(strip_url_credentials( + 'https://username:password@www.example.com:443/index.html', origin_only=True), + 'https://www.example.com/') + if __name__ == "__main__": unittest.main() From 8864d0e8c19d0ddb52175be1952a4013b13cd86f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 24 Oct 2016 18:21:49 +0200 Subject: [PATCH 17/41] Rename helper function to strip_url() + add more tests --- scrapy/spidermiddlewares/referer.py | 8 +- scrapy/utils/url.py | 30 +++-- tests/test_utils_url.py | 190 +++++++++++++++++++--------- 3 files changed, 150 insertions(+), 78 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index e40e798b8..24e5eac40 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -8,7 +8,7 @@ from scrapy import signals from scrapy.utils.python import to_native_str from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object -from scrapy.utils.url import strip_url_credentials +from scrapy.utils.url import strip_url LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) @@ -53,7 +53,11 @@ class ReferrerPolicy(object): return None parsed_url = urlparse_cached(req_or_resp) if parsed_url.scheme not in self.NOREFERRER_SCHEMES: - return strip_url_credentials(parsed_url, origin_only=origin_only) + return strip_url(parsed_url, + strip_credentials=True, + strip_fragment=True, + strip_default_port=True, + origin_only=origin_only) def origin(self, req_or_resp): """Return serialized origin (scheme, host, path) for a request or response URL.""" diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index f3ccfb0e8..9864f353d 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -105,33 +105,37 @@ def guess_scheme(url): return add_http_if_no_scheme(url) -def strip_url_credentials(url, origin_only=False, keep_fragments=False): +def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True): + + """Strip URL string from some of its components: + + - `strip_credentials` removes "user:password@" + - `strip_default_port` removes ":80" (resp. ":443", ":21") + from http:// (resp. https://, ftp://) URLs + - `origin_only` replaces path component with "/", also dropping + query and fragment components ; it also strips credentials + - `strip_fragment` drops any #fragment component + """ if url is None: return None - if not isinstance(url, ParseResult): parsed_url = urlparse(url) else: parsed_url = url - netloc = parsed_url.netloc - # strip username and password if present - if parsed_url.username or parsed_url.password: + if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password): netloc = netloc.split('@')[-1] - - # strip standard protocol numbers - # Note: strictly speaking, standard port numbers should only be - # stripped when comparing origins - if parsed_url.port: - if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443)): + if strip_default_port and parsed_url.port: + if (parsed_url.scheme, parsed_url.port) in (('http', 80), + ('https', 443), + ('ftp', 21)): netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '') - return urlunparse(( parsed_url.scheme, netloc, '/' if origin_only else parsed_url.path, '' if origin_only else parsed_url.params, '' if origin_only else parsed_url.query, - '' if not keep_fragments else parsed_url.fragment + '' if strip_fragment else parsed_url.fragment )) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index f1a5c3196..1f9845d82 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -7,7 +7,7 @@ from six.moves.urllib.parse import urlparse from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, add_http_if_no_scheme, guess_scheme, - parse_url, strip_url_credentials) + parse_url, strip_url) __doctests__ = ['scrapy.utils.url'] @@ -242,94 +242,158 @@ for k, args in enumerate ([ setattr (GuessSchemeTest, t_method.__name__, t_method) -class StripUrlCredentials(unittest.TestCase): +class StripUrl(unittest.TestCase): def test_noop(self): - self.assertEqual(strip_url_credentials( + self.assertEqual(strip_url( 'http://www.example.com/index.html'), 'http://www.example.com/index.html') def test_noop_query_string(self): - self.assertEqual(strip_url_credentials( + self.assertEqual(strip_url( 'http://www.example.com/index.html?somekey=somevalue'), 'http://www.example.com/index.html?somekey=somevalue') def test_fragments(self): - self.assertEqual(strip_url_credentials( - 'http://www.example.com/index.html?somekey=somevalue#section', keep_fragments=True), + self.assertEqual(strip_url( + 'http://www.example.com/index.html?somekey=somevalue#section', strip_fragment=False), 'http://www.example.com/index.html?somekey=somevalue#section') - def test_noop_trailing_path(self): - self.assertEqual(strip_url_credentials( - 'http://www.example.com/'), - 'http://www.example.com/') + def test_path(self): + for input_url, origin, output_url in [ + ('http://www.example.com/', + False, + 'http://www.example.com/'), - def test_noop_trailing_path2(self): - self.assertEqual(strip_url_credentials( - 'http://www.example.com'), - 'http://www.example.com') + ('http://www.example.com', + False, + 'http://www.example.com'), - def test_trailing_path_origin(self): - self.assertEqual(strip_url_credentials( - 'http://www.example.com', origin_only=True), - 'http://www.example.com/') + ('http://www.example.com', + True, + 'http://www.example.com/'), + ]: + self.assertEqual(strip_url(input_url, origin_only=origin), output_url) + self.assertEqual(strip_url(urlparse(input_url), origin_only=origin), output_url) - def test_username(self): - # username is stripped (and fragment too) - self.assertEqual(strip_url_credentials( - 'http://username@www.example.com/index.html?somekey=somevalue#section'), - 'http://www.example.com/index.html?somekey=somevalue') + def test_credentials(self): + for i, o in [ + ('http://username@www.example.com/index.html?somekey=somevalue#section', + 'http://www.example.com/index.html?somekey=somevalue'), - def test_username_empty_pass(self): - # same as above - self.assertEqual(strip_url_credentials( - 'https://username:@www.example.com/index.html?somekey=somevalue#section'), - 'https://www.example.com/index.html?somekey=somevalue') + ('https://username:@www.example.com/index.html?somekey=somevalue#section', + 'https://www.example.com/index.html?somekey=somevalue'), - def test_username_password(self): - self.assertEqual(strip_url_credentials( - 'ftp://username:password@www.example.com/index.html?somekey=somevalue#section'), - 'ftp://www.example.com/index.html?somekey=somevalue') + ('ftp://username:password@www.example.com/index.html?somekey=somevalue#section', + 'ftp://www.example.com/index.html?somekey=somevalue'), + ]: + self.assertEqual(strip_url(i, strip_credentials=True), o) + self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o) - def test_default_http_port(self): - self.assertEqual(strip_url_credentials( - 'http://username:password@www.example.com:80/index.html'), - 'http://www.example.com/index.html') + def test_default_ports_creds_off(self): + for i, o in [ + ('http://username:password@www.example.com:80/index.html?somekey=somevalue#section', + 'http://www.example.com/index.html?somekey=somevalue'), - def test_non_default_http_port(self): - self.assertEqual(strip_url_credentials( - 'http://username:password@www.example.com:8080/index.html'), - 'http://www.example.com:8080/index.html') + ('http://username:password@www.example.com:8080/index.html#section', + 'http://www.example.com:8080/index.html'), - def test_default_https_port(self): - self.assertEqual(strip_url_credentials( - 'https://username:password@www.example.com:443/index.html'), - 'https://www.example.com/index.html') + ('http://username:password@www.example.com:443/index.html?somekey=somevalue&someotherkey=sov#section', + 'http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov'), - def test_non_default_https_port(self): - self.assertEqual(strip_url_credentials( - 'https://username:password@www.example.com:442/index.html'), - 'https://www.example.com:442/index.html') + ('https://username:password@www.example.com:443/index.html', + 'https://www.example.com/index.html'), + + ('https://username:password@www.example.com:442/index.html', + 'https://www.example.com:442/index.html'), + + ('https://username:password@www.example.com:80/index.html', + 'https://www.example.com:80/index.html'), + + ('ftp://username:password@www.example.com:21/file.txt', + 'ftp://www.example.com/file.txt'), + + ('ftp://username:password@www.example.com:221/file.txt', + 'ftp://www.example.com:221/file.txt'), + ]: + self.assertEqual(strip_url(i), o) + self.assertEqual(strip_url(urlparse(i)), o) + + def test_default_ports(self): + for i, o in [ + ('http://username:password@www.example.com:80/index.html', + 'http://username:password@www.example.com/index.html'), + + ('http://username:password@www.example.com:8080/index.html', + 'http://username:password@www.example.com:8080/index.html'), + + ('http://username:password@www.example.com:443/index.html', + 'http://username:password@www.example.com:443/index.html'), + + ('https://username:password@www.example.com:443/index.html', + 'https://username:password@www.example.com/index.html'), + + ('https://username:password@www.example.com:442/index.html', + 'https://username:password@www.example.com:442/index.html'), + + ('https://username:password@www.example.com:80/index.html', + 'https://username:password@www.example.com:80/index.html'), + + ('ftp://username:password@www.example.com:21/file.txt', + 'ftp://username:password@www.example.com/file.txt'), + + ('ftp://username:password@www.example.com:221/file.txt', + 'ftp://username:password@www.example.com:221/file.txt'), + ]: + self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o) + self.assertEqual(strip_url(urlparse(i), strip_default_port=True, strip_credentials=False), o) + + def test_default_ports_keep(self): + for i, o in [ + ('http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov#section', + 'http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov'), + + ('http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov#section', + 'http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov'), + + ('http://username:password@www.example.com:443/index.html', + 'http://username:password@www.example.com:443/index.html'), + + ('https://username:password@www.example.com:443/index.html', + 'https://username:password@www.example.com:443/index.html'), + + ('https://username:password@www.example.com:442/index.html', + 'https://username:password@www.example.com:442/index.html'), + + ('https://username:password@www.example.com:80/index.html', + 'https://username:password@www.example.com:80/index.html'), + + ('ftp://username:password@www.example.com:21/file.txt', + 'ftp://username:password@www.example.com:21/file.txt'), + + ('ftp://username:password@www.example.com:221/file.txt', + 'ftp://username:password@www.example.com:221/file.txt'), + ]: + self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o) + self.assertEqual(strip_url(urlparse(i), strip_default_port=False, strip_credentials=False), o) def test_origin_only(self): - self.assertEqual(strip_url_credentials( - 'http://username:password@www.example.com/index.html', origin_only=True), - 'http://www.example.com/') + for i, o in [ + ('http://username:password@www.example.com/index.html', + 'http://www.example.com/'), - def test_default_http_port_origin_only(self): - self.assertEqual(strip_url_credentials( - 'http://username:password@www.example.com:80/index.html', origin_only=True), - 'http://www.example.com/') + ('http://username:password@www.example.com:80/foo/bar?query=value#somefrag', + 'http://www.example.com/'), - def test_non_default_http_port_origin_only(self): - self.assertEqual(strip_url_credentials( - 'http://username:password@www.example.com:8008/index.html', origin_only=True), - 'http://www.example.com:8008/') + ('http://username:password@www.example.com:8008/foo/bar?query=value#somefrag', + 'http://www.example.com:8008/'), + + ('https://username:password@www.example.com:443/index.html', + 'https://www.example.com/'), + ]: + self.assertEqual(strip_url(i, origin_only=True), o) + self.assertEqual(strip_url(urlparse(i), origin_only=True), o) - def test_default_https_port_origin_only(self): - self.assertEqual(strip_url_credentials( - 'https://username:password@www.example.com:443/index.html', origin_only=True), - 'https://www.example.com/') if __name__ == "__main__": unittest.main() From 0a0b60a59f75ef1b4976b811011bd50b78f9cec8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 26 Oct 2016 12:41:00 +0200 Subject: [PATCH 18/41] Add tests for stripping userinfo with percent-encoded delimiters --- tests/test_utils_url.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 1f9845d82..9182d0fda 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -290,6 +290,26 @@ class StripUrl(unittest.TestCase): self.assertEqual(strip_url(i, strip_credentials=True), o) self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o) + def test_credentials_encoded_delims(self): + for i, o in [ + # user: "username@" + # password: none + ('http://username%40@www.example.com/index.html?somekey=somevalue#section', + 'http://www.example.com/index.html?somekey=somevalue'), + + # user: "username:pass" + # password: "" + ('https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section', + 'https://www.example.com/index.html?somekey=somevalue'), + + # user: "me" + # password: "user@domain.com" + ('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section', + 'ftp://www.example.com/index.html?somekey=somevalue'), + ]: + self.assertEqual(strip_url(i, strip_credentials=True), o) + self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o) + def test_default_ports_creds_off(self): for i, o in [ ('http://username:password@www.example.com:80/index.html?somekey=somevalue#section', From c808a97c74718f72f53a3c00eca77f60c26fb6ba Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 12 Jan 2017 18:22:18 +0100 Subject: [PATCH 19/41] Add new "strict-" policies --- scrapy/spidermiddlewares/referer.py | 49 +++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 24e5eac40..5d2a0e7b1 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -17,7 +17,9 @@ POLICY_NO_REFERRER = "no-referrer" POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade" POLICY_SAME_ORIGIN = "same-origin" POLICY_ORIGIN = "origin" +POLICY_STRICT_ORIGIN = "strict-origin" POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin" +POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN = "strict-origin-when-cross-origin" POLICY_UNSAFE_URL = "unsafe-url" POLICY_SCRAPY_DEFAULT = "scrapy-default" @@ -139,6 +141,26 @@ class OriginPolicy(ReferrerPolicy): return self.origin_referrer(response) +class StrictOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin + + The "strict-origin" policy sends the ASCII serialization + of the origin of the request client when making requests: + - from a TLS-protected environment settings object to a potentially trustworthy URL, and + - from non-TLS-protected environment settings objects to any origin. + + Requests from TLS-protected request clients to non- potentially trustworthy URLs, + on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + """ + name = POLICY_STRICT_ORIGIN + + def referrer(self, response, request): + if urlparse_cached(response).scheme == urlparse_cached(request).scheme: + return self.origin_referrer(response) + + class OriginWhenCrossOriginPolicy(ReferrerPolicy): """ https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin @@ -160,6 +182,33 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): return origin +class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin + + The "strict-origin-when-cross-origin" policy specifies that a full URL, + stripped for use as a referrer, is sent as referrer information + when making same-origin requests from a particular request client, + and only the ASCII serialization of the origin of the request client + when making cross-origin requests: + + - from a TLS-protected environment settings object to a potentially trustworthy URL, and + - from non-TLS-protected environment settings objects to any origin. + + Requests from TLS-protected clients to non- potentially trustworthy URLs, + on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + """ + name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN + + def referrer(self, response, request): + origin = self.origin(response) + if origin == self.origin(request): + return self.stripped_referrer(response) + else: + return origin + + class UnsafeUrlPolicy(ReferrerPolicy): """ https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url From 5cef67ae75dca229bd1b1f3ac61f52309d1ca5a4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 14:18:33 +0100 Subject: [PATCH 20/41] Update Referrer tests for "strict-" policies --- tests/test_spidermiddleware_referer.py | 75 +++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 9555817d9..df20dfbb9 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -8,6 +8,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware, \ POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \ + POLICY_STRICT_ORIGIN, POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, \ DefaultReferrerPolicy, \ NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ OriginWhenCrossOriginPolicy, OriginPolicy, \ @@ -39,7 +40,6 @@ class TestRefererMiddleware(TestCase): for origin, target, referrer in self.scenarii: response = self.get_response(origin) request = self.get_request(target) - out = list(self.mw.process_spider_output(response, [request], self.spider)) self.assertEquals(out[0].headers.get('Referer'), referrer) @@ -154,6 +154,25 @@ class MixinOrigin(object): ] +class MixinStrictOrigin(object): + scenarii = [ + # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), + ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), + + # downgrade: send nothing + ('https://example.com/page.html', 'http://scrapy.org', None), + + # upgrade: send origin + ('http://example.com/page.html', 'https://scrapy.org', b'http://example.com/'), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('https://user:password@example.com/page.html', 'http://scrapy.org', None), + ] + + class MixinOriginWhenCrossOrigin(object): scenarii = [ # Same origin (protocol, host, port): send referrer @@ -189,6 +208,44 @@ class MixinOriginWhenCrossOrigin(object): ] +class MixinStrictOriginWhenCrossOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: send origin as referrer + ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), + ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), + ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + # exact match required + ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), + + # Different port: send origin as referrer + ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), + ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), + + # downgrade + ('https://example4.com/page.html', 'http://example4.com/not-page.html', None), + ('https://example4.com/page.html', 'http://not.example4.com/', None), + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', None), + + # Different protocols: send origin as referrer + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + + # test for user/password stripping + ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + + # TLS to non-TLS downgrade: send nothing + ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', None), + ] + + class MixinUnsafeUrl(object): scenarii = [ # TLS to TLS: send referrer @@ -241,10 +298,18 @@ class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware): settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} +class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} + + class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} +class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} + + class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} @@ -297,10 +362,18 @@ class TestRefererMiddlewareOrigin(MixinOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_ORIGIN} +class TestRefererMiddlewareSrictOrigin(MixinStrictOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN} + + class TestRefererMiddlewareOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} +class TestRefererMiddlewareStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} + + class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_UNSAFE_URL} From 77aec5a79681128c0608e65bc77e317252172d98 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 14:19:19 +0100 Subject: [PATCH 21/41] Fix implementation --- scrapy/spidermiddlewares/referer.py | 40 ++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 5d2a0e7b1..a6316cd0c 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -31,13 +31,13 @@ class ReferrerPolicy(object): def referrer(self, response, request): raise NotImplementedError() - def stripped_referrer(self, req_or_resp): - return self.strip_url(req_or_resp) + def stripped_referrer(self, r): + return self.strip_url(r) - def origin_referrer(self, req_or_resp): - return self.strip_url(req_or_resp, origin_only=True) + def origin_referrer(self, r): + return self.strip_url(r, origin_only=True) - def strip_url(self, req_or_resp, origin_only=False): + def strip_url(self, r, origin_only=False): """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -51,9 +51,9 @@ class ReferrerPolicy(object): Set url's query to null. Return url. """ - if req_or_resp.url is None or not req_or_resp.url: + if r is None or not r.url: return None - parsed_url = urlparse_cached(req_or_resp) + parsed_url = urlparse_cached(r) if parsed_url.scheme not in self.NOREFERRER_SCHEMES: return strip_url(parsed_url, strip_credentials=True, @@ -61,9 +61,19 @@ class ReferrerPolicy(object): strip_default_port=True, origin_only=origin_only) - def origin(self, req_or_resp): + def origin(self, r): """Return serialized origin (scheme, host, path) for a request or response URL.""" - return self.strip_url(req_or_resp, origin_only=True) + return self.strip_url(r, origin_only=True) + + def potentially_trustworthy(self, r): + # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy + parsed_url = urlparse_cached(r) + if parsed_url.scheme in ('data',): + return False + return self.tls_protected(r) + + def tls_protected(self, r): + return urlparse_cached(r).scheme in ('https', 'ftps') class NoReferrerPolicy(ReferrerPolicy): @@ -157,7 +167,9 @@ class StrictOriginPolicy(ReferrerPolicy): name = POLICY_STRICT_ORIGIN def referrer(self, response, request): - if urlparse_cached(response).scheme == urlparse_cached(request).scheme: + if ((urlparse_cached(response).scheme == 'https' and + self.potentially_trustworthy(request)) + or urlparse_cached(response).scheme == 'http'): return self.origin_referrer(response) @@ -205,8 +217,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): origin = self.origin(response) if origin == self.origin(request): return self.stripped_referrer(response) - else: - return origin + elif ((urlparse_cached(response).scheme in ('https', 'ftps') and + self.potentially_trustworthy(request)) + or urlparse_cached(response).scheme == 'http'): + return self.origin_referrer(response) class UnsafeUrlPolicy(ReferrerPolicy): @@ -244,7 +258,9 @@ _policy_classes = {p.name: p for p in ( NoReferrerWhenDowngradePolicy, SameOriginPolicy, OriginPolicy, + StrictOriginPolicy, OriginWhenCrossOriginPolicy, + StrictOriginWhenCrossOriginPolicy, UnsafeUrlPolicy, DefaultReferrerPolicy, )} From deb8567116db8488a8f5b890aad975df095ddcfe Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 16:22:49 +0100 Subject: [PATCH 22/41] Update NoReferrerWhenDowngradePolicy --- scrapy/spidermiddlewares/referer.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a6316cd0c..d64c79108 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -94,12 +94,11 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): """ https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade - The "no-referrer-when-downgrade" policy sends a full URL - along with requests from a TLS-protected environment settings object - to a a priori authenticated URL, - and requests from request clients which are not TLS-protected to any origin. + The "no-referrer-when-downgrade" policy sends a full URL along with requests + from a TLS-protected environment settings object to a potentially trustworthy URL, + and requests from clients which are not TLS-protected to any origin. - Requests from TLS-protected request clients to non-a priori authenticated URLs, + Requests from TLS-protected clients to non-potentially trustworthy URLs, on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. @@ -108,15 +107,8 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): name = POLICY_NO_REFERRER_WHEN_DOWNGRADE def referrer(self, response, request): - # https://www.w3.org/TR/referrer-policy/#determine-requests-referrer: - # - # If environment is TLS-protected - # and the origin of request's current URL is not an a priori authenticated URL, - # then return no referrer. - if urlparse_cached(response).scheme in ('https', 'ftps') and \ - urlparse_cached(request).scheme in ('http',): - return None - return self.stripped_referrer(response) + if not self.tls_protected(response) or self.tls_protected(request): + return self.stripped_referrer(response) class SameOriginPolicy(ReferrerPolicy): From ebcacd3f549de034d05ed946c446e0f71b11bc6f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 17:18:00 +0100 Subject: [PATCH 23/41] Update StrictOriginPolicy --- scrapy/spidermiddlewares/referer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index d64c79108..8ee04120a 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -159,9 +159,8 @@ class StrictOriginPolicy(ReferrerPolicy): name = POLICY_STRICT_ORIGIN def referrer(self, response, request): - if ((urlparse_cached(response).scheme == 'https' and - self.potentially_trustworthy(request)) - or urlparse_cached(response).scheme == 'http'): + if ((self.tls_protected(response) and self.potentially_trustworthy(request)) + or not self.tls_protected(response)): return self.origin_referrer(response) From b6c761d2b4c9cafe94010075ebf39807921dec9c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 17:57:17 +0100 Subject: [PATCH 24/41] Fix tests --- scrapy/spidermiddlewares/referer.py | 7 ++++--- tests/test_spidermiddleware_referer.py | 8 +++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 8ee04120a..4f50db689 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -159,7 +159,8 @@ class StrictOriginPolicy(ReferrerPolicy): name = POLICY_STRICT_ORIGIN def referrer(self, response, request): - if ((self.tls_protected(response) and self.potentially_trustworthy(request)) + if ((self.tls_protected(response) and + self.potentially_trustworthy(request)) or not self.tls_protected(response)): return self.origin_referrer(response) @@ -208,9 +209,9 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): origin = self.origin(response) if origin == self.origin(request): return self.stripped_referrer(response) - elif ((urlparse_cached(response).scheme in ('https', 'ftps') and + elif ((self.tls_protected(response) and self.potentially_trustworthy(request)) - or urlparse_cached(response).scheme == 'http'): + or not self.tls_protected(response)): return self.origin_referrer(response) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index df20dfbb9..4779b0ed1 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -232,7 +232,13 @@ class MixinStrictOriginWhenCrossOrigin(object): # downgrade ('https://example4.com/page.html', 'http://example4.com/not-page.html', None), ('https://example4.com/page.html', 'http://not.example4.com/', None), - ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', None), + + # non-TLS to non-TLS + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), + + # upgrade + ('http://example4.com/page.html', 'https://example4.com/not-page.html', b'http://example4.com/'), + ('http://example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/'), # Different protocols: send origin as referrer ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), From c86f568b9cd1fc69c67762ba39714e38c9ba70fb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 22:31:29 +0100 Subject: [PATCH 25/41] Update docs with "strict-..." policies --- docs/topics/spider-middleware.rst | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index a9d3d4568..e8325d7ef 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -344,9 +344,18 @@ This setting accepts: - a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass, either a custom one or one of the built-in ones (see ``scrapy.spidermiddlewares.referer``), -- or one of the standard W3C-defined string values, i.e. ``"no-referrer"``, - ``"no-referrer-when-downgrade"``, ``"same-origin"``, ``"origin"``, - ``"origin-when-cross-origin"`` or ``"unsafe-url"``. +- or one of the standard W3C-defined string values: + + - `"no-referrer" `_, + - `"no-referrer-when-downgrade" `_, + - `"same-origin" `_, + - `"origin" `_, + - `"strict-origin" `_, + - `"origin-when-cross-origin" `_, + - `"strict-origin-when-cross-origin" `_, + - or `"unsafe-url" `_ + (not recommended). + (It can also be the non-standard value ``"scrapy-default"`` to use Scrapy's default referrer policy.) From e249abc32bcb171e4676b8fd9f11f04201f80a75 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 18 Jan 2017 15:48:28 +0100 Subject: [PATCH 26/41] Update docs --- docs/topics/spider-middleware.rst | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index e8325d7ef..0ddf027ea 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -333,7 +333,7 @@ Whether to enable referer middleware. REFERER_POLICY ^^^^^^^^^^^^^^ -.. versionadded:: 1.3 +.. versionadded:: 1.4 Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` @@ -346,14 +346,14 @@ This setting accepts: (see ``scrapy.spidermiddlewares.referer``), - or one of the standard W3C-defined string values: - - `"no-referrer" `_, - - `"no-referrer-when-downgrade" `_, - - `"same-origin" `_, - - `"origin" `_, - - `"strict-origin" `_, - - `"origin-when-cross-origin" `_, - - `"strict-origin-when-cross-origin" `_, - - or `"unsafe-url" `_ + - `"no-referrer"`_, + - `"no-referrer-when-downgrade"`_, + - `"same-origin"`_, + - `"origin"`_, + - `"strict-origin"`_, + - `"origin-when-cross-origin"`_, + - `"strict-origin-when-cross-origin"`_, + - or `"unsafe-url"`_ (not recommended). (It can also be the non-standard value ``"scrapy-default"`` to use @@ -364,14 +364,22 @@ with the addition that "Referrer" is not sent if the parent request was using ``file://`` or ``s3://`` scheme. .. warning:: - By default, Scrapy's default referrer policy, just like `"no-referrer-when-downgrade"`_, + Scrapy's default referrer policy, just like `"no-referrer-when-downgrade"`_, will send a non-empty "Referer" header from any ``https://`` to any ``https://`` URL, even if the domain is different. ``same-origin`` may be a better choice if you want to remove referrer information for cross-domain requests. .. _Referrer Policy: https://www.w3.org/TR/referrer-policy +.. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer .. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade +.. _"same-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin +.. _"origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin +.. _"strict-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin +.. _"origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin +.. _"strict-origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin +.. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url + UrlLengthMiddleware ------------------- From 03ff19d1882edb709cf627795f3ce22934235254 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 18 Jan 2017 16:29:20 +0100 Subject: [PATCH 27/41] Update docs for new "referrer_policy" Request.meta key --- docs/topics/request-response.rst | 1 + docs/topics/spider-middleware.rst | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 3d110b02d..67f8ec285 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -307,6 +307,7 @@ Those are: * :reqmeta:`proxy` * ``ftp_user`` (See :setting:`FTP_USER` for more info) * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) +* :reqmeta:`referrer_policy` .. reqmeta:: bindaddress diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 0ddf027ea..a4ac45b41 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -337,6 +337,8 @@ REFERER_POLICY Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` +.. reqmeta:: referrer_policy + `Referrer Policy`_ to apply when populating Request "Referer" header. This setting accepts: @@ -370,6 +372,11 @@ using ``file://`` or ``s3://`` scheme. ``same-origin`` may be a better choice if you want to remove referrer information for cross-domain requests. +.. note:: + You can also override the Referrer Policy per request, + using the special ``"referrer_policy"`` :ref:`Request.meta ` key, + with the same acceptable values as for the ``REFERER_POLICY`` setting. + .. _Referrer Policy: https://www.w3.org/TR/referrer-policy .. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer .. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade From eb07285a63d6fdeefe113051943f4bf7e36f7b33 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 18 Jan 2017 17:20:35 +0100 Subject: [PATCH 28/41] Reword warning on no-referrer-when-downgrade policy --- docs/topics/spider-middleware.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index a4ac45b41..ada4c46c0 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -349,7 +349,8 @@ This setting accepts: - or one of the standard W3C-defined string values: - `"no-referrer"`_, - - `"no-referrer-when-downgrade"`_, + - `"no-referrer-when-downgrade"`_ + (the W3C-recommended default, used by major web browsers), - `"same-origin"`_, - `"origin"`_, - `"strict-origin"`_, @@ -358,18 +359,19 @@ This setting accepts: - or `"unsafe-url"`_ (not recommended). - (It can also be the non-standard value ``"scrapy-default"`` to use - Scrapy's default referrer policy.) +It can also be the non-standard value ``"scrapy-default"`` to use +Scrapy's default referrer policy. Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_, with the addition that "Referrer" is not sent if the parent request was using ``file://`` or ``s3://`` scheme. .. warning:: - Scrapy's default referrer policy, just like `"no-referrer-when-downgrade"`_, - will send a non-empty "Referer" header from any ``https://`` to any ``https://`` URL, + Scrapy's default referrer policy—just like `"no-referrer-when-downgrade"`_, + the W3C-recommended value for browsers—will send a non-empty + "Referer" header from any ``http(s)://`` to any ``https://`` URL, even if the domain is different. - ``same-origin`` may be a better choice if you want to remove referrer + `"same-origin"`_ may be a better choice if you want to remove referrer information for cross-domain requests. .. note:: From 605935f015b1862b83f051380cd08b2931cce784 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 26 Jan 2017 12:09:34 +0100 Subject: [PATCH 29/41] Edit text --- docs/topics/spider-middleware.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index ada4c46c0..a792364b3 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -363,12 +363,12 @@ It can also be the non-standard value ``"scrapy-default"`` to use Scrapy's default referrer policy. Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_, -with the addition that "Referrer" is not sent if the parent request was +with the addition that "Referer" is not sent if the parent request was using ``file://`` or ``s3://`` scheme. .. warning:: - Scrapy's default referrer policy—just like `"no-referrer-when-downgrade"`_, - the W3C-recommended value for browsers—will send a non-empty + Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_, + the W3C-recommended value for browsers — will send a non-empty "Referer" header from any ``http(s)://`` to any ``https://`` URL, even if the domain is different. `"same-origin"`_ may be a better choice if you want to remove referrer From 3dc09eeceb16ca97a0c5851bfdc41f921772910b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 30 Jan 2017 15:25:49 +0100 Subject: [PATCH 30/41] Use table for referrer policy options --- docs/topics/spider-middleware.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index a792364b3..12aa53012 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -346,21 +346,21 @@ This setting accepts: - a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass, either a custom one or one of the built-in ones (see ``scrapy.spidermiddlewares.referer``), -- or one of the standard W3C-defined string values: +- or one of the standard W3C-defined string values - - `"no-referrer"`_, - - `"no-referrer-when-downgrade"`_ - (the W3C-recommended default, used by major web browsers), - - `"same-origin"`_, - - `"origin"`_, - - `"strict-origin"`_, - - `"origin-when-cross-origin"`_, - - `"strict-origin-when-cross-origin"`_, - - or `"unsafe-url"`_ - (not recommended). - -It can also be the non-standard value ``"scrapy-default"`` to use -Scrapy's default referrer policy. +======================================= ======================================================================== ======================================================= +String value Class name +======================================= ======================================================================== ======================================================= +`"no-referrer"`_ ``'scrapy.spidermiddlewares.referer.NoReferrerPolicy'`` +`"no-referrer-when-downgrade"`_ ``'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'`` the W3C-recommended default, used by major web browsers +`"same-origin"`_ ``'scrapy.spidermiddlewares.referer.SameOriginPolicy'`` +`"origin"`_ ``'scrapy.spidermiddlewares.referer.OriginPolicy'`` +`"strict-origin"`_ ``'scrapy.spidermiddlewares.referer.StrictOriginPolicy'`` +`"origin-when-cross-origin"`_ ``'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'`` +`"strict-origin-when-cross-origin"`_ ``'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'`` +`"unsafe-url"`_ ``'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'`` NOT recommended +``"scrapy-default"`` ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` Scrapy's default policy (see below) +======================================= ======================================================================== ======================================================= Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_, with the addition that "Referer" is not sent if the parent request was From 537683f945906ff9865cf4e2704a432612f10eb4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 21 Feb 2017 16:47:57 +0100 Subject: [PATCH 31/41] Add autoclass directives to document built-in policies --- docs/topics/spider-middleware.rst | 70 +++++++++++++++++++---------- scrapy/spidermiddlewares/referer.py | 6 ++- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 12aa53012..349dbb3a1 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -341,43 +341,65 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` `Referrer Policy`_ to apply when populating Request "Referer" header. -This setting accepts: +.. note:: + You can also set the Referrer Policy per request, + using the special ``"referrer_policy"`` :ref:`Request.meta ` key, + with the same acceptable values as for the ``REFERER_POLICY`` setting. -- a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass, - either a custom one or one of the built-in ones - (see ``scrapy.spidermiddlewares.referer``), -- or one of the standard W3C-defined string values +Acceptable values for REFERER_POLICY +************************************ -======================================= ======================================================================== ======================================================= -String value Class name -======================================= ======================================================================== ======================================================= -`"no-referrer"`_ ``'scrapy.spidermiddlewares.referer.NoReferrerPolicy'`` -`"no-referrer-when-downgrade"`_ ``'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'`` the W3C-recommended default, used by major web browsers -`"same-origin"`_ ``'scrapy.spidermiddlewares.referer.SameOriginPolicy'`` -`"origin"`_ ``'scrapy.spidermiddlewares.referer.OriginPolicy'`` -`"strict-origin"`_ ``'scrapy.spidermiddlewares.referer.StrictOriginPolicy'`` -`"origin-when-cross-origin"`_ ``'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'`` -`"strict-origin-when-cross-origin"`_ ``'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'`` -`"unsafe-url"`_ ``'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'`` NOT recommended -``"scrapy-default"`` ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` Scrapy's default policy (see below) -======================================= ======================================================================== ======================================================= +- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` + subclass — a custom policy or one of the built-in ones (see classes below), +- or one of the standard W3C-defined string values, +- or the special ``"scrapy-default"``. -Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_, -with the addition that "Referer" is not sent if the parent request was -using ``file://`` or ``s3://`` scheme. +======================================= ======================================================================== +String value Class name (as a string) +======================================= ======================================================================== +``"scrapy-default"`` (default) :class:`scrapy.spidermiddlewares.referer.DefaultReferrerPolicy` +`"no-referrer"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerPolicy` +`"no-referrer-when-downgrade"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy` +`"same-origin"`_ :class:`scrapy.spidermiddlewares.referer.SameOriginPolicy` +`"origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginPolicy` +`"strict-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginPolicy` +`"origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy` +`"strict-origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy` +`"unsafe-url"`_ :class:`scrapy.spidermiddlewares.referer.UnsafeUrlPolicy` +======================================= ======================================================================== +.. autoclass:: DefaultReferrerPolicy .. warning:: Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_, the W3C-recommended value for browsers — will send a non-empty "Referer" header from any ``http(s)://`` to any ``https://`` URL, even if the domain is different. + `"same-origin"`_ may be a better choice if you want to remove referrer information for cross-domain requests. +.. autoclass:: NoReferrerPolicy + +.. autoclass:: NoReferrerWhenDowngradePolicy .. note:: - You can also override the Referrer Policy per request, - using the special ``"referrer_policy"`` :ref:`Request.meta ` key, - with the same acceptable values as for the ``REFERER_POLICY`` setting. + "no-referrer-when-downgrade" policy is the W3C-recommended default, + and is used by major web browsers. + + However, it is NOT Scrapy's default referrer policy (see :class:`DefaultReferrerPolicy`). + +.. autoclass:: SameOriginPolicy + +.. autoclass:: OriginPolicy + +.. autoclass:: StrictOriginPolicy + +.. autoclass:: OriginWhenCrossOriginPolicy + +.. autoclass:: StrictOriginWhenCrossOriginPolicy + +.. autoclass:: UnsafeUrlPolicy +.. warning:: + "unsafe-url" policy is NOT recommended. .. _Referrer Policy: https://www.w3.org/TR/referrer-policy .. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 4f50db689..c015e13c8 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -240,7 +240,11 @@ class LegacyPolicy(ReferrerPolicy): class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): - + """ + A variant of "no-referrer-when-downgrade", + with the addition that "Referer" is not sent if the parent request was + using ``file://`` or ``s3://`` scheme. + """ NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') name = POLICY_SCRAPY_DEFAULT From bc200d1155e0f93f37897ac206aa32945daf89bd Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 21 Feb 2017 17:08:39 +0100 Subject: [PATCH 32/41] Rename setting to REFERRER_POLICY (with 2 Rs) --- docs/topics/spider-middleware.rst | 12 ++++----- scrapy/settings/default_settings.py | 2 +- scrapy/spidermiddlewares/referer.py | 2 +- tests/test_spidermiddleware_referer.py | 36 +++++++++++++------------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 349dbb3a1..9a0ccd0c1 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -328,10 +328,10 @@ Default: ``True`` Whether to enable referer middleware. -.. setting:: REFERER_POLICY +.. setting:: REFERRER_POLICY -REFERER_POLICY -^^^^^^^^^^^^^^ +REFERRER_POLICY +^^^^^^^^^^^^^^^ .. versionadded:: 1.4 @@ -344,10 +344,10 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` .. note:: You can also set the Referrer Policy per request, using the special ``"referrer_policy"`` :ref:`Request.meta ` key, - with the same acceptable values as for the ``REFERER_POLICY`` setting. + with the same acceptable values as for the ``REFERRER_POLICY`` setting. -Acceptable values for REFERER_POLICY -************************************ +Acceptable values for REFERRER_POLICY +************************************* - either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass — a custom policy or one of the built-in ones (see classes below), diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 15a134dd3..35d9844a7 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -234,7 +234,7 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True -REFERER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' +REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index c015e13c8..24c163089 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -266,7 +266,7 @@ class RefererMiddleware(object): def __init__(self, settings=None): self.default_policy = DefaultReferrerPolicy if settings is not None: - policy = settings.get('REFERER_POLICY') + policy = settings.get('REFERRER_POLICY') if policy is not None: # expect a string for the path to the policy class try: diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 4779b0ed1..81868efab 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -289,35 +289,35 @@ class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware): # --- Tests using settings to set policy using class path class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} class CustomPythonOrgPolicy(ReferrerPolicy): @@ -336,7 +336,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware): - settings = {'REFERER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} + settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} scenarii = [ ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'), @@ -385,17 +385,17 @@ class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} req_meta = {'referrer_policy': POLICY_NO_REFERRER} class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} @@ -411,7 +411,7 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: - settings = Settings({'REFERER_POLICY': s}) + settings = Settings({'REFERRER_POLICY': s}) mw = RefererMiddleware(settings) self.assertEquals(mw.default_policy, p) @@ -425,24 +425,24 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: - settings = Settings({'REFERER_POLICY': s.upper()}) + settings = Settings({'REFERRER_POLICY': s.upper()}) mw = RefererMiddleware(settings) self.assertEquals(mw.default_policy, p) def test_invalid_name(self): - settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'}) + settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'}) with self.assertRaises(NotConfigured): mw = RefererMiddleware(settings) class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} From 04e4d08612364480c4d72d2b55e1cac161397136 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 27 Feb 2017 16:04:37 +0100 Subject: [PATCH 33/41] Pass URLs around instead of Request/Responses --- scrapy/spidermiddlewares/referer.py | 53 ++++++++++++++++------------- scrapy/utils/url.py | 7 +--- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 24c163089..f2910a0c8 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,6 +2,8 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ +from six.moves.urllib.parse import urlparse + from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals @@ -32,12 +34,14 @@ class ReferrerPolicy(object): raise NotImplementedError() def stripped_referrer(self, r): - return self.strip_url(r) + if urlparse(r).scheme not in self.NOREFERRER_SCHEMES: + return self.strip_url(r) def origin_referrer(self, r): - return self.strip_url(r, origin_only=True) + if urlparse(r).scheme not in self.NOREFERRER_SCHEMES: + return self.origin(r) - def strip_url(self, r, origin_only=False): + def strip_url(self, url, origin_only=False): """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -51,29 +55,27 @@ class ReferrerPolicy(object): Set url's query to null. Return url. """ - if r is None or not r.url: + if not url: return None - parsed_url = urlparse_cached(r) - if parsed_url.scheme not in self.NOREFERRER_SCHEMES: - return strip_url(parsed_url, - strip_credentials=True, - strip_fragment=True, - strip_default_port=True, - origin_only=origin_only) + return strip_url(url, + strip_credentials=True, + strip_fragment=True, + strip_default_port=True, + origin_only=origin_only) def origin(self, r): """Return serialized origin (scheme, host, path) for a request or response URL.""" return self.strip_url(r, origin_only=True) - def potentially_trustworthy(self, r): + def potentially_trustworthy(self, url): # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy - parsed_url = urlparse_cached(r) + parsed_url = urlparse(url) if parsed_url.scheme in ('data',): return False - return self.tls_protected(r) + return self.tls_protected(url) - def tls_protected(self, r): - return urlparse_cached(r).scheme in ('https', 'ftps') + def tls_protected(self, url): + return urlparse(url).scheme in ('https', 'ftps') class NoReferrerPolicy(ReferrerPolicy): @@ -287,12 +289,17 @@ class RefererMiddleware(object): crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled) return mw - def policy(self, response, request): + def policy(self, resp_or_url, request): + """ + Determine Referrer-Policy to use from a parent Response (or URL), + and a Request to be sent. + """ # policy set in request's meta dict takes precedence over default policy policy_name = request.meta.get('referrer_policy') if policy_name is None: - policy_name = to_native_str( - response.headers.get('Referrer-Policy', '').decode('latin1')) + if isinstance(resp_or_url, Response): + policy_name = to_native_str( + resp_or_url.headers.get('Referrer-Policy', '').decode('latin1')) cls = _policy_classes.get(policy_name.lower(), self.default_policy) return cls() @@ -300,7 +307,7 @@ class RefererMiddleware(object): def process_spider_output(self, response, result, spider): def _set_referer(r): if isinstance(r, Request): - referrer = self.policy(response, r).referrer(response, r) + referrer = self.policy(response, r).referrer(response.url, r.url) if referrer is not None: r.headers.setdefault('Referer', referrer) return r @@ -313,9 +320,9 @@ class RefererMiddleware(object): request_referrer = request.headers.get('Referer') # we don't patch the referrer value if there is none if request_referrer is not None: - faked_response = Response(redirected_urls[0]) - policy_referrer = self.policy(faked_response, - request).referrer(faked_response, request) + initial_url = redirected_urls[0] + policy_referrer = self.policy(initial_url, + request).referrer(orig_url, request.url) if policy_referrer != request_referrer: if policy_referrer is None: request.headers.pop('Referer') diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 9864f353d..8eed31060 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -117,12 +117,7 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only= - `strip_fragment` drops any #fragment component """ - if url is None: - return None - if not isinstance(url, ParseResult): - parsed_url = urlparse(url) - else: - parsed_url = url + parsed_url = urlparse(url) netloc = parsed_url.netloc if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password): netloc = netloc.split('@')[-1] From d2aa51c0fb81b4e264fc351b9184d9ce06855021 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 27 Feb 2017 16:05:22 +0100 Subject: [PATCH 34/41] Update tests --- tests/test_spidermiddleware_referer.py | 43 ++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 81868efab..39bbaab5d 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,3 +1,4 @@ +from six.moves.urllib.parse import urlparse from unittest import TestCase from scrapy.exceptions import NotConfigured @@ -326,9 +327,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): depending on the scheme of the target URL. """ def referrer(self, response, request): - from scrapy.utils.httpobj import urlparse_cached - - scheme = urlparse_cached(request).scheme + scheme = urlparse(request).scheme if scheme == 'https': return b'https://python.org/' elif scheme == 'http': @@ -446,3 +445,41 @@ class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefere class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} + + +class TestReferrerPolicyOnRedirect(TestRefererMiddleware): + + req_meta = {} + resp_headers = {} + #settings = {} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + scenarii = [ + ( # origin + 'http://scrapytest.org/1', + + # target + redirection + ['http://scrapytest.org/2', + 'http://scrapytest.org/3',], + + b'http://scrapytest.org/1', # expected initial referer + b'http://scrapytest.org/1', # expected referer for the redirection request + + ), + ] + + def setUp(self): + self.spider = Spider('foo') + settings = Settings(self.settings) + self.mw = RefererMiddleware(settings) + + def test_(self): + + for origin, target_chain, init_referrer, final_referrer in self.scenarii: + response = self.get_response(origin) + request = self.get_request(target_chain.pop()) + + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), init_referrer) + + request.meta['redirected_urls'] = target_chain From 8226e77010d79f64c7ec5ebdb9e7cee7803600ed Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 1 Mar 2017 12:41:33 +0100 Subject: [PATCH 35/41] Add test for Referer header on HTTP redirections --- scrapy/spidermiddlewares/referer.py | 16 +- tests/test_spidermiddleware_referer.py | 398 ++++++++++++++++++++++--- 2 files changed, 364 insertions(+), 50 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index f2910a0c8..bb184cc12 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -4,6 +4,8 @@ originated it. """ from six.moves.urllib.parse import urlparse +from w3lib.url import safe_url_string + from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals @@ -300,8 +302,7 @@ class RefererMiddleware(object): if isinstance(resp_or_url, Response): policy_name = to_native_str( resp_or_url.headers.get('Referrer-Policy', '').decode('latin1')) - - cls = _policy_classes.get(policy_name.lower(), self.default_policy) + cls = _policy_classes.get(policy_name.lower()) if policy_name else self.default_policy return cls() def process_spider_output(self, response, result, spider): @@ -320,9 +321,14 @@ class RefererMiddleware(object): request_referrer = request.headers.get('Referer') # we don't patch the referrer value if there is none if request_referrer is not None: - initial_url = redirected_urls[0] - policy_referrer = self.policy(initial_url, - request).referrer(orig_url, request.url) + # the request's referrer header value acts as a surrogate + # for the parent response URL + # + # Note: if the 3xx response contained a Referrer-Policy header, + # the information is not available using this hook + parent_url = safe_url_string(request_referrer) + policy_referrer = self.policy(parent_url, request).referrer( + parent_url, request.url) if policy_referrer != request_referrer: if policy_referrer is None: request.headers.pop('Referer') diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 39bbaab5d..28c694169 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -5,6 +5,7 @@ from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.spidermiddlewares.referer import RefererMiddleware, \ POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ @@ -13,6 +14,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware, \ DefaultReferrerPolicy, \ NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ OriginWhenCrossOriginPolicy, OriginPolicy, \ + StrictOriginWhenCrossOriginPolicy, StrictOriginPolicy, \ SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy @@ -289,35 +291,35 @@ class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware): # --- Tests using settings to set policy using class path -class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): +class TestSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} -class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): +class TestSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} -class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): +class TestSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} -class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware): +class TestSettingsOrigin(MixinOrigin, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} -class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): +class TestSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} -class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): +class TestSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} -class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): +class TestSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} -class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): +class TestSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} @@ -334,7 +336,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): return b'http://python.org/' -class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware): +class TestSettingsCustomPolicy(TestRefererMiddleware): settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} scenarii = [ ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), @@ -347,58 +349,58 @@ class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware): ] # --- Tests using Request meta dict to set policy -class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware): +class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT} -class TestRefererMiddlewareNoReferrer(MixinNoReferrer, TestRefererMiddleware): +class TestRequestMetaNoReferrer(MixinNoReferrer, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_NO_REFERRER} -class TestRefererMiddlewareNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): +class TestRequestMetaNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE} -class TestRefererMiddlewareSameOrigin(MixinSameOrigin, TestRefererMiddleware): +class TestRequestMetaSameOrigin(MixinSameOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_SAME_ORIGIN} -class TestRefererMiddlewareOrigin(MixinOrigin, TestRefererMiddleware): +class TestRequestMetaOrigin(MixinOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_ORIGIN} -class TestRefererMiddlewareSrictOrigin(MixinStrictOrigin, TestRefererMiddleware): +class TestRequestMetaSrictOrigin(MixinStrictOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN} -class TestRefererMiddlewareOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): +class TestRequestMetaOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} -class TestRefererMiddlewareStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): +class TestRequestMetaStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} -class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): +class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_UNSAFE_URL} -class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): +class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} -class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): +class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} req_meta = {'referrer_policy': POLICY_NO_REFERRER} -class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): +class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} -class TestRefererMiddlewareSettingsPolicyByName(TestCase): +class TestSettingsPolicyByName(TestCase): def test_valid_name(self): for s, p in [ @@ -407,7 +409,9 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), (POLICY_SAME_ORIGIN, SameOriginPolicy), (POLICY_ORIGIN, OriginPolicy), + (POLICY_STRICT_ORIGIN, StrictOriginPolicy), (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: settings = Settings({'REFERRER_POLICY': s}) @@ -421,7 +425,9 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), (POLICY_SAME_ORIGIN, SameOriginPolicy), (POLICY_ORIGIN, OriginPolicy), + (POLICY_STRICT_ORIGIN, StrictOriginPolicy), (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: settings = Settings({'REFERRER_POLICY': s.upper()}) @@ -434,52 +440,354 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): mw = RefererMiddleware(settings) -class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): +class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} -class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): +class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} -class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): +class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} -class TestReferrerPolicyOnRedirect(TestRefererMiddleware): +class TestReferrerOnRedirect(TestRefererMiddleware): - req_meta = {} - resp_headers = {} - #settings = {} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} scenarii = [ - ( # origin - 'http://scrapytest.org/1', - - # target + redirection - ['http://scrapytest.org/2', - 'http://scrapytest.org/3',], - + ( 'http://scrapytest.org/1', # parent + 'http://scrapytest.org/2', # target + ( + # redirections: code, URL + (301, 'http://scrapytest.org/3'), + (301, 'http://scrapytest.org/4'), + ), b'http://scrapytest.org/1', # expected initial referer b'http://scrapytest.org/1', # expected referer for the redirection request - + ), + ( 'https://scrapytest.org/1', + 'https://scrapytest.org/2', + ( + # redirecting to non-secure URL + (301, 'http://scrapytest.org/3'), + ), + b'https://scrapytest.org/1', + b'https://scrapytest.org/1', + ), + ( 'https://scrapytest.org/1', + 'https://scrapytest.com/2', + ( + # redirecting to non-secure URL: different origin + (301, 'http://scrapytest.com/3'), + ), + b'https://scrapytest.org/1', + b'https://scrapytest.org/1', ), ] def setUp(self): self.spider = Spider('foo') settings = Settings(self.settings) - self.mw = RefererMiddleware(settings) + self.referrermw = RefererMiddleware(settings) + self.redirectmw = RedirectMiddleware(settings) - def test_(self): + def test(self): - for origin, target_chain, init_referrer, final_referrer in self.scenarii: - response = self.get_response(origin) - request = self.get_request(target_chain.pop()) + for parent, target, redirections, init_referrer, final_referrer in self.scenarii: + response = self.get_response(parent) + request = self.get_request(target) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) + out = list(self.referrermw.process_spider_output(response, [request], self.spider)) self.assertEquals(out[0].headers.get('Referer'), init_referrer) - request.meta['redirected_urls'] = target_chain + for status, url in redirections: + response = Response(request.url, headers={'Location': url}, status=status) + request = self.redirectmw.process_response(request, response, self.spider) + self.referrermw.request_scheduled(request, self.spider) + + assert isinstance(request, Request) + self.assertEquals(request.headers.get('Referer'), final_referrer) + + +class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): + """ + No Referrer policy never sets the "Referer" header. + HTTP redirections should not change that. + """ + settings = {'REFERRER_POLICY': 'no-referrer'} + scenarii = [ + ( 'http://scrapytest.org/1', # parent + 'http://scrapytest.org/2', # target + ( + # redirections: code, URL + (301, 'http://scrapytest.org/3'), + (301, 'http://scrapytest.org/4'), + ), + None, # expected initial "Referer" + None, # expected "Referer" for the redirection request + ), + ( 'https://scrapytest.org/1', + 'https://scrapytest.org/2', + ( + (301, 'http://scrapytest.org/3'), + ), + None, + None, + ), + ( 'https://scrapytest.org/1', + 'https://example.com/2', # different origin + ( + (301, 'http://scrapytest.com/3'), + ), + None, + None, + ), + ] + + +class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect): + """ + Same Origin policy sends the full URL as "Referer" if the target origin + is the same as the parent response (same protocol, same domain, same port). + + HTTP redirections to a different domain or a lower secure level + should have the "Referer" removed. + """ + settings = {'REFERRER_POLICY': 'same-origin'} + scenarii = [ + ( 'http://scrapytest.org/101', # origin + 'http://scrapytest.org/102', # target + ( + # redirections: code, URL + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/101', # expected initial "Referer" + b'http://scrapytest.org/101', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting from secure to non-secure URL == different origin + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/201', + None, + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # different domain == different origin + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/301', + None, + ), + ] + + +class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect): + """ + Strict Origin policy will always send the "origin" as referrer + (think of it as the parent URL without the path part), + unless the security level is lower and no "Referer" is sent. + + Redirections from secure to non-secure URLs should have the + "Referrer" header removed if necessary. + """ + settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN} + scenarii = [ + ( 'http://scrapytest.org/101', + 'http://scrapytest.org/102', + ( + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/', # send origin + b'http://scrapytest.org/', # redirects to same origin: send origin + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting to non-secure URL: no referrer + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/', + None, + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # redirecting to non-secure URL (different domain): no referrer + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/', + None, + ), + ( 'http://scrapy.org/401', + 'http://example.com/402', + ( + (301, 'http://scrapytest.org/403'), + ), + b'http://scrapy.org/', + b'http://scrapy.org/', + ), + ( 'https://scrapy.org/501', + 'https://example.com/502', + ( + # HTTPS all along, so origin referrer is kept as-is + (301, 'https://google.com/503'), + (301, 'https://facebook.com/504'), + ), + b'https://scrapy.org/', + b'https://scrapy.org/', + ), + ( 'https://scrapytest.org/601', + 'http://scrapytest.org/602', # TLS to non-TLS: no referrer + ( + (301, 'https://scrapytest.org/603'), # TLS URL again: (still) no referrer + ), + None, + None, + ), + ] + + +class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect): + """ + Origin When Cross-Origin policy sends the full URL as "Referer", + unless the target's origin is different (different domain, different protocol) + in which case only the origin is sent. + + Redirections to a different origin should strip the "Referer" + to the parent origin. + """ + settings = {'REFERRER_POLICY': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} + scenarii = [ + ( 'http://scrapytest.org/101', # origin + 'http://scrapytest.org/102', # target + redirection + ( + # redirections: code, URL + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/101', # expected initial referer + b'http://scrapytest.org/101', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting to non-secure URL: send origin + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/201', + b'https://scrapytest.org/', + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # redirecting to non-secure URL (different domain): send origin + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/301', + b'https://scrapytest.org/', + ), + ( 'http://scrapy.org/401', + 'http://example.com/402', + ( + (301, 'http://scrapytest.org/403'), + ), + b'http://scrapy.org/', + b'http://scrapy.org/', + ), + ( 'https://scrapy.org/501', + 'https://example.com/502', + ( + # all different domains: send origin + (301, 'https://google.com/503'), + (301, 'https://facebook.com/504'), + ), + b'https://scrapy.org/', + b'https://scrapy.org/', + ), + ( 'https://scrapytest.org/301', + 'http://scrapytest.org/302', # TLS to non-TLS: send origin + ( + (301, 'https://scrapytest.org/303'), # TLS URL again: send origin (also) + ), + b'https://scrapytest.org/', + b'https://scrapytest.org/', + ), + ] + + +class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect): + """ + Strict Origin When Cross-Origin policy sends the full URL as "Referer", + unless the target's origin is different (different domain, different protocol) + in which case only the origin is sent... + Unless there's also a downgrade in security and then the "Referer" header + is not sent. + + Redirections to a different origin should strip the "Referer" to the parent origin, + and from https:// to http:// will remove the "Referer" header. + """ + settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} + scenarii = [ + ( 'http://scrapytest.org/101', # origin + 'http://scrapytest.org/102', # target + redirection + ( + # redirections: code, URL + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/101', # expected initial referer + b'http://scrapytest.org/101', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting to non-secure URL: do not send the "Referer" header + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/201', + None, + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # redirecting to non-secure URL (different domain): send origin + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/301', + None, + ), + ( 'http://scrapy.org/401', + 'http://example.com/402', + ( + (301, 'http://scrapytest.org/403'), + ), + b'http://scrapy.org/', + b'http://scrapy.org/', + ), + ( 'https://scrapy.org/501', + 'https://example.com/502', + ( + # all different domains: send origin + (301, 'https://google.com/503'), + (301, 'https://facebook.com/504'), + ), + b'https://scrapy.org/', + b'https://scrapy.org/', + ), + ( 'https://scrapytest.org/601', + 'http://scrapytest.org/602', # TLS to non-TLS: do not send "Referer" + ( + (301, 'https://scrapytest.org/603'), # TLS URL again: (still) send nothing + ), + None, + None, + ), + ] From 6916dd6240940cab06e2d02b3fceeb32c70691de Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 1 Mar 2017 17:42:46 +0100 Subject: [PATCH 36/41] Warn or fail with exception on unknown policies --- scrapy/spidermiddlewares/referer.py | 55 +++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index bb184cc12..5ed40791b 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -3,6 +3,7 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ from six.moves.urllib.parse import urlparse +import warnings from w3lib.url import safe_url_string @@ -265,45 +266,69 @@ _policy_classes = {p.name: p for p in ( DefaultReferrerPolicy, )} + +def _load_policy_class(policy, warning_only=False): + """ + Expect a string for the path to the policy class, + otherwise try to interpret the string as a standard value + from https://www.w3.org/TR/referrer-policy/#referrer-policies + """ + try: + return load_object(policy) + except ValueError: + try: + return _policy_classes[policy.lower()] + except KeyError: + msg = "Could not load referrer policy %r" % policy + if not warning_only: + raise RuntimeError(msg) + else: + warnings.warn(msg, RuntimeWarning) + return None + + class RefererMiddleware(object): def __init__(self, settings=None): self.default_policy = DefaultReferrerPolicy if settings is not None: - policy = settings.get('REFERRER_POLICY') - if policy is not None: - # expect a string for the path to the policy class - try: - self.default_policy = load_object(policy) - except ValueError: - # otherwise try to interpret the string as standard - # https://www.w3.org/TR/referrer-policy/#referrer-policies - try: - self.default_policy = _policy_classes[policy.lower()] - except: - raise NotConfigured("Unknown referrer policy name %r" % policy) + self.default_policy = _load_policy_class( + settings.get('REFERRER_POLICY')) @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured mw = cls(crawler.settings) + + # Note: this hook is a bit of a hack to intercept redirections crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled) + return mw def policy(self, resp_or_url, request): """ Determine Referrer-Policy to use from a parent Response (or URL), and a Request to be sent. + + - if a valid policy is set in Request meta, it is used. + - if the policy is set in meta but is wrong (e.g. a typo error), + the policy from settings is used + - if the policy is not set in Request meta, + but there is a Referrer-policy header in the parent response, + it is used if valid + - otherwise, the policy from settings is used. """ - # policy set in request's meta dict takes precedence over default policy policy_name = request.meta.get('referrer_policy') if policy_name is None: if isinstance(resp_or_url, Response): policy_name = to_native_str( resp_or_url.headers.get('Referrer-Policy', '').decode('latin1')) - cls = _policy_classes.get(policy_name.lower()) if policy_name else self.default_policy - return cls() + if policy_name is None: + return self.default_policy() + + cls = _load_policy_class(policy_name, warning_only=True) + return cls() if cls else self.default_policy() def process_spider_output(self, response, result, spider): def _set_referer(r): From efa50039ec4dc8d5cd5103e64c47ad8c86b9ed4e Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 1 Mar 2017 17:43:47 +0100 Subject: [PATCH 37/41] Add tests for policy fallback on unknown policies from meta and headers --- tests/test_spidermiddleware_referer.py | 76 +++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 28c694169..b1c815876 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,5 +1,6 @@ from six.moves.urllib.parse import urlparse from unittest import TestCase +import warnings from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request @@ -400,6 +401,79 @@ class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_UNSAFE_URL} +class TestRequestMetaSettingFallback(TestCase): + + params = [ + ( + # When an unknown policy is referenced in Request.meta + # (here, a typo error), + # the policy defined in settings takes precedence + {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}, + {}, + {'referrer_policy': 'ssscrapy-default'}, + OriginWhenCrossOriginPolicy, + True + ), + ( + # same as above but with string value for settings policy + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {}, + {'referrer_policy': 'ssscrapy-default'}, + OriginWhenCrossOriginPolicy, + True + ), + ( + # request meta references a wrong policy but it is set, + # so the Referrer-Policy header in response is not used, + # and the settings' policy is applied + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {'Referrer-Policy': 'unsafe-url'}, + {'referrer_policy': 'ssscrapy-default'}, + OriginWhenCrossOriginPolicy, + True + ), + ( + # here, request meta does not set the policy + # so response headers take precedence + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {'Referrer-Policy': 'unsafe-url'}, + {}, + UnsafeUrlPolicy, + False + ), + ( + # here, request meta does not set the policy, + # but response headers also use an unknown policy, + # so the settings' policy is used + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {'Referrer-Policy': 'unknown'}, + {}, + OriginWhenCrossOriginPolicy, + True + ) + ] + + def test(self): + + origin = 'http://www.scrapy.org' + target = 'http://www.example.com' + + for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]: + spider = Spider('foo') + mw = RefererMiddleware(Settings(settings)) + + response = Response(origin, headers=response_headers) + request = Request(target, meta=request_meta) + + with warnings.catch_warnings(record=True) as w: + policy = mw.policy(response, request) + self.assertIsInstance(policy, policy_class) + + if check_warning: + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, RuntimeWarning, w[0].message) + + class TestSettingsPolicyByName(TestCase): def test_valid_name(self): @@ -436,7 +510,7 @@ class TestSettingsPolicyByName(TestCase): def test_invalid_name(self): settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'}) - with self.assertRaises(NotConfigured): + with self.assertRaises(RuntimeError): mw = RefererMiddleware(settings) From 2d55d838ca1e2936e5729e67a6ab34a4bb58ba3b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 1 Mar 2017 20:59:52 +0100 Subject: [PATCH 38/41] Fix strip_url() tests --- tests/test_utils_url.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 9182d0fda..c2b9fc176 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -274,7 +274,6 @@ class StripUrl(unittest.TestCase): 'http://www.example.com/'), ]: self.assertEqual(strip_url(input_url, origin_only=origin), output_url) - self.assertEqual(strip_url(urlparse(input_url), origin_only=origin), output_url) def test_credentials(self): for i, o in [ @@ -288,7 +287,6 @@ class StripUrl(unittest.TestCase): 'ftp://www.example.com/index.html?somekey=somevalue'), ]: self.assertEqual(strip_url(i, strip_credentials=True), o) - self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o) def test_credentials_encoded_delims(self): for i, o in [ @@ -308,7 +306,6 @@ class StripUrl(unittest.TestCase): 'ftp://www.example.com/index.html?somekey=somevalue'), ]: self.assertEqual(strip_url(i, strip_credentials=True), o) - self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o) def test_default_ports_creds_off(self): for i, o in [ @@ -337,7 +334,6 @@ class StripUrl(unittest.TestCase): 'ftp://www.example.com:221/file.txt'), ]: self.assertEqual(strip_url(i), o) - self.assertEqual(strip_url(urlparse(i)), o) def test_default_ports(self): for i, o in [ @@ -366,7 +362,6 @@ class StripUrl(unittest.TestCase): 'ftp://username:password@www.example.com:221/file.txt'), ]: self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o) - self.assertEqual(strip_url(urlparse(i), strip_default_port=True, strip_credentials=False), o) def test_default_ports_keep(self): for i, o in [ @@ -395,7 +390,6 @@ class StripUrl(unittest.TestCase): 'ftp://username:password@www.example.com:221/file.txt'), ]: self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o) - self.assertEqual(strip_url(urlparse(i), strip_default_port=False, strip_credentials=False), o) def test_origin_only(self): for i, o in [ @@ -412,7 +406,6 @@ class StripUrl(unittest.TestCase): 'https://www.example.com/'), ]: self.assertEqual(strip_url(i, origin_only=True), o) - self.assertEqual(strip_url(urlparse(i), origin_only=True), o) if __name__ == "__main__": From c2c503192f24a198b4515ab006e3174e28062731 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Mar 2017 22:53:27 +0100 Subject: [PATCH 39/41] Rename arguments --- scrapy/spidermiddlewares/referer.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 5ed40791b..2bc8b1782 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -36,13 +36,13 @@ class ReferrerPolicy(object): def referrer(self, response, request): raise NotImplementedError() - def stripped_referrer(self, r): - if urlparse(r).scheme not in self.NOREFERRER_SCHEMES: - return self.strip_url(r) + def stripped_referrer(self, url): + if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: + return self.strip_url(url) - def origin_referrer(self, r): - if urlparse(r).scheme not in self.NOREFERRER_SCHEMES: - return self.origin(r) + def origin_referrer(self, url): + if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: + return self.origin(url) def strip_url(self, url, origin_only=False): """ @@ -66,9 +66,9 @@ class ReferrerPolicy(object): strip_default_port=True, origin_only=origin_only) - def origin(self, r): + def origin(self, url): """Return serialized origin (scheme, host, path) for a request or response URL.""" - return self.strip_url(r, origin_only=True) + return self.strip_url(url, origin_only=True) def potentially_trustworthy(self, url): # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy @@ -241,7 +241,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): class LegacyPolicy(ReferrerPolicy): def referrer(self, response, request): - return response.url + return response class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): From db176f872b91c4f51f10ea1e732875dc40a9e7bb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Mar 2017 22:56:23 +0100 Subject: [PATCH 40/41] Remove Legacy Policy which is equivalent to UnsafeUrl Policy --- scrapy/spidermiddlewares/referer.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 2bc8b1782..7b41bfaf7 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -239,11 +239,6 @@ class UnsafeUrlPolicy(ReferrerPolicy): return self.stripped_referrer(response) -class LegacyPolicy(ReferrerPolicy): - def referrer(self, response, request): - return response - - class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): """ A variant of "no-referrer-when-downgrade", From fad499ab602e7f12bf56a514f5aae1a2f80670f7 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Mar 2017 23:06:04 +0100 Subject: [PATCH 41/41] Rename arguments (bis) --- scrapy/spidermiddlewares/referer.py | 58 ++++++++++++++--------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 7b41bfaf7..b444e34bb 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -33,7 +33,7 @@ class ReferrerPolicy(object): NOREFERRER_SCHEMES = LOCAL_SCHEMES - def referrer(self, response, request): + def referrer(self, response_url, request_url): raise NotImplementedError() def stripped_referrer(self, url): @@ -91,7 +91,7 @@ class NoReferrerPolicy(ReferrerPolicy): """ name = POLICY_NO_REFERRER - def referrer(self, response, request): + def referrer(self, response_url, request_url): return None @@ -111,9 +111,9 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): """ name = POLICY_NO_REFERRER_WHEN_DOWNGRADE - def referrer(self, response, request): - if not self.tls_protected(response) or self.tls_protected(request): - return self.stripped_referrer(response) + def referrer(self, response_url, request_url): + if not self.tls_protected(response_url) or self.tls_protected(request_url): + return self.stripped_referrer(response_url) class SameOriginPolicy(ReferrerPolicy): @@ -128,9 +128,9 @@ class SameOriginPolicy(ReferrerPolicy): """ name = POLICY_SAME_ORIGIN - def referrer(self, response, request): - if self.origin(response) == self.origin(request): - return self.stripped_referrer(response) + def referrer(self, response_url, request_url): + if self.origin(response_url) == self.origin(request_url): + return self.stripped_referrer(response_url) class OriginPolicy(ReferrerPolicy): @@ -144,8 +144,8 @@ class OriginPolicy(ReferrerPolicy): """ name = POLICY_ORIGIN - def referrer(self, response, request): - return self.origin_referrer(response) + def referrer(self, response_url, request_url): + return self.origin_referrer(response_url) class StrictOriginPolicy(ReferrerPolicy): @@ -163,11 +163,11 @@ class StrictOriginPolicy(ReferrerPolicy): """ name = POLICY_STRICT_ORIGIN - def referrer(self, response, request): - if ((self.tls_protected(response) and - self.potentially_trustworthy(request)) - or not self.tls_protected(response)): - return self.origin_referrer(response) + def referrer(self, response_url, request_url): + if ((self.tls_protected(response_url) and + self.potentially_trustworthy(request_url)) + or not self.tls_protected(response_url)): + return self.origin_referrer(response_url) class OriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -183,10 +183,10 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): """ name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN - def referrer(self, response, request): - origin = self.origin(response) - if origin == self.origin(request): - return self.stripped_referrer(response) + def referrer(self, response_url, request_url): + origin = self.origin(response_url) + if origin == self.origin(request_url): + return self.stripped_referrer(response_url) else: return origin @@ -210,14 +210,14 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): """ name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN - def referrer(self, response, request): - origin = self.origin(response) - if origin == self.origin(request): - return self.stripped_referrer(response) - elif ((self.tls_protected(response) and - self.potentially_trustworthy(request)) - or not self.tls_protected(response)): - return self.origin_referrer(response) + def referrer(self, response_url, request_url): + origin = self.origin(response_url) + if origin == self.origin(request_url): + return self.stripped_referrer(response_url) + elif ((self.tls_protected(response_url) and + self.potentially_trustworthy(request_url)) + or not self.tls_protected(response_url)): + return self.origin_referrer(response_url) class UnsafeUrlPolicy(ReferrerPolicy): @@ -235,8 +235,8 @@ class UnsafeUrlPolicy(ReferrerPolicy): """ name = POLICY_UNSAFE_URL - def referrer(self, response, request): - return self.stripped_referrer(response) + def referrer(self, response_url, request_url): + return self.stripped_referrer(response_url) class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):