diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 21b340f22..1895aa95d 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -11,6 +11,15 @@ from scrapy.utils.python import to_native_str LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) +POLICY_NO_REFERRER = "no-referrer" +POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade" +POLICY_SAME_ORIGIN = "same-origin" +POLICY_ORIGIN = "origin" +POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin" +POLICY_UNSAFE_URL = "unsafe-url" +POLICY_SCRAPY_DEFAULT = "scrapy-default" + + class ReferrerPolicy(object): NOREFERRER_SCHEMES = LOCAL_SCHEMES @@ -38,17 +47,29 @@ class ReferrerPolicy(object): if parsed.scheme in self.NOREFERRER_SCHEMES: return None + + netloc = parsed.netloc + # strip username and password if present if parsed.username or parsed.password: - netloc = parsed.netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') - else: - netloc = parsed.netloc + netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') + + # strip standard protocol numbers + # Note: strictly speaking, standard port numbers should only be + # stripped when comparing origins + if parsed.port: + if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)): + netloc = netloc.replace(':{p.port}'.format(p=parsed), '') + return urlunsplit(( parsed.scheme, netloc, - '' if origin_only else parsed.path, + '/' if origin_only else parsed.path, '' if origin_only else parsed.query, '')) + def origin(self, url): + return self.strip_url(url, origin_only=True) + class NoReferrerPolicy(ReferrerPolicy): """ @@ -58,7 +79,7 @@ class NoReferrerPolicy(ReferrerPolicy): is to be sent along with requests made from a particular request client to any origin. The header will be omitted entirely. """ - name = "no-referrer" + name = POLICY_NO_REFERRER def referrer(self, response, request): return None @@ -79,7 +100,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): This is a user agent's default behavior, if no policy is otherwise specified. """ - name = "no-referrer-when-downgrade" + name = POLICY_NO_REFERRER_WHEN_DOWNGRADE def referrer(self, response, request): target_url = request.url @@ -108,12 +129,12 @@ class SameOriginPolicy(ReferrerPolicy): Cross-origin requests, on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = "same-origin" + name = POLICY_SAME_ORIGIN def referrer(self, response, request): target_url = request.url referrer_source = response.url - if urlsplit(referrer_source).netloc == urlsplit(target_url).netloc: + if self.origin(referrer_source) == self.origin(target_url): return self.strip_url(referrer_source) else: return None @@ -128,10 +149,10 @@ class OriginPolicy(ReferrerPolicy): when making both same-origin requests and cross-origin requests from a particular request client. """ - name = "origin" + name = POLICY_ORIGIN def referrer(self, response, request): - return self.strip_url(referrer_source, origin_only=True) + return self.strip_url(response.url, origin_only=True) class OriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -145,17 +166,17 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): is sent as referrer information when making cross-origin requests from a particular request client. """ - name = "origin-when-cross-origin" + name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response, request): target_url = request.url referrer_source = response.url + source_origin = self.origin(referrer_source) + if source_origin == self.origin(target_url): + return self.strip_url(referrer_source, origin_only=False) + else: + return source_origin - # same origin --> send full referrer - # different origin --> send only "origin" as referrer - if urlsplit(referrer_source).netloc != urlsplit(target_url).netloc: - origin_only = True - return self.strip_url(referrer_source, origin_only=origin_only) class UnsafeUrlPolicy(ReferrerPolicy): @@ -171,7 +192,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): to insecure origins. Carefully consider the impact of setting such a policy for potentially sensitive documents. """ - name = "unsafe-url" + name = POLICY_UNSAFE_URL def referrer(self, response, request): referrer_source = response.url @@ -186,15 +207,17 @@ class LegacyPolicy(ReferrerPolicy): class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') + name = POLICY_SCRAPY_DEFAULT -_policies = {p.name: p for p in ( +_policy_classes = {p.name: p for p in ( NoReferrerPolicy, NoReferrerWhenDowngradePolicy, SameOriginPolicy, OriginPolicy, OriginWhenCrossOriginPolicy, UnsafeUrlPolicy, + DefaultReferrerPolicy, )} class RefererMiddleware(object): @@ -211,10 +234,11 @@ class RefererMiddleware(object): def policy(self, response, request): policy_name = request.meta.get('referrer_policy') if policy_name is None: - policy_name = to_native_str(response.headers.get('Referrer-Policy', '').decode('latin1')) + policy_name = to_native_str( + response.headers.get('Referrer-Policy', '').decode('latin1')) - policy_class = _policies.get(policy_name.lower(), self.default_policy) - return policy_class() + cls = _policy_classes.get(policy_name.lower(), self.default_policy) + return cls() def process_spider_output(self, response, result, spider): def _set_referer(r): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index f109bb248..8458fe90b 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -2,7 +2,10 @@ from unittest import TestCase from scrapy.http import Response, Request from scrapy.spiders import Spider -from scrapy.spidermiddlewares.referer import RefererMiddleware +from scrapy.spidermiddlewares.referer import RefererMiddleware, \ + POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ + POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ + POLICY_UNSAFE_URL class TestRefererMiddleware(TestCase): @@ -25,71 +28,206 @@ class TestRefererMiddleware(TestCase): with some additional filtering of s3:// """ - # a) https:// --> https:// -- include Referer header - origin = Response('https://example.com/') - target = Request('https://scrapy.org/') + for origin, target, referrer in [ + ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'), + ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'), + ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'), + ('https://example.com/', 'http://scrapy.org/', None), - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'https://example.com/') + # no credentials leak + ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'), - # b.1) http:// --> http:// -- include Referer header - origin = Response('http://example.com/') - target = Request('http://scrapy.org/') + # no referrer leak for local schemes + ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://example.com/') + # no referrer leak for s3 origins + ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None), + ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None), + ]: + response = Response(origin) + request = Request(target) - # b.2) http:// --> https:// -- include Referer header - origin = Response('http://example.com/') - target = Request('https://scrapy.org/') + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://example.com/') + def test_policy_no_referrer(self): - # c) https:// --> http:// -- Referer header NOT sent - origin = Response('https://example.com/') - target = Request('http://scrapy.org/') + for origin, target, referrer in [ + ('https://example.com/page.html', 'https://example.com/', None), + ('http://www.example.com/', 'https://scrapy.org/', None), + ('http://www.example.com/', 'http://scrapy.org/', None), + ('https://www.example.com/', 'http://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER}) - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) - def test_policy_default_no_credentials_leak(self): - origin = Response('http://user:password@example.com/') - target = Request('https://scrapy.org/') + def test_policy_no_referrer_when_downgrade(self): - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://example.com/') + for origin, target, referrer in [ + # TLS to TLS: send non-empty referrer + ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), + ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'), + ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'), - def test_policy_default_file_no_referrer_leak(self): - # file:// --> https:// -- Referrer NOT sent - origin = Response('file:///home/path/to/somefile.html') - target = Request('https://scrapy.org/') + # TLS to non-TLS: do not send referrer + ('https://example.com/page.html', 'http://not.example.com/', None), + ('https://example.com/page.html', 'http://scrapy.org/', None), + ('ftps://example.com/urls.zip', 'http://scrapy.org/', None), - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + # non-TLS to TLS or non-TLS: send referrer + ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'), + ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'), + ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'), + ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'), - # file:// --> http:// -- Referrer NOT sent - origin = Response('file:///home/path/to/somefile.html') - target = Request('http://scrapy.org/') + # test for user/password stripping + ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE}) - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) - def test_policy_default_s3_no_referrer_leak(self): - # s3:// --> https:// -- Referrer NOT sent - origin = Response('s3://mybucket/path/to/data.csv') - target = Request('https://scrapy.org/') + def test_policy_same_origin(self): - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + for origin, target, referrer in [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), - # s3:// --> http:// -- Referrer NOT sent - origin = Response('s3://mybucket/path/to/data.csv') - target = Request('http://scrapy.org/') + # Different host: do NOT send referrer + ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None), - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + # Different port: do NOT send referrer + ('https://example.com:444/page.html', 'https://example.com/not-page.html', None), + ('http://example.com:81/page.html', 'http://example.com/not-page.html', None), + ('http://example.com/page.html', 'http://example.com:81/not-page.html', None), + + # Different protocols: do NOT send refferer + ('https://example.com/page.html', 'http://example.com/not-page.html', None), + ('https://example.com/page.html', 'http://not.example.com/', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_SAME_ORIGIN}) + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) + + def test_policy_origin(self): + + for origin, target, referrer in [ + # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), + ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_ORIGIN}) + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) + + def test_policy_origin_when_cross_origin(self): + + for origin, target, referrer in [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: send origin as referrer + ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), + ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), + ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + # exact match required + ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), + + # Different port: send origin as referrer + ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), + ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), + + # Different protocols: send origin as referrer + ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'), + ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + + # test for user/password stripping + ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + # TLS to non-TLS downgrade: send origin + ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}) + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) + + def test_policy_unsafe_url(self): + + for origin, target, referrer in [ + # TLS to TLS: send referrer + ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), + ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'), + ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'), + ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'), + + # TLS to non-TLS: send referrer (yes, it's unsafe) + ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'), + ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'), + ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'), + + # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe) + ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'), + ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'), + ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'), + ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), + + # test for user/password stripping + ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), + ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_UNSAFE_URL}) + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer)