diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 8458fe90b..b724d7999 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,233 +1,316 @@ from unittest import TestCase +from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request +from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.spidermiddlewares.referer import RefererMiddleware, \ POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ - POLICY_UNSAFE_URL + POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \ + DefaultReferrerPolicy, \ + NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ + OriginWhenCrossOriginPolicy, OriginPolicy, \ + SameOriginPolicy, UnsafeUrlPolicy class TestRefererMiddleware(TestCase): + req_meta = {} + resp_headers = {} + settings = {} + scenarii = [ + ('http://scrapytest.org', 'http://scrapytest.org/', b'http://scrapytest.org'), + ] + def setUp(self): self.spider = Spider('foo') - self.mw = RefererMiddleware() + settings = Settings(self.settings) + self.mw = RefererMiddleware(settings) - def test_process_spider_output(self): - res = Response('http://scrapytest.org') - reqs = [Request('http://scrapytest.org/')] + def get_request(self, target): + return Request(target, meta=self.req_meta) - out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://scrapytest.org') + def get_response(self, origin): + return Response(origin, headers=self.resp_headers) - def test_policy_default(self): - """ - Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + def test(self): - with some additional filtering of s3:// - """ - for origin, target, referrer in [ - ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'), - ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'), - ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'), - ('https://example.com/', 'http://scrapy.org/', None), - - # no credentials leak - ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'), - - # no referrer leak for local schemes - ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None), - ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), - - # no referrer leak for s3 origins - ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None), - ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None), - ]: - response = Response(origin) - request = Request(target) + for origin, target, referrer in self.scenarii: + response = self.get_response(origin) + request = self.get_request(target) out = list(self.mw.process_spider_output(response, [request], self.spider)) self.assertEquals(out[0].headers.get('Referer'), referrer) - def test_policy_no_referrer(self): - for origin, target, referrer in [ - ('https://example.com/page.html', 'https://example.com/', None), - ('http://www.example.com/', 'https://scrapy.org/', None), - ('http://www.example.com/', 'http://scrapy.org/', None), - ('https://www.example.com/', 'http://scrapy.org/', None), - ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), +class MixinDefault(object): + """ + Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + + with some additional filtering of s3:// + """ + scenarii = [ + ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'), + ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'), + ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'), + ('https://example.com/', 'http://scrapy.org/', None), + + # no credentials leak + ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'), + + # no referrer leak for local schemes + ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + + # no referrer leak for s3 origins + ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None), + ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None), + ] + + +class MixinNoReferrer(object): + scenarii = [ + ('https://example.com/page.html', 'https://example.com/', None), + ('http://www.example.com/', 'https://scrapy.org/', None), + ('http://www.example.com/', 'http://scrapy.org/', None), + ('https://www.example.com/', 'http://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + ] + + +class MixinNoReferrerWhenDowngrade(object): + scenarii = [ + # TLS to TLS: send non-empty referrer + ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), + ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'), + ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'), + + # TLS to non-TLS: do not send referrer + ('https://example.com/page.html', 'http://not.example.com/', None), + ('https://example.com/page.html', 'http://scrapy.org/', None), + ('ftps://example.com/urls.zip', 'http://scrapy.org/', None), + + # non-TLS to TLS or non-TLS: send referrer + ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'), + ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'), + ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'), + ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'), + + # test for user/password stripping + ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ] + + +class MixinSameOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: do NOT send referrer + ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None), + + # Different port: do NOT send referrer + ('https://example.com:444/page.html', 'https://example.com/not-page.html', None), + ('http://example.com:81/page.html', 'http://example.com/not-page.html', None), + ('http://example.com/page.html', 'http://example.com:81/not-page.html', None), + + # Different protocols: do NOT send refferer + ('https://example.com/page.html', 'http://example.com/not-page.html', None), + ('https://example.com/page.html', 'http://not.example.com/', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), + ] + + +class MixinOrigin(object): + scenarii = [ + # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), + ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ] + + +class MixinOriginWhenCrossOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: send origin as referrer + ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), + ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), + ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + # exact match required + ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), + + # Different port: send origin as referrer + ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), + ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), + + # Different protocols: send origin as referrer + ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'), + ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + + # test for user/password stripping + ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + # TLS to non-TLS downgrade: send origin + ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), + ] + + +class MixinUnsafeUrl(object): + scenarii = [ + # TLS to TLS: send referrer + ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), + ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'), + ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'), + ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'), + + # TLS to non-TLS: send referrer (yes, it's unsafe) + ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'), + ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'), + ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'), + + # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe) + ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'), + ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'), + ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'), + ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), + + # test for user/password stripping + ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), + ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), + ] + + +class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware): + pass + + +# --- Tests using settings to set policy using class path +class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} + + +class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + + +class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + + +class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} + + +class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + + +class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + + +# --- Tests using Request meta dict to set policy +class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT} + + +class TestRefererMiddlewareNoReferrer(MixinNoReferrer, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_NO_REFERRER} + + +class TestRefererMiddlewareNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE} + + +class TestRefererMiddlewareSameOrigin(MixinSameOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_SAME_ORIGIN} + + +class TestRefererMiddlewareOrigin(MixinOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_ORIGIN} + + +class TestRefererMiddlewareOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} + + +class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + + +class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + +class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + req_meta = {'referrer_policy': POLICY_NO_REFERRER} + + +class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + +class TestRefererMiddlewareSettingsPolicyByName(TestCase): + + def test_valid_name(self): + for s, p in [ + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER}) + settings = Settings({'REFERER_POLICY': s}) + mw = RefererMiddleware(settings) + self.assertEquals(mw.default_policy, p) - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_no_referrer_when_downgrade(self): - - for origin, target, referrer in [ - # TLS to TLS: send non-empty referrer - ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), - ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), - ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), - ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'), - ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'), - - # TLS to non-TLS: do not send referrer - ('https://example.com/page.html', 'http://not.example.com/', None), - ('https://example.com/page.html', 'http://scrapy.org/', None), - ('ftps://example.com/urls.zip', 'http://scrapy.org/', None), - - # non-TLS to TLS or non-TLS: send referrer - ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), - ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'), - ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'), - ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'), - ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'), - ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'), - ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'), - - # test for user/password stripping - ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_same_origin(self): - - for origin, target, referrer in [ - # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), - - # Different host: do NOT send referrer - ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), - ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None), - ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None), - - # Different port: do NOT send referrer - ('https://example.com:444/page.html', 'https://example.com/not-page.html', None), - ('http://example.com:81/page.html', 'http://example.com/not-page.html', None), - ('http://example.com/page.html', 'http://example.com:81/not-page.html', None), - - # Different protocols: do NOT send refferer - ('https://example.com/page.html', 'http://example.com/not-page.html', None), - ('https://example.com/page.html', 'http://not.example.com/', None), - ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), - ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None), - ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), - - # test for user/password stripping - ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_SAME_ORIGIN}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_origin(self): - - for origin, target, referrer in [ - # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), - ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), - ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'), - ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), - - # test for user/password stripping - ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_ORIGIN}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_origin_when_cross_origin(self): - - for origin, target, referrer in [ - # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), - - # Different host: send origin as referrer - ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), - ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), - ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), - # exact match required - ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), - - # Different port: send origin as referrer - ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), - ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), - - # Different protocols: send origin as referrer - ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'), - ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'), - ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), - ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), - ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), - - # test for user/password stripping - ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), - # TLS to non-TLS downgrade: send origin - ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_unsafe_url(self): - - for origin, target, referrer in [ - # TLS to TLS: send referrer - ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), - ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'), - ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), - ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), - ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'), - ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'), - - # TLS to non-TLS: send referrer (yes, it's unsafe) - ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'), - ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'), - ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'), - - # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe) - ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'), - ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'), - ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'), - ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'), - ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'), - ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'), - ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'), - ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), - - # test for user/password stripping - ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), - ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_UNSAFE_URL}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) + def test_invalid_name(self): + settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'}) + with self.assertRaises(NotConfigured): + mw = RefererMiddleware(settings)