diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 3d110b02d..67f8ec285 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -307,6 +307,7 @@ Those are: * :reqmeta:`proxy` * ``ftp_user`` (See :setting:`FTP_USER` for more info) * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) +* :reqmeta:`referrer_policy` .. reqmeta:: bindaddress diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 8360827e8..9a0ccd0c1 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -95,7 +95,7 @@ following methods: it has processed the response. :meth:`process_spider_output` must return an iterable of - :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` + :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` objects. :param response: the response which generated this output from the @@ -328,6 +328,90 @@ Default: ``True`` Whether to enable referer middleware. +.. setting:: REFERRER_POLICY + +REFERRER_POLICY +^^^^^^^^^^^^^^^ + +.. versionadded:: 1.4 + +Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` + +.. reqmeta:: referrer_policy + +`Referrer Policy`_ to apply when populating Request "Referer" header. + +.. note:: + You can also set the Referrer Policy per request, + using the special ``"referrer_policy"`` :ref:`Request.meta ` key, + with the same acceptable values as for the ``REFERRER_POLICY`` setting. + +Acceptable values for REFERRER_POLICY +************************************* + +- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` + subclass — a custom policy or one of the built-in ones (see classes below), +- or one of the standard W3C-defined string values, +- or the special ``"scrapy-default"``. + +======================================= ======================================================================== +String value Class name (as a string) +======================================= ======================================================================== +``"scrapy-default"`` (default) :class:`scrapy.spidermiddlewares.referer.DefaultReferrerPolicy` +`"no-referrer"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerPolicy` +`"no-referrer-when-downgrade"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy` +`"same-origin"`_ :class:`scrapy.spidermiddlewares.referer.SameOriginPolicy` +`"origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginPolicy` +`"strict-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginPolicy` +`"origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy` +`"strict-origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy` +`"unsafe-url"`_ :class:`scrapy.spidermiddlewares.referer.UnsafeUrlPolicy` +======================================= ======================================================================== + +.. autoclass:: DefaultReferrerPolicy +.. warning:: + Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_, + the W3C-recommended value for browsers — will send a non-empty + "Referer" header from any ``http(s)://`` to any ``https://`` URL, + even if the domain is different. + + `"same-origin"`_ may be a better choice if you want to remove referrer + information for cross-domain requests. + +.. autoclass:: NoReferrerPolicy + +.. autoclass:: NoReferrerWhenDowngradePolicy +.. note:: + "no-referrer-when-downgrade" policy is the W3C-recommended default, + and is used by major web browsers. + + However, it is NOT Scrapy's default referrer policy (see :class:`DefaultReferrerPolicy`). + +.. autoclass:: SameOriginPolicy + +.. autoclass:: OriginPolicy + +.. autoclass:: StrictOriginPolicy + +.. autoclass:: OriginWhenCrossOriginPolicy + +.. autoclass:: StrictOriginWhenCrossOriginPolicy + +.. autoclass:: UnsafeUrlPolicy +.. warning:: + "unsafe-url" policy is NOT recommended. + +.. _Referrer Policy: https://www.w3.org/TR/referrer-policy +.. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer +.. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade +.. _"same-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin +.. _"origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin +.. _"strict-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin +.. _"origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin +.. _"strict-origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin +.. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url + + UrlLengthMiddleware ------------------- diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a5931a3d5..35d9844a7 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -234,6 +234,7 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True +REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 6a8c46543..b444e34bb 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,22 +2,355 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ +from six.moves.urllib.parse import urlparse +import warnings -from scrapy.http import Request +from w3lib.url import safe_url_string + +from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured +from scrapy import signals +from scrapy.utils.python import to_native_str +from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import load_object +from scrapy.utils.url import strip_url + + +LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) + +POLICY_NO_REFERRER = "no-referrer" +POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade" +POLICY_SAME_ORIGIN = "same-origin" +POLICY_ORIGIN = "origin" +POLICY_STRICT_ORIGIN = "strict-origin" +POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin" +POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN = "strict-origin-when-cross-origin" +POLICY_UNSAFE_URL = "unsafe-url" +POLICY_SCRAPY_DEFAULT = "scrapy-default" + + +class ReferrerPolicy(object): + + NOREFERRER_SCHEMES = LOCAL_SCHEMES + + def referrer(self, response_url, request_url): + raise NotImplementedError() + + def stripped_referrer(self, url): + if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: + return self.strip_url(url) + + def origin_referrer(self, url): + if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: + return self.origin(url) + + def strip_url(self, url, origin_only=False): + """ + https://www.w3.org/TR/referrer-policy/#strip-url + + If url is null, return no referrer. + If url's scheme is a local scheme, then return no referrer. + Set url's username to the empty string. + Set url's password to null. + Set url's fragment to null. + If the origin-only flag is true, then: + Set url's path to null. + Set url's query to null. + Return url. + """ + if not url: + return None + return strip_url(url, + strip_credentials=True, + strip_fragment=True, + strip_default_port=True, + origin_only=origin_only) + + def origin(self, url): + """Return serialized origin (scheme, host, path) for a request or response URL.""" + return self.strip_url(url, origin_only=True) + + def potentially_trustworthy(self, url): + # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy + parsed_url = urlparse(url) + if parsed_url.scheme in ('data',): + return False + return self.tls_protected(url) + + def tls_protected(self, url): + return urlparse(url).scheme in ('https', 'ftps') + + +class NoReferrerPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer + + The simplest policy is "no-referrer", which specifies that no referrer information + is to be sent along with requests made from a particular request client to any origin. + The header will be omitted entirely. + """ + name = POLICY_NO_REFERRER + + def referrer(self, response_url, request_url): + return None + + +class NoReferrerWhenDowngradePolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + + The "no-referrer-when-downgrade" policy sends a full URL along with requests + from a TLS-protected environment settings object to a potentially trustworthy URL, + and requests from clients which are not TLS-protected to any origin. + + Requests from TLS-protected clients to non-potentially trustworthy URLs, + on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + + This is a user agent's default behavior, if no policy is otherwise specified. + """ + name = POLICY_NO_REFERRER_WHEN_DOWNGRADE + + def referrer(self, response_url, request_url): + if not self.tls_protected(response_url) or self.tls_protected(request_url): + return self.stripped_referrer(response_url) + + +class SameOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin + + The "same-origin" policy specifies that a full URL, stripped for use as a referrer, + is sent as referrer information when making same-origin requests from a particular request client. + + Cross-origin requests, on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + """ + name = POLICY_SAME_ORIGIN + + def referrer(self, response_url, request_url): + if self.origin(response_url) == self.origin(request_url): + return self.stripped_referrer(response_url) + + +class OriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-origin + + The "origin" policy specifies that only the ASCII serialization + of the origin of the request client is sent as referrer information + when making both same-origin requests and cross-origin requests + from a particular request client. + """ + name = POLICY_ORIGIN + + def referrer(self, response_url, request_url): + return self.origin_referrer(response_url) + + +class StrictOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin + + The "strict-origin" policy sends the ASCII serialization + of the origin of the request client when making requests: + - from a TLS-protected environment settings object to a potentially trustworthy URL, and + - from non-TLS-protected environment settings objects to any origin. + + Requests from TLS-protected request clients to non- potentially trustworthy URLs, + on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + """ + name = POLICY_STRICT_ORIGIN + + def referrer(self, response_url, request_url): + if ((self.tls_protected(response_url) and + self.potentially_trustworthy(request_url)) + or not self.tls_protected(response_url)): + return self.origin_referrer(response_url) + + +class OriginWhenCrossOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin + + The "origin-when-cross-origin" policy specifies that a full URL, + stripped for use as a referrer, is sent as referrer information + when making same-origin requests from a particular request client, + and only the ASCII serialization of the origin of the request client + is sent as referrer information when making cross-origin requests + from a particular request client. + """ + name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN + + def referrer(self, response_url, request_url): + origin = self.origin(response_url) + if origin == self.origin(request_url): + return self.stripped_referrer(response_url) + else: + return origin + + +class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin + + The "strict-origin-when-cross-origin" policy specifies that a full URL, + stripped for use as a referrer, is sent as referrer information + when making same-origin requests from a particular request client, + and only the ASCII serialization of the origin of the request client + when making cross-origin requests: + + - from a TLS-protected environment settings object to a potentially trustworthy URL, and + - from non-TLS-protected environment settings objects to any origin. + + Requests from TLS-protected clients to non- potentially trustworthy URLs, + on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + """ + name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN + + def referrer(self, response_url, request_url): + origin = self.origin(response_url) + if origin == self.origin(request_url): + return self.stripped_referrer(response_url) + elif ((self.tls_protected(response_url) and + self.potentially_trustworthy(request_url)) + or not self.tls_protected(response_url)): + return self.origin_referrer(response_url) + + +class UnsafeUrlPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url + + The "unsafe-url" policy specifies that a full URL, stripped for use as a referrer, + is sent along with both cross-origin requests + and same-origin requests made from a particular request client. + + Note: The policy's name doesn't lie; it is unsafe. + This policy will leak origins and paths from TLS-protected resources + to insecure origins. + Carefully consider the impact of setting such a policy for potentially sensitive documents. + """ + name = POLICY_UNSAFE_URL + + def referrer(self, response_url, request_url): + return self.stripped_referrer(response_url) + + +class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): + """ + A variant of "no-referrer-when-downgrade", + with the addition that "Referer" is not sent if the parent request was + using ``file://`` or ``s3://`` scheme. + """ + NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') + name = POLICY_SCRAPY_DEFAULT + + +_policy_classes = {p.name: p for p in ( + NoReferrerPolicy, + NoReferrerWhenDowngradePolicy, + SameOriginPolicy, + OriginPolicy, + StrictOriginPolicy, + OriginWhenCrossOriginPolicy, + StrictOriginWhenCrossOriginPolicy, + UnsafeUrlPolicy, + DefaultReferrerPolicy, +)} + + +def _load_policy_class(policy, warning_only=False): + """ + Expect a string for the path to the policy class, + otherwise try to interpret the string as a standard value + from https://www.w3.org/TR/referrer-policy/#referrer-policies + """ + try: + return load_object(policy) + except ValueError: + try: + return _policy_classes[policy.lower()] + except KeyError: + msg = "Could not load referrer policy %r" % policy + if not warning_only: + raise RuntimeError(msg) + else: + warnings.warn(msg, RuntimeWarning) + return None + class RefererMiddleware(object): + def __init__(self, settings=None): + self.default_policy = DefaultReferrerPolicy + if settings is not None: + self.default_policy = _load_policy_class( + settings.get('REFERRER_POLICY')) + @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured - return cls() + mw = cls(crawler.settings) + + # Note: this hook is a bit of a hack to intercept redirections + crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled) + + return mw + + def policy(self, resp_or_url, request): + """ + Determine Referrer-Policy to use from a parent Response (or URL), + and a Request to be sent. + + - if a valid policy is set in Request meta, it is used. + - if the policy is set in meta but is wrong (e.g. a typo error), + the policy from settings is used + - if the policy is not set in Request meta, + but there is a Referrer-policy header in the parent response, + it is used if valid + - otherwise, the policy from settings is used. + """ + policy_name = request.meta.get('referrer_policy') + if policy_name is None: + if isinstance(resp_or_url, Response): + policy_name = to_native_str( + resp_or_url.headers.get('Referrer-Policy', '').decode('latin1')) + if policy_name is None: + return self.default_policy() + + cls = _load_policy_class(policy_name, warning_only=True) + return cls() if cls else self.default_policy() def process_spider_output(self, response, result, spider): def _set_referer(r): if isinstance(r, Request): - r.headers.setdefault('Referer', response.url) + referrer = self.policy(response, r).referrer(response.url, r.url) + if referrer is not None: + r.headers.setdefault('Referer', referrer) return r return (_set_referer(r) for r in result or ()) + def request_scheduled(self, request, spider): + # check redirected request to patch "Referer" header if necessary + redirected_urls = request.meta.get('redirect_urls', []) + if redirected_urls: + request_referrer = request.headers.get('Referer') + # we don't patch the referrer value if there is none + if request_referrer is not None: + # the request's referrer header value acts as a surrogate + # for the parent response URL + # + # Note: if the 3xx response contained a Referrer-Policy header, + # the information is not available using this hook + parent_url = safe_url_string(request_referrer) + policy_referrer = self.policy(parent_url, request).referrer( + parent_url, request.url) + if policy_referrer != request_referrer: + if policy_referrer is None: + request.headers.pop('Referer') + else: + request.headers['Referer'] = policy_referrer diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index dc1cce4ac..8eed31060 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead. """ import posixpath import re -from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse) +from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse) # scrapy.utils.url was moved to w3lib.url and import * ensures this # move doesn't break old code @@ -103,3 +103,34 @@ def guess_scheme(url): return any_to_uri(url) else: return add_http_if_no_scheme(url) + + +def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True): + + """Strip URL string from some of its components: + + - `strip_credentials` removes "user:password@" + - `strip_default_port` removes ":80" (resp. ":443", ":21") + from http:// (resp. https://, ftp://) URLs + - `origin_only` replaces path component with "/", also dropping + query and fragment components ; it also strips credentials + - `strip_fragment` drops any #fragment component + """ + + parsed_url = urlparse(url) + netloc = parsed_url.netloc + if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password): + netloc = netloc.split('@')[-1] + if strip_default_port and parsed_url.port: + if (parsed_url.scheme, parsed_url.port) in (('http', 80), + ('https', 443), + ('ftp', 21)): + netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '') + return urlunparse(( + parsed_url.scheme, + netloc, + '/' if origin_only else parsed_url.path, + '' if origin_only else parsed_url.params, + '' if origin_only else parsed_url.query, + '' if strip_fragment else parsed_url.fragment + )) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index bd7673efb..b1c815876 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,21 +1,867 @@ +from six.moves.urllib.parse import urlparse from unittest import TestCase +import warnings +from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request +from scrapy.settings import Settings from scrapy.spiders import Spider -from scrapy.spidermiddlewares.referer import RefererMiddleware +from scrapy.downloadermiddlewares.redirect import RedirectMiddleware +from scrapy.spidermiddlewares.referer import RefererMiddleware, \ + POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ + POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ + POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \ + POLICY_STRICT_ORIGIN, POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, \ + DefaultReferrerPolicy, \ + NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ + OriginWhenCrossOriginPolicy, OriginPolicy, \ + StrictOriginWhenCrossOriginPolicy, StrictOriginPolicy, \ + SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy class TestRefererMiddleware(TestCase): + req_meta = {} + resp_headers = {} + settings = {} + scenarii = [ + ('http://scrapytest.org', 'http://scrapytest.org/', b'http://scrapytest.org'), + ] + def setUp(self): self.spider = Spider('foo') - self.mw = RefererMiddleware() + settings = Settings(self.settings) + self.mw = RefererMiddleware(settings) - def test_process_spider_output(self): - res = Response('http://scrapytest.org') - reqs = [Request('http://scrapytest.org/')] + def get_request(self, target): + return Request(target, meta=self.req_meta) - out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://scrapytest.org') + def get_response(self, origin): + return Response(origin, headers=self.resp_headers) + def test(self): + + for origin, target, referrer in self.scenarii: + response = self.get_response(origin) + request = self.get_request(target) + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) + + +class MixinDefault(object): + """ + Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + + with some additional filtering of s3:// + """ + scenarii = [ + ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'), + ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'), + ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'), + ('https://example.com/', 'http://scrapy.org/', None), + + # no credentials leak + ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'), + + # no referrer leak for local schemes + ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + + # no referrer leak for s3 origins + ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None), + ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None), + ] + + +class MixinNoReferrer(object): + scenarii = [ + ('https://example.com/page.html', 'https://example.com/', None), + ('http://www.example.com/', 'https://scrapy.org/', None), + ('http://www.example.com/', 'http://scrapy.org/', None), + ('https://www.example.com/', 'http://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + ] + + +class MixinNoReferrerWhenDowngrade(object): + scenarii = [ + # TLS to TLS: send non-empty referrer + ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), + ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'), + ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'), + + # TLS to non-TLS: do not send referrer + ('https://example.com/page.html', 'http://not.example.com/', None), + ('https://example.com/page.html', 'http://scrapy.org/', None), + ('ftps://example.com/urls.zip', 'http://scrapy.org/', None), + + # non-TLS to TLS or non-TLS: send referrer + ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'), + ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'), + ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'), + ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'), + + # test for user/password stripping + ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ] + + +class MixinSameOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: do NOT send referrer + ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None), + + # Different port: do NOT send referrer + ('https://example.com:444/page.html', 'https://example.com/not-page.html', None), + ('http://example.com:81/page.html', 'http://example.com/not-page.html', None), + ('http://example.com/page.html', 'http://example.com:81/not-page.html', None), + + # Different protocols: do NOT send refferer + ('https://example.com/page.html', 'http://example.com/not-page.html', None), + ('https://example.com/page.html', 'http://not.example.com/', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), + ] + + +class MixinOrigin(object): + scenarii = [ + # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), + ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ] + + +class MixinStrictOrigin(object): + scenarii = [ + # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), + ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), + + # downgrade: send nothing + ('https://example.com/page.html', 'http://scrapy.org', None), + + # upgrade: send origin + ('http://example.com/page.html', 'https://scrapy.org', b'http://example.com/'), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('https://user:password@example.com/page.html', 'http://scrapy.org', None), + ] + + +class MixinOriginWhenCrossOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: send origin as referrer + ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), + ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), + ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + # exact match required + ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), + + # Different port: send origin as referrer + ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), + ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), + + # Different protocols: send origin as referrer + ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'), + ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + + # test for user/password stripping + ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + # TLS to non-TLS downgrade: send origin + ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), + ] + + +class MixinStrictOriginWhenCrossOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: send origin as referrer + ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), + ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), + ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + # exact match required + ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), + + # Different port: send origin as referrer + ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), + ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), + + # downgrade + ('https://example4.com/page.html', 'http://example4.com/not-page.html', None), + ('https://example4.com/page.html', 'http://not.example4.com/', None), + + # non-TLS to non-TLS + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), + + # upgrade + ('http://example4.com/page.html', 'https://example4.com/not-page.html', b'http://example4.com/'), + ('http://example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/'), + + # Different protocols: send origin as referrer + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + + # test for user/password stripping + ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + + # TLS to non-TLS downgrade: send nothing + ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', None), + ] + + +class MixinUnsafeUrl(object): + scenarii = [ + # TLS to TLS: send referrer + ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), + ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'), + ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'), + ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'), + + # TLS to non-TLS: send referrer (yes, it's unsafe) + ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'), + ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'), + ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'), + + # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe) + ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'), + ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'), + ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'), + ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), + + # test for user/password stripping + ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), + ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), + ] + + +class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware): + pass + + +# --- Tests using settings to set policy using class path +class TestSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} + + +class TestSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + + +class TestSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + + +class TestSettingsOrigin(MixinOrigin, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} + + +class TestSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} + + +class TestSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + + +class TestSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} + + +class TestSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + + +class CustomPythonOrgPolicy(ReferrerPolicy): + """ + A dummy policy that returns referrer as http(s)://python.org + depending on the scheme of the target URL. + """ + def referrer(self, response, request): + scheme = urlparse(request).scheme + if scheme == 'https': + return b'https://python.org/' + elif scheme == 'http': + return b'http://python.org/' + + +class TestSettingsCustomPolicy(TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} + scenarii = [ + ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), + ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'), + ('http://example.com/', 'https://scrapy.org/', b'https://python.org/'), + ('https://example.com/', 'http://scrapy.org/', b'http://python.org/'), + ('file:///home/path/to/somefile.html', 'https://scrapy.org/', b'https://python.org/'), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', b'http://python.org/'), + + ] + +# --- Tests using Request meta dict to set policy +class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT} + + +class TestRequestMetaNoReferrer(MixinNoReferrer, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_NO_REFERRER} + + +class TestRequestMetaNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE} + + +class TestRequestMetaSameOrigin(MixinSameOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_SAME_ORIGIN} + + +class TestRequestMetaOrigin(MixinOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_ORIGIN} + + +class TestRequestMetaSrictOrigin(MixinStrictOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN} + + +class TestRequestMetaOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} + + +class TestRequestMetaStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} + + +class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + +class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + +class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + req_meta = {'referrer_policy': POLICY_NO_REFERRER} + + +class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + +class TestRequestMetaSettingFallback(TestCase): + + params = [ + ( + # When an unknown policy is referenced in Request.meta + # (here, a typo error), + # the policy defined in settings takes precedence + {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}, + {}, + {'referrer_policy': 'ssscrapy-default'}, + OriginWhenCrossOriginPolicy, + True + ), + ( + # same as above but with string value for settings policy + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {}, + {'referrer_policy': 'ssscrapy-default'}, + OriginWhenCrossOriginPolicy, + True + ), + ( + # request meta references a wrong policy but it is set, + # so the Referrer-Policy header in response is not used, + # and the settings' policy is applied + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {'Referrer-Policy': 'unsafe-url'}, + {'referrer_policy': 'ssscrapy-default'}, + OriginWhenCrossOriginPolicy, + True + ), + ( + # here, request meta does not set the policy + # so response headers take precedence + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {'Referrer-Policy': 'unsafe-url'}, + {}, + UnsafeUrlPolicy, + False + ), + ( + # here, request meta does not set the policy, + # but response headers also use an unknown policy, + # so the settings' policy is used + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {'Referrer-Policy': 'unknown'}, + {}, + OriginWhenCrossOriginPolicy, + True + ) + ] + + def test(self): + + origin = 'http://www.scrapy.org' + target = 'http://www.example.com' + + for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]: + spider = Spider('foo') + mw = RefererMiddleware(Settings(settings)) + + response = Response(origin, headers=response_headers) + request = Request(target, meta=request_meta) + + with warnings.catch_warnings(record=True) as w: + policy = mw.policy(response, request) + self.assertIsInstance(policy, policy_class) + + if check_warning: + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, RuntimeWarning, w[0].message) + + +class TestSettingsPolicyByName(TestCase): + + def test_valid_name(self): + for s, p in [ + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_STRICT_ORIGIN, StrictOriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), + ]: + settings = Settings({'REFERRER_POLICY': s}) + mw = RefererMiddleware(settings) + self.assertEquals(mw.default_policy, p) + + def test_valid_name_casevariants(self): + for s, p in [ + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_STRICT_ORIGIN, StrictOriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), + ]: + settings = Settings({'REFERRER_POLICY': s.upper()}) + mw = RefererMiddleware(settings) + self.assertEquals(mw.default_policy, p) + + def test_invalid_name(self): + settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'}) + with self.assertRaises(RuntimeError): + mw = RefererMiddleware(settings) + + +class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} + +class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} + +class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} + + +class TestReferrerOnRedirect(TestRefererMiddleware): + + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + scenarii = [ + ( 'http://scrapytest.org/1', # parent + 'http://scrapytest.org/2', # target + ( + # redirections: code, URL + (301, 'http://scrapytest.org/3'), + (301, 'http://scrapytest.org/4'), + ), + b'http://scrapytest.org/1', # expected initial referer + b'http://scrapytest.org/1', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/1', + 'https://scrapytest.org/2', + ( + # redirecting to non-secure URL + (301, 'http://scrapytest.org/3'), + ), + b'https://scrapytest.org/1', + b'https://scrapytest.org/1', + ), + ( 'https://scrapytest.org/1', + 'https://scrapytest.com/2', + ( + # redirecting to non-secure URL: different origin + (301, 'http://scrapytest.com/3'), + ), + b'https://scrapytest.org/1', + b'https://scrapytest.org/1', + ), + ] + + def setUp(self): + self.spider = Spider('foo') + settings = Settings(self.settings) + self.referrermw = RefererMiddleware(settings) + self.redirectmw = RedirectMiddleware(settings) + + def test(self): + + for parent, target, redirections, init_referrer, final_referrer in self.scenarii: + response = self.get_response(parent) + request = self.get_request(target) + + out = list(self.referrermw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), init_referrer) + + for status, url in redirections: + response = Response(request.url, headers={'Location': url}, status=status) + request = self.redirectmw.process_response(request, response, self.spider) + self.referrermw.request_scheduled(request, self.spider) + + assert isinstance(request, Request) + self.assertEquals(request.headers.get('Referer'), final_referrer) + + +class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): + """ + No Referrer policy never sets the "Referer" header. + HTTP redirections should not change that. + """ + settings = {'REFERRER_POLICY': 'no-referrer'} + scenarii = [ + ( 'http://scrapytest.org/1', # parent + 'http://scrapytest.org/2', # target + ( + # redirections: code, URL + (301, 'http://scrapytest.org/3'), + (301, 'http://scrapytest.org/4'), + ), + None, # expected initial "Referer" + None, # expected "Referer" for the redirection request + ), + ( 'https://scrapytest.org/1', + 'https://scrapytest.org/2', + ( + (301, 'http://scrapytest.org/3'), + ), + None, + None, + ), + ( 'https://scrapytest.org/1', + 'https://example.com/2', # different origin + ( + (301, 'http://scrapytest.com/3'), + ), + None, + None, + ), + ] + + +class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect): + """ + Same Origin policy sends the full URL as "Referer" if the target origin + is the same as the parent response (same protocol, same domain, same port). + + HTTP redirections to a different domain or a lower secure level + should have the "Referer" removed. + """ + settings = {'REFERRER_POLICY': 'same-origin'} + scenarii = [ + ( 'http://scrapytest.org/101', # origin + 'http://scrapytest.org/102', # target + ( + # redirections: code, URL + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/101', # expected initial "Referer" + b'http://scrapytest.org/101', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting from secure to non-secure URL == different origin + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/201', + None, + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # different domain == different origin + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/301', + None, + ), + ] + + +class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect): + """ + Strict Origin policy will always send the "origin" as referrer + (think of it as the parent URL without the path part), + unless the security level is lower and no "Referer" is sent. + + Redirections from secure to non-secure URLs should have the + "Referrer" header removed if necessary. + """ + settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN} + scenarii = [ + ( 'http://scrapytest.org/101', + 'http://scrapytest.org/102', + ( + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/', # send origin + b'http://scrapytest.org/', # redirects to same origin: send origin + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting to non-secure URL: no referrer + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/', + None, + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # redirecting to non-secure URL (different domain): no referrer + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/', + None, + ), + ( 'http://scrapy.org/401', + 'http://example.com/402', + ( + (301, 'http://scrapytest.org/403'), + ), + b'http://scrapy.org/', + b'http://scrapy.org/', + ), + ( 'https://scrapy.org/501', + 'https://example.com/502', + ( + # HTTPS all along, so origin referrer is kept as-is + (301, 'https://google.com/503'), + (301, 'https://facebook.com/504'), + ), + b'https://scrapy.org/', + b'https://scrapy.org/', + ), + ( 'https://scrapytest.org/601', + 'http://scrapytest.org/602', # TLS to non-TLS: no referrer + ( + (301, 'https://scrapytest.org/603'), # TLS URL again: (still) no referrer + ), + None, + None, + ), + ] + + +class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect): + """ + Origin When Cross-Origin policy sends the full URL as "Referer", + unless the target's origin is different (different domain, different protocol) + in which case only the origin is sent. + + Redirections to a different origin should strip the "Referer" + to the parent origin. + """ + settings = {'REFERRER_POLICY': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} + scenarii = [ + ( 'http://scrapytest.org/101', # origin + 'http://scrapytest.org/102', # target + redirection + ( + # redirections: code, URL + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/101', # expected initial referer + b'http://scrapytest.org/101', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting to non-secure URL: send origin + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/201', + b'https://scrapytest.org/', + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # redirecting to non-secure URL (different domain): send origin + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/301', + b'https://scrapytest.org/', + ), + ( 'http://scrapy.org/401', + 'http://example.com/402', + ( + (301, 'http://scrapytest.org/403'), + ), + b'http://scrapy.org/', + b'http://scrapy.org/', + ), + ( 'https://scrapy.org/501', + 'https://example.com/502', + ( + # all different domains: send origin + (301, 'https://google.com/503'), + (301, 'https://facebook.com/504'), + ), + b'https://scrapy.org/', + b'https://scrapy.org/', + ), + ( 'https://scrapytest.org/301', + 'http://scrapytest.org/302', # TLS to non-TLS: send origin + ( + (301, 'https://scrapytest.org/303'), # TLS URL again: send origin (also) + ), + b'https://scrapytest.org/', + b'https://scrapytest.org/', + ), + ] + + +class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect): + """ + Strict Origin When Cross-Origin policy sends the full URL as "Referer", + unless the target's origin is different (different domain, different protocol) + in which case only the origin is sent... + Unless there's also a downgrade in security and then the "Referer" header + is not sent. + + Redirections to a different origin should strip the "Referer" to the parent origin, + and from https:// to http:// will remove the "Referer" header. + """ + settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} + scenarii = [ + ( 'http://scrapytest.org/101', # origin + 'http://scrapytest.org/102', # target + redirection + ( + # redirections: code, URL + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/101', # expected initial referer + b'http://scrapytest.org/101', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting to non-secure URL: do not send the "Referer" header + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/201', + None, + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # redirecting to non-secure URL (different domain): send origin + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/301', + None, + ), + ( 'http://scrapy.org/401', + 'http://example.com/402', + ( + (301, 'http://scrapytest.org/403'), + ), + b'http://scrapy.org/', + b'http://scrapy.org/', + ), + ( 'https://scrapy.org/501', + 'https://example.com/502', + ( + # all different domains: send origin + (301, 'https://google.com/503'), + (301, 'https://facebook.com/504'), + ), + b'https://scrapy.org/', + b'https://scrapy.org/', + ), + ( 'https://scrapytest.org/601', + 'http://scrapytest.org/602', # TLS to non-TLS: do not send "Referer" + ( + (301, 'https://scrapytest.org/603'), # TLS URL again: (still) send nothing + ), + None, + None, + ), + ] diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index f46d1d927..c2b9fc176 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -6,7 +6,8 @@ from six.moves.urllib.parse import urlparse from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, - add_http_if_no_scheme, guess_scheme, parse_url) + add_http_if_no_scheme, guess_scheme, + parse_url, strip_url) __doctests__ = ['scrapy.utils.url'] @@ -241,5 +242,171 @@ for k, args in enumerate ([ setattr (GuessSchemeTest, t_method.__name__, t_method) +class StripUrl(unittest.TestCase): + + def test_noop(self): + self.assertEqual(strip_url( + 'http://www.example.com/index.html'), + 'http://www.example.com/index.html') + + def test_noop_query_string(self): + self.assertEqual(strip_url( + 'http://www.example.com/index.html?somekey=somevalue'), + 'http://www.example.com/index.html?somekey=somevalue') + + def test_fragments(self): + self.assertEqual(strip_url( + 'http://www.example.com/index.html?somekey=somevalue#section', strip_fragment=False), + 'http://www.example.com/index.html?somekey=somevalue#section') + + def test_path(self): + for input_url, origin, output_url in [ + ('http://www.example.com/', + False, + 'http://www.example.com/'), + + ('http://www.example.com', + False, + 'http://www.example.com'), + + ('http://www.example.com', + True, + 'http://www.example.com/'), + ]: + self.assertEqual(strip_url(input_url, origin_only=origin), output_url) + + def test_credentials(self): + for i, o in [ + ('http://username@www.example.com/index.html?somekey=somevalue#section', + 'http://www.example.com/index.html?somekey=somevalue'), + + ('https://username:@www.example.com/index.html?somekey=somevalue#section', + 'https://www.example.com/index.html?somekey=somevalue'), + + ('ftp://username:password@www.example.com/index.html?somekey=somevalue#section', + 'ftp://www.example.com/index.html?somekey=somevalue'), + ]: + self.assertEqual(strip_url(i, strip_credentials=True), o) + + def test_credentials_encoded_delims(self): + for i, o in [ + # user: "username@" + # password: none + ('http://username%40@www.example.com/index.html?somekey=somevalue#section', + 'http://www.example.com/index.html?somekey=somevalue'), + + # user: "username:pass" + # password: "" + ('https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section', + 'https://www.example.com/index.html?somekey=somevalue'), + + # user: "me" + # password: "user@domain.com" + ('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section', + 'ftp://www.example.com/index.html?somekey=somevalue'), + ]: + self.assertEqual(strip_url(i, strip_credentials=True), o) + + def test_default_ports_creds_off(self): + for i, o in [ + ('http://username:password@www.example.com:80/index.html?somekey=somevalue#section', + 'http://www.example.com/index.html?somekey=somevalue'), + + ('http://username:password@www.example.com:8080/index.html#section', + 'http://www.example.com:8080/index.html'), + + ('http://username:password@www.example.com:443/index.html?somekey=somevalue&someotherkey=sov#section', + 'http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov'), + + ('https://username:password@www.example.com:443/index.html', + 'https://www.example.com/index.html'), + + ('https://username:password@www.example.com:442/index.html', + 'https://www.example.com:442/index.html'), + + ('https://username:password@www.example.com:80/index.html', + 'https://www.example.com:80/index.html'), + + ('ftp://username:password@www.example.com:21/file.txt', + 'ftp://www.example.com/file.txt'), + + ('ftp://username:password@www.example.com:221/file.txt', + 'ftp://www.example.com:221/file.txt'), + ]: + self.assertEqual(strip_url(i), o) + + def test_default_ports(self): + for i, o in [ + ('http://username:password@www.example.com:80/index.html', + 'http://username:password@www.example.com/index.html'), + + ('http://username:password@www.example.com:8080/index.html', + 'http://username:password@www.example.com:8080/index.html'), + + ('http://username:password@www.example.com:443/index.html', + 'http://username:password@www.example.com:443/index.html'), + + ('https://username:password@www.example.com:443/index.html', + 'https://username:password@www.example.com/index.html'), + + ('https://username:password@www.example.com:442/index.html', + 'https://username:password@www.example.com:442/index.html'), + + ('https://username:password@www.example.com:80/index.html', + 'https://username:password@www.example.com:80/index.html'), + + ('ftp://username:password@www.example.com:21/file.txt', + 'ftp://username:password@www.example.com/file.txt'), + + ('ftp://username:password@www.example.com:221/file.txt', + 'ftp://username:password@www.example.com:221/file.txt'), + ]: + self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o) + + def test_default_ports_keep(self): + for i, o in [ + ('http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov#section', + 'http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov'), + + ('http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov#section', + 'http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov'), + + ('http://username:password@www.example.com:443/index.html', + 'http://username:password@www.example.com:443/index.html'), + + ('https://username:password@www.example.com:443/index.html', + 'https://username:password@www.example.com:443/index.html'), + + ('https://username:password@www.example.com:442/index.html', + 'https://username:password@www.example.com:442/index.html'), + + ('https://username:password@www.example.com:80/index.html', + 'https://username:password@www.example.com:80/index.html'), + + ('ftp://username:password@www.example.com:21/file.txt', + 'ftp://username:password@www.example.com:21/file.txt'), + + ('ftp://username:password@www.example.com:221/file.txt', + 'ftp://username:password@www.example.com:221/file.txt'), + ]: + self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o) + + def test_origin_only(self): + for i, o in [ + ('http://username:password@www.example.com/index.html', + 'http://www.example.com/'), + + ('http://username:password@www.example.com:80/foo/bar?query=value#somefrag', + 'http://www.example.com/'), + + ('http://username:password@www.example.com:8008/foo/bar?query=value#somefrag', + 'http://www.example.com:8008/'), + + ('https://username:password@www.example.com:443/index.html', + 'https://www.example.com/'), + ]: + self.assertEqual(strip_url(i, origin_only=True), o) + + if __name__ == "__main__": unittest.main()