diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 24c163089..f2910a0c8 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,6 +2,8 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ +from six.moves.urllib.parse import urlparse + from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals @@ -32,12 +34,14 @@ class ReferrerPolicy(object): raise NotImplementedError() def stripped_referrer(self, r): - return self.strip_url(r) + if urlparse(r).scheme not in self.NOREFERRER_SCHEMES: + return self.strip_url(r) def origin_referrer(self, r): - return self.strip_url(r, origin_only=True) + if urlparse(r).scheme not in self.NOREFERRER_SCHEMES: + return self.origin(r) - def strip_url(self, r, origin_only=False): + def strip_url(self, url, origin_only=False): """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -51,29 +55,27 @@ class ReferrerPolicy(object): Set url's query to null. Return url. """ - if r is None or not r.url: + if not url: return None - parsed_url = urlparse_cached(r) - if parsed_url.scheme not in self.NOREFERRER_SCHEMES: - return strip_url(parsed_url, - strip_credentials=True, - strip_fragment=True, - strip_default_port=True, - origin_only=origin_only) + return strip_url(url, + strip_credentials=True, + strip_fragment=True, + strip_default_port=True, + origin_only=origin_only) def origin(self, r): """Return serialized origin (scheme, host, path) for a request or response URL.""" return self.strip_url(r, origin_only=True) - def potentially_trustworthy(self, r): + def potentially_trustworthy(self, url): # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy - parsed_url = urlparse_cached(r) + parsed_url = urlparse(url) if parsed_url.scheme in ('data',): return False - return self.tls_protected(r) + return self.tls_protected(url) - def tls_protected(self, r): - return urlparse_cached(r).scheme in ('https', 'ftps') + def tls_protected(self, url): + return urlparse(url).scheme in ('https', 'ftps') class NoReferrerPolicy(ReferrerPolicy): @@ -287,12 +289,17 @@ class RefererMiddleware(object): crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled) return mw - def policy(self, response, request): + def policy(self, resp_or_url, request): + """ + Determine Referrer-Policy to use from a parent Response (or URL), + and a Request to be sent. + """ # policy set in request's meta dict takes precedence over default policy policy_name = request.meta.get('referrer_policy') if policy_name is None: - policy_name = to_native_str( - response.headers.get('Referrer-Policy', '').decode('latin1')) + if isinstance(resp_or_url, Response): + policy_name = to_native_str( + resp_or_url.headers.get('Referrer-Policy', '').decode('latin1')) cls = _policy_classes.get(policy_name.lower(), self.default_policy) return cls() @@ -300,7 +307,7 @@ class RefererMiddleware(object): def process_spider_output(self, response, result, spider): def _set_referer(r): if isinstance(r, Request): - referrer = self.policy(response, r).referrer(response, r) + referrer = self.policy(response, r).referrer(response.url, r.url) if referrer is not None: r.headers.setdefault('Referer', referrer) return r @@ -313,9 +320,9 @@ class RefererMiddleware(object): request_referrer = request.headers.get('Referer') # we don't patch the referrer value if there is none if request_referrer is not None: - faked_response = Response(redirected_urls[0]) - policy_referrer = self.policy(faked_response, - request).referrer(faked_response, request) + initial_url = redirected_urls[0] + policy_referrer = self.policy(initial_url, + request).referrer(orig_url, request.url) if policy_referrer != request_referrer: if policy_referrer is None: request.headers.pop('Referer') diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 9864f353d..8eed31060 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -117,12 +117,7 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only= - `strip_fragment` drops any #fragment component """ - if url is None: - return None - if not isinstance(url, ParseResult): - parsed_url = urlparse(url) - else: - parsed_url = url + parsed_url = urlparse(url) netloc = parsed_url.netloc if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password): netloc = netloc.split('@')[-1]