From f2ee6be3bb311bb8f89ccd929bc0ebd8412d0183 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 18:33:31 +0200 Subject: [PATCH] Use urlparse_cached() for OriginPolicy --- scrapy/spidermiddlewares/referer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index cbbcd3e97..bf2a3c037 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -193,7 +193,9 @@ class OriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN def referrer(self, response, request): - return self.strip_url(response.url, origin_only=True) + stripped = self.strip_url_parsed(response, origin_only=True) + if stripped is not None: + return urlunparse(stripped) class OriginWhenCrossOriginPolicy(ReferrerPolicy):