From db408528928b2d15043593032913fe40d6eb6783 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 20 Oct 2016 04:26:12 +0200 Subject: [PATCH] Do not interpret non-ASCII bytes in "Location" and percent-encode them (#2322) * Do not interpret non-ASCII bytes in "Location" and percent-encode them Fixes GH-2321 The idea is to not guess the encoding of "Location" header value and simply percent-encode non-ASCII bytes, which should then be re-interpreted correctly by the remote website in whatever encoding was used originally. See https://tools.ietf.org/html/rfc3987#section-3.2 This is similar to the changes to safe_url_string in https://github.com/scrapy/w3lib/pull/45 * Remove unused import --- scrapy/downloadermiddlewares/redirect.py | 6 +++--- tests/test_downloadermiddleware_redirect.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 4ed7e4c24..db276eefb 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -1,9 +1,10 @@ import logging from six.moves.urllib.parse import urljoin +from w3lib.url import safe_url_string + from scrapy.http import HtmlResponse from scrapy.utils.response import get_meta_refresh -from scrapy.utils.python import to_native_str from scrapy.exceptions import IgnoreRequest, NotConfigured logger = logging.getLogger(__name__) @@ -65,8 +66,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): if 'Location' not in response.headers or response.status not in allowed_status: return response - # HTTP header is ascii or latin1, redirected url will be percent-encoded utf-8 - location = to_native_str(response.headers['location'].decode('latin1')) + location = safe_url_string(response.headers['location']) redirected_url = urljoin(request.url, location) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 9db073cc5..e8c92affa 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -157,15 +157,15 @@ class RedirectMiddlewareTest(unittest.TestCase): latin1_location = u'/ação'.encode('latin1') # HTTP historically supports latin1 resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) - perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o' + perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o' self.assertEquals(perc_encoded_utf8_url, req_result.url) - def test_location_with_wrong_encoding(self): + def test_utf8_location(self): req = Request('http://scrapytest.org/first') - utf8_location = u'/ação' # header with wrong encoding (utf-8) + utf8_location = u'/ação'.encode('utf-8') # header using UTF-8 encoding resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) - perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%83%C2%A7%C3%83%C2%A3o' + perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o' self.assertEquals(perc_encoded_utf8_url, req_result.url)