diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 4ed7e4c24..db276eefb 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -1,9 +1,10 @@ import logging from six.moves.urllib.parse import urljoin +from w3lib.url import safe_url_string + from scrapy.http import HtmlResponse from scrapy.utils.response import get_meta_refresh -from scrapy.utils.python import to_native_str from scrapy.exceptions import IgnoreRequest, NotConfigured logger = logging.getLogger(__name__) @@ -65,8 +66,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): if 'Location' not in response.headers or response.status not in allowed_status: return response - # HTTP header is ascii or latin1, redirected url will be percent-encoded utf-8 - location = to_native_str(response.headers['location'].decode('latin1')) + location = safe_url_string(response.headers['location']) redirected_url = urljoin(request.url, location) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 9db073cc5..e8c92affa 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -157,15 +157,15 @@ class RedirectMiddlewareTest(unittest.TestCase): latin1_location = u'/ação'.encode('latin1') # HTTP historically supports latin1 resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) - perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o' + perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o' self.assertEquals(perc_encoded_utf8_url, req_result.url) - def test_location_with_wrong_encoding(self): + def test_utf8_location(self): req = Request('http://scrapytest.org/first') - utf8_location = u'/ação' # header with wrong encoding (utf-8) + utf8_location = u'/ação'.encode('utf-8') # header using UTF-8 encoding resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) - perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%83%C2%A7%C3%83%C2%A3o' + perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o' self.assertEquals(perc_encoded_utf8_url, req_result.url)