Do not interpret non-ASCII bytes in "Location" and percent-encode them (#2322)

* Do not interpret non-ASCII bytes in "Location" and percent-encode them

Fixes GH-2321

The idea is to not guess the encoding of "Location" header value
and simply percent-encode non-ASCII bytes,
which should then be re-interpreted correctly by the remote website
in whatever encoding was used originally.

See https://tools.ietf.org/html/rfc3987#section-3.2

This is similar to the changes to safe_url_string in
https://github.com/scrapy/w3lib/pull/45

* Remove unused import

scrapy/downloadermiddlewares/redirect.py

@@ -1,9 +1,10 @@
 import logging
 from six.moves.urllib.parse import urljoin
 
+from w3lib.url import safe_url_string
+
 from scrapy.http import HtmlResponse
 from scrapy.utils.response import get_meta_refresh
-from scrapy.utils.python import to_native_str
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 
 logger = logging.getLogger(__name__)
@@ -65,8 +66,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
         if 'Location' not in response.headers or response.status not in allowed_status:
             return response
 
-        # HTTP header is ascii or latin1, redirected url will be percent-encoded utf-8
-        location = to_native_str(response.headers['location'].decode('latin1'))
+        location = safe_url_string(response.headers['location'])
 
         redirected_url = urljoin(request.url, location)
 

tests/test_downloadermiddleware_redirect.py

@@ -157,15 +157,15 @@ class RedirectMiddlewareTest(unittest.TestCase):
         latin1_location = u'/ação'.encode('latin1')  # HTTP historically supports latin1
         resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302)
         req_result = self.mw.process_response(req, resp, self.spider)
-        perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
+        perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o'
         self.assertEquals(perc_encoded_utf8_url, req_result.url)
 
-    def test_location_with_wrong_encoding(self):
+    def test_utf8_location(self):
         req = Request('http://scrapytest.org/first')
-        utf8_location = u'/ação'  # header with wrong encoding (utf-8)
+        utf8_location = u'/ação'.encode('utf-8')  # header using UTF-8 encoding
         resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302)
         req_result = self.mw.process_response(req, resp, self.spider)
-        perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%83%C2%A7%C3%83%C2%A3o'
+        perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
         self.assertEquals(perc_encoded_utf8_url, req_result.url)