From 0e65eb33c3182896ce45feec2ba32fb7e5b0cb60 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 26 Apr 2016 15:12:00 +0200 Subject: [PATCH] Use six.PY2 instead of six.PY3 for Python version variations Also don't test passed encoding against 'utf8'; Just consider that if encoding failed, it must have been another encoding. --- scrapy/utils/url.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 716513648..069666dbc 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -83,17 +83,16 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, try: scheme, netloc, path, params, query, fragment = _safe_ParseResult( parse_url(url), encoding=encoding) - except UnicodeError as e: - if encoding != 'utf8': - scheme, netloc, path, params, query, fragment = _safe_ParseResult( - parse_url(url), encoding='utf8') - else: - raise + except UnicodeEncodeError as e: + scheme, netloc, path, params, query, fragment = _safe_ParseResult( + parse_url(url), encoding='utf8') # 1. decode query-string as UTF-8 (or keep raw bytes), # sort values, # and percent-encode them back - if not six.PY2: + if six.PY2: + keyvals = parse_qsl(query, keep_blank_values) + else: # Python3's urllib.parse.parse_qsl does not work as wanted # for percent-encoded characters that do not match passed encoding, # they get lost. @@ -118,8 +117,6 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, # IRIs (namely, to be able to include non-ASCII characters) can only be # used if the query part is encoded in UTF-8. keyvals = parse_qsl_to_bytes(query, keep_blank_values) - else: - keyvals = parse_qsl(query, keep_blank_values) keyvals.sort() query = urlencode(keyvals) @@ -138,16 +135,17 @@ def _unquotepath(path): for reserved in ('2f', '2F', '3f', '3F'): path = path.replace('%' + reserved, '%25' + reserved.upper()) - if six.PY3: - # standard lib's unquote() does not work in Python 3 - # for non-UTF-8 percent-escaped characters, they get lost. + if six.PY2: + # in Python 2, '%a3' becomes '\xa3', which is what we want + return unquote(path) + else: + # in Python 3, + # standard lib's unquote() does not work for non-UTF-8 + # percent-escaped characters, they get lost. # e.g., '%a3' becomes 'REPLACEMENT CHARACTER' (U+FFFD) # # unquote_to_bytes() returns raw bytes instead return unquote_to_bytes(path) - else: - # in Python 2, '%a3' becomes '\xa3', which is what we want - return unquote(path) def parse_url(url, encoding=None):