Use w3lib data URI parser.

scrapy/core/downloader/handlers/data.py

@@ -1,106 +1,23 @@
-import base64
-import re
-
-import six
-
-if six.PY2:
-    from urllib import unquote
-else:
-    from urllib.parse import unquote_to_bytes as unquote
+from w3lib.url import parse_data_uri
 
 from scrapy.http import TextResponse
 from scrapy.responsetypes import responsetypes
-from scrapy.utils.datatypes import CaselessDict
 from scrapy.utils.decorators import defers
 
 
-# ASCII characters.
-_char = set(map(chr, range(127)))
-
-# RFC 2045 token.
-_token = r'[{}]+'.format(re.escape(''.join(_char -
-                                           # Control characters.
-                                           set(map(chr, range(0, 32))) -
-                                           # tspecials and space.
-                                           set('()<>@,;:\\"/[]?= '))))
-
-# RFC 822 quoted-string, without surrounding quotation marks.
-_quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format(
-    re.escape(''.join(_char - {'"', '\\', '\r'})),
-    re.escape(''.join(_char))
-)
-
-# Encode the regular expression strings to make them into bytes, as Python 3
-# bytes have no format() method, but bytes must be passed to re.compile() in
-# order to make a pattern object that can be used to match on bytes.
-
-# RFC 2397 mediatype.
-_mediatype_pattern = re.compile(
-    r'{token}/{token}'.format(token=_token).encode()
-)
-
-_mediatype_parameter_pattern = re.compile(
-    r';({token})=(?:({token})|"({quoted})")'.format(token=_token,
-                                                    quoted=_quoted_string
-                                                    ).encode()
-)
-
-
 class DataURIDownloadHandler(object):
     def __init__(self, settings):
         super(DataURIDownloadHandler, self).__init__()
 
     @defers
     def download_request(self, request, spider):
-        url = request.url
-
-        scheme, url = url.split(':', 1)
-        if scheme != 'data':
-            raise ValueError("not a data URI")
-
-        # RFC 3986 section 2.1 allows percent encoding to escape characters
-        # that would be interpreted as delimiters, implying that actual
-        # delimiters should not be percent-encoded.
-        # Decoding before parsing will allow malformed URIs with
-        # percent-encoded delimiters, but it makes parsing easier and should
-        # not affect well-formed URIs, as the delimiters used in this URI
-        # scheme are not allowed, percent-encoded or not, in tokens.
-        url = unquote(url)
-
-        media_type = "text/plain"
-        media_type_params = CaselessDict()
-
-        m = _mediatype_pattern.match(url)
-        if m:
-            media_type = m.group().decode()
-            url = url[m.end():]
-        else:
-            media_type_params['charset'] = "US-ASCII"
-
-        while True:
-            m = _mediatype_parameter_pattern.match(url)
-            if m:
-                attribute, value, value_quoted = m.groups()
-                if value_quoted:
-                    value = re.sub(br'\\(.)', r'\1', value_quoted)
-                media_type_params[attribute.decode()] = value.decode()
-                url = url[m.end():]
-            else:
-                break
-
-        is_base64, data = url.split(b',', 1)
-        if is_base64:
-            if is_base64 != b";base64":
-                raise ValueError("invalid data URI")
-            data = base64.b64decode(data)
-
-        respcls = responsetypes.from_mimetype(media_type)
+        uri = parse_data_uri(request.url)
+        respcls = responsetypes.from_mimetype(uri.media_type)
 
         resp_kwargs = {}
+        if (issubclass(respcls, TextResponse) and
+                uri.media_type.split('/')[0] == 'text'):
+            charset = uri.media_type_parameters.get('charset')
+            resp_kwargs['encoding'] = charset
 
-        if media_type:
-            media_type = media_type.split('/')
-            if issubclass(respcls, TextResponse) and media_type[0] == 'text':
-                resp_kwargs['encoding'] = media_type_params.get('charset')
-
-        return respcls(url=request.url, body=data, **resp_kwargs)
+        return respcls(url=request.url, body=uri.data, **resp_kwargs)