data URI download handler.

scrapy/core/downloader/handlers/data.py

@@ -0,0 +1,93 @@
+import base64
+import re
+from six.moves.urllib.parse import unquote
+
+from scrapy.http import TextResponse
+from scrapy.responsetypes import responsetypes
+from scrapy.utils.datatypes import CaselessDict
+from scrapy.utils.decorators import defers
+
+
+# ASCII characters.
+_char = set(map(chr, range(127)))
+
+# RFC 2045 token.
+_token = r'[{}]+'.format(re.escape(''.join(_char -
+                                           # Control characters.
+                                           set(map(chr, range(0, 32))) -
+                                           # tspecials and space.
+                                           set('()<>@,;:\\"/[]?= '))))
+
+# RFC 822 quoted-string, without surrounding quotation marks.
+_quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format(
+    re.escape(''.join(_char - {'"', '\\', '\r'})),
+    re.escape(''.join(_char))
+)
+
+# RFC 2397 mediatype.
+_mediatype_pattern = re.compile(r'{token}/{token}'.format(token=_token))
+
+_mediatype_parameter_pattern = re.compile(
+    r';({token})=(?:({token})|"({quoted})")'.format(token=_token,
+                                                    quoted=_quoted_string)
+)
+
+
+class DataURIDownloadHandler(object):
+    def __init__(self, settings):
+        super(DataURIDownloadHandler, self).__init__()
+
+    @defers
+    def download_request(self, request, spider):
+        url = request.url
+
+        scheme, url = url.split(':', 1)
+        if scheme != 'data':
+            raise ValueError("not a data URI")
+
+        # RFC 3986 section 2.1 allows percent encoding to escape characters
+        # that would be interpreted as delimiters, implying that actual
+        # delimiters should not be percent-encoded.
+        # Decoding before parsing will allow malformed URIs with
+        # percent-encoded delimiters, but it makes parsing easier and should
+        # not affect well-formed URIs, as the delimiters used in this URI
+        # scheme are not allowed, percent-encoded or not, in tokens.
+        url = unquote(url)
+
+        media_type = "text/plain"
+        media_type_params = CaselessDict()
+
+        m = _mediatype_pattern.match(url)
+        if m:
+            media_type = m.group()
+            url = url[m.end():]
+        else:
+            media_type_params['charset'] = "US-ASCII"
+
+        while True:
+            m = _mediatype_parameter_pattern.match(url)
+            if m:
+                attribute, value, value_quoted = m.groups()
+                if value_quoted:
+                    value = re.sub(r'\\(.)', '\1', value_quoted)
+                media_type_params[attribute] = value
+                url = url[m.end():]
+            else:
+                break
+
+        is_base64, data = url.split(',', 1)
+        if is_base64:
+            if is_base64 != ";base64":
+                raise ValueError("invalid data URI")
+            data = base64.b64decode(data)
+
+        respcls = responsetypes.from_mimetype(media_type)
+
+        resp_kwargs = {}
+
+        if media_type:
+            media_type = media_type.split('/')
+            if issubclass(respcls, TextResponse) and media_type[0] == 'text':
+                resp_kwargs['encoding'] = media_type_params.get('charset')
+
+        return respcls(url=request.url, body=data, **resp_kwargs)

tests/test_downloader_handlers.py

@@ -6,7 +6,6 @@ try:
     from unittest import mock
 except ImportError:
     import mock
-import shutil
 
 from twisted.trial import unittest
 from twisted.protocols.policies import WrappingFactory
@@ -20,6 +19,7 @@ from twisted.cred import portal, checkers, credentials
 from w3lib.url import path_to_file_uri
 
 from scrapy.core.downloader.handlers import DownloadHandlers
+from scrapy.core.downloader.handlers.data import DataURIDownloadHandler
 from scrapy.core.downloader.handlers.file import FileDownloadHandler
 from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler
 from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
@@ -29,6 +29,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.http.response.text import TextResponse
+from scrapy.responsetypes import responsetypes
 from scrapy.settings import Settings
 from scrapy.utils.test import get_crawler, skip_if_no_boto
 from scrapy.utils.python import to_bytes
@@ -828,3 +829,58 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
 
     def tearDown(self):
         shutil.rmtree(self.directory)
+
+
+class DataURITestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.download_handler = DataURIDownloadHandler(Settings())
+        self.download_request = self.download_handler.download_request
+        self.spider = Spider('foo')
+
+    def test_default_mediatype_encoding(self):
+        def _test(response):
+            self.assertEquals(response.text, 'A brief note')
+            self.assertEquals(type(response),
+                              responsetypes.from_mimetype("text/plain"))
+            self.assertEquals(response.encoding, "US-ASCII")
+
+        request = Request("data:,A%20brief%20note")
+        return self.download_request(request, self.spider).addCallback(_test)
+
+    def test_default_mediatype(self):
+        def _test(response):
+            self.assertEquals(response.text, u'\u038e\u03a3\u038e')
+            self.assertEquals(type(response),
+                              responsetypes.from_mimetype("text/plain"))
+            self.assertEquals(response.encoding, "iso-8859-7")
+
+        request = Request("data:;charset=iso-8859-7,%be%d3%be")
+        return self.download_request(request, self.spider).addCallback(_test)
+
+    def test_text_charset(self):
+        def _test(response):
+            self.assertEquals(response.text, u'\u038e\u03a3\u038e')
+            self.assertEquals(response.encoding, "iso-8859-7")
+
+        request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be")
+        return self.download_request(request, self.spider).addCallback(_test)
+
+    def test_mediatype_parameters(self):
+        def _test(response):
+            self.assertEquals(response.text, u'\u038e\u03a3\u038e')
+            self.assertEquals(type(response),
+                              responsetypes.from_mimetype("text/plain"))
+            self.assertEquals(response.encoding, "utf-8")
+
+        request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;'
+                          'charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22'
+                          ',%CE%8E%CE%A3%CE%8E')
+        return self.download_request(request, self.spider).addCallback(_test)
+
+    def test_base64(self):
+        def _test(response):
+            self.assertEquals(response.text, 'Hello, world.')
+
+        request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D')
+        return self.download_request(request, self.spider).addCallback(_test)