From d60642e1755b004e46b87e2011c1873f25895bb8 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Fri, 12 Aug 2016 00:45:42 -0300 Subject: [PATCH] data URI download handler. --- scrapy/core/downloader/handlers/data.py | 93 +++++++++++++++++++++++++ tests/test_downloader_handlers.py | 58 ++++++++++++++- 2 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 scrapy/core/downloader/handlers/data.py diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.py new file mode 100644 index 000000000..637a6f9c0 --- /dev/null +++ b/scrapy/core/downloader/handlers/data.py @@ -0,0 +1,93 @@ +import base64 +import re +from six.moves.urllib.parse import unquote + +from scrapy.http import TextResponse +from scrapy.responsetypes import responsetypes +from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.decorators import defers + + +# ASCII characters. +_char = set(map(chr, range(127))) + +# RFC 2045 token. +_token = r'[{}]+'.format(re.escape(''.join(_char - + # Control characters. + set(map(chr, range(0, 32))) - + # tspecials and space. + set('()<>@,;:\\"/[]?= ')))) + +# RFC 822 quoted-string, without surrounding quotation marks. +_quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format( + re.escape(''.join(_char - {'"', '\\', '\r'})), + re.escape(''.join(_char)) +) + +# RFC 2397 mediatype. +_mediatype_pattern = re.compile(r'{token}/{token}'.format(token=_token)) + +_mediatype_parameter_pattern = re.compile( + r';({token})=(?:({token})|"({quoted})")'.format(token=_token, + quoted=_quoted_string) +) + + +class DataURIDownloadHandler(object): + def __init__(self, settings): + super(DataURIDownloadHandler, self).__init__() + + @defers + def download_request(self, request, spider): + url = request.url + + scheme, url = url.split(':', 1) + if scheme != 'data': + raise ValueError("not a data URI") + + # RFC 3986 section 2.1 allows percent encoding to escape characters + # that would be interpreted as delimiters, implying that actual + # delimiters should not be percent-encoded. + # Decoding before parsing will allow malformed URIs with + # percent-encoded delimiters, but it makes parsing easier and should + # not affect well-formed URIs, as the delimiters used in this URI + # scheme are not allowed, percent-encoded or not, in tokens. + url = unquote(url) + + media_type = "text/plain" + media_type_params = CaselessDict() + + m = _mediatype_pattern.match(url) + if m: + media_type = m.group() + url = url[m.end():] + else: + media_type_params['charset'] = "US-ASCII" + + while True: + m = _mediatype_parameter_pattern.match(url) + if m: + attribute, value, value_quoted = m.groups() + if value_quoted: + value = re.sub(r'\\(.)', '\1', value_quoted) + media_type_params[attribute] = value + url = url[m.end():] + else: + break + + is_base64, data = url.split(',', 1) + if is_base64: + if is_base64 != ";base64": + raise ValueError("invalid data URI") + data = base64.b64decode(data) + + respcls = responsetypes.from_mimetype(media_type) + + resp_kwargs = {} + + if media_type: + media_type = media_type.split('/') + if issubclass(respcls, TextResponse) and media_type[0] == 'text': + resp_kwargs['encoding'] = media_type_params.get('charset') + + return respcls(url=request.url, body=data, **resp_kwargs) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c1683fb3e..c21a1670f 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -6,7 +6,6 @@ try: from unittest import mock except ImportError: import mock -import shutil from twisted.trial import unittest from twisted.protocols.policies import WrappingFactory @@ -20,6 +19,7 @@ from twisted.cred import portal, checkers, credentials from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers +from scrapy.core.downloader.handlers.data import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler @@ -29,6 +29,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.spiders import Spider from scrapy.http import Request from scrapy.http.response.text import TextResponse +from scrapy.responsetypes import responsetypes from scrapy.settings import Settings from scrapy.utils.test import get_crawler, skip_if_no_boto from scrapy.utils.python import to_bytes @@ -828,3 +829,58 @@ class AnonymousFTPTestCase(BaseFTPTestCase): def tearDown(self): shutil.rmtree(self.directory) + + +class DataURITestCase(unittest.TestCase): + + def setUp(self): + self.download_handler = DataURIDownloadHandler(Settings()) + self.download_request = self.download_handler.download_request + self.spider = Spider('foo') + + def test_default_mediatype_encoding(self): + def _test(response): + self.assertEquals(response.text, 'A brief note') + self.assertEquals(type(response), + responsetypes.from_mimetype("text/plain")) + self.assertEquals(response.encoding, "US-ASCII") + + request = Request("data:,A%20brief%20note") + return self.download_request(request, self.spider).addCallback(_test) + + def test_default_mediatype(self): + def _test(response): + self.assertEquals(response.text, u'\u038e\u03a3\u038e') + self.assertEquals(type(response), + responsetypes.from_mimetype("text/plain")) + self.assertEquals(response.encoding, "iso-8859-7") + + request = Request("data:;charset=iso-8859-7,%be%d3%be") + return self.download_request(request, self.spider).addCallback(_test) + + def test_text_charset(self): + def _test(response): + self.assertEquals(response.text, u'\u038e\u03a3\u038e') + self.assertEquals(response.encoding, "iso-8859-7") + + request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") + return self.download_request(request, self.spider).addCallback(_test) + + def test_mediatype_parameters(self): + def _test(response): + self.assertEquals(response.text, u'\u038e\u03a3\u038e') + self.assertEquals(type(response), + responsetypes.from_mimetype("text/plain")) + self.assertEquals(response.encoding, "utf-8") + + request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;' + 'charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22' + ',%CE%8E%CE%A3%CE%8E') + return self.download_request(request, self.spider).addCallback(_test) + + def test_base64(self): + def _test(response): + self.assertEquals(response.text, 'Hello, world.') + + request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D') + return self.download_request(request, self.spider).addCallback(_test)