From d60642e1755b004e46b87e2011c1873f25895bb8 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Fri, 12 Aug 2016 00:45:42 -0300 Subject: [PATCH 1/7] data URI download handler. --- scrapy/core/downloader/handlers/data.py | 93 +++++++++++++++++++++++++ tests/test_downloader_handlers.py | 58 ++++++++++++++- 2 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 scrapy/core/downloader/handlers/data.py diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.py new file mode 100644 index 000000000..637a6f9c0 --- /dev/null +++ b/scrapy/core/downloader/handlers/data.py @@ -0,0 +1,93 @@ +import base64 +import re +from six.moves.urllib.parse import unquote + +from scrapy.http import TextResponse +from scrapy.responsetypes import responsetypes +from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.decorators import defers + + +# ASCII characters. +_char = set(map(chr, range(127))) + +# RFC 2045 token. +_token = r'[{}]+'.format(re.escape(''.join(_char - + # Control characters. + set(map(chr, range(0, 32))) - + # tspecials and space. + set('()<>@,;:\\"/[]?= ')))) + +# RFC 822 quoted-string, without surrounding quotation marks. +_quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format( + re.escape(''.join(_char - {'"', '\\', '\r'})), + re.escape(''.join(_char)) +) + +# RFC 2397 mediatype. +_mediatype_pattern = re.compile(r'{token}/{token}'.format(token=_token)) + +_mediatype_parameter_pattern = re.compile( + r';({token})=(?:({token})|"({quoted})")'.format(token=_token, + quoted=_quoted_string) +) + + +class DataURIDownloadHandler(object): + def __init__(self, settings): + super(DataURIDownloadHandler, self).__init__() + + @defers + def download_request(self, request, spider): + url = request.url + + scheme, url = url.split(':', 1) + if scheme != 'data': + raise ValueError("not a data URI") + + # RFC 3986 section 2.1 allows percent encoding to escape characters + # that would be interpreted as delimiters, implying that actual + # delimiters should not be percent-encoded. + # Decoding before parsing will allow malformed URIs with + # percent-encoded delimiters, but it makes parsing easier and should + # not affect well-formed URIs, as the delimiters used in this URI + # scheme are not allowed, percent-encoded or not, in tokens. + url = unquote(url) + + media_type = "text/plain" + media_type_params = CaselessDict() + + m = _mediatype_pattern.match(url) + if m: + media_type = m.group() + url = url[m.end():] + else: + media_type_params['charset'] = "US-ASCII" + + while True: + m = _mediatype_parameter_pattern.match(url) + if m: + attribute, value, value_quoted = m.groups() + if value_quoted: + value = re.sub(r'\\(.)', '\1', value_quoted) + media_type_params[attribute] = value + url = url[m.end():] + else: + break + + is_base64, data = url.split(',', 1) + if is_base64: + if is_base64 != ";base64": + raise ValueError("invalid data URI") + data = base64.b64decode(data) + + respcls = responsetypes.from_mimetype(media_type) + + resp_kwargs = {} + + if media_type: + media_type = media_type.split('/') + if issubclass(respcls, TextResponse) and media_type[0] == 'text': + resp_kwargs['encoding'] = media_type_params.get('charset') + + return respcls(url=request.url, body=data, **resp_kwargs) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c1683fb3e..c21a1670f 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -6,7 +6,6 @@ try: from unittest import mock except ImportError: import mock -import shutil from twisted.trial import unittest from twisted.protocols.policies import WrappingFactory @@ -20,6 +19,7 @@ from twisted.cred import portal, checkers, credentials from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers +from scrapy.core.downloader.handlers.data import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler @@ -29,6 +29,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.spiders import Spider from scrapy.http import Request from scrapy.http.response.text import TextResponse +from scrapy.responsetypes import responsetypes from scrapy.settings import Settings from scrapy.utils.test import get_crawler, skip_if_no_boto from scrapy.utils.python import to_bytes @@ -828,3 +829,58 @@ class AnonymousFTPTestCase(BaseFTPTestCase): def tearDown(self): shutil.rmtree(self.directory) + + +class DataURITestCase(unittest.TestCase): + + def setUp(self): + self.download_handler = DataURIDownloadHandler(Settings()) + self.download_request = self.download_handler.download_request + self.spider = Spider('foo') + + def test_default_mediatype_encoding(self): + def _test(response): + self.assertEquals(response.text, 'A brief note') + self.assertEquals(type(response), + responsetypes.from_mimetype("text/plain")) + self.assertEquals(response.encoding, "US-ASCII") + + request = Request("data:,A%20brief%20note") + return self.download_request(request, self.spider).addCallback(_test) + + def test_default_mediatype(self): + def _test(response): + self.assertEquals(response.text, u'\u038e\u03a3\u038e') + self.assertEquals(type(response), + responsetypes.from_mimetype("text/plain")) + self.assertEquals(response.encoding, "iso-8859-7") + + request = Request("data:;charset=iso-8859-7,%be%d3%be") + return self.download_request(request, self.spider).addCallback(_test) + + def test_text_charset(self): + def _test(response): + self.assertEquals(response.text, u'\u038e\u03a3\u038e') + self.assertEquals(response.encoding, "iso-8859-7") + + request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") + return self.download_request(request, self.spider).addCallback(_test) + + def test_mediatype_parameters(self): + def _test(response): + self.assertEquals(response.text, u'\u038e\u03a3\u038e') + self.assertEquals(type(response), + responsetypes.from_mimetype("text/plain")) + self.assertEquals(response.encoding, "utf-8") + + request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;' + 'charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22' + ',%CE%8E%CE%A3%CE%8E') + return self.download_request(request, self.spider).addCallback(_test) + + def test_base64(self): + def _test(response): + self.assertEquals(response.text, 'Hello, world.') + + request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D') + return self.download_request(request, self.spider).addCallback(_test) From 7e9f2c31d78dcc798a281e5ba6ddf91714dc9cc7 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Fri, 12 Aug 2016 02:09:35 -0300 Subject: [PATCH 2/7] Ensure bytes objects when needed in data URI downloader. --- scrapy/core/downloader/handlers/data.py | 29 ++++++++++++++++++------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.py index 637a6f9c0..30c39865f 100644 --- a/scrapy/core/downloader/handlers/data.py +++ b/scrapy/core/downloader/handlers/data.py @@ -1,6 +1,12 @@ import base64 import re -from six.moves.urllib.parse import unquote + +import six + +if six.PY2: + from urllib import unquote +else: + from urllib.parse import unquote_to_bytes as unquote from scrapy.http import TextResponse from scrapy.responsetypes import responsetypes @@ -24,12 +30,19 @@ _quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format( re.escape(''.join(_char)) ) +# Encode the regular expression strings to make them into bytes, as Python 3 +# bytes have no format() method, but bytes must be passed to re.compile() in +# order to make a pattern object that can be used to match on bytes. + # RFC 2397 mediatype. -_mediatype_pattern = re.compile(r'{token}/{token}'.format(token=_token)) +_mediatype_pattern = re.compile( + r'{token}/{token}'.format(token=_token).encode() +) _mediatype_parameter_pattern = re.compile( r';({token})=(?:({token})|"({quoted})")'.format(token=_token, - quoted=_quoted_string) + quoted=_quoted_string + ).encode() ) @@ -59,7 +72,7 @@ class DataURIDownloadHandler(object): m = _mediatype_pattern.match(url) if m: - media_type = m.group() + media_type = m.group().decode() url = url[m.end():] else: media_type_params['charset'] = "US-ASCII" @@ -69,15 +82,15 @@ class DataURIDownloadHandler(object): if m: attribute, value, value_quoted = m.groups() if value_quoted: - value = re.sub(r'\\(.)', '\1', value_quoted) - media_type_params[attribute] = value + value = re.sub(br'\\(.)', r'\1', value_quoted) + media_type_params[attribute.decode()] = value.decode() url = url[m.end():] else: break - is_base64, data = url.split(',', 1) + is_base64, data = url.split(b',', 1) if is_base64: - if is_base64 != ";base64": + if is_base64 != b";base64": raise ValueError("invalid data URI") data = base64.b64decode(data) From 3397d27574d03a222b6a790004fcaa1a03711d75 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 8 Feb 2017 12:32:00 -0200 Subject: [PATCH 3/7] Test for binary body content from data URI downloader. --- tests/test_downloader_handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c21a1670f..cfcdcd8f8 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -861,6 +861,7 @@ class DataURITestCase(unittest.TestCase): def test_text_charset(self): def _test(response): self.assertEquals(response.text, u'\u038e\u03a3\u038e') + self.assertEquals(response.body, b'\xbe\xd3\xbe') self.assertEquals(response.encoding, "iso-8859-7") request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") From c847e7d4d00647a953a3c5458d198439c30b87f7 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 8 Feb 2017 12:38:49 -0200 Subject: [PATCH 4/7] Use w3lib data URI parser. --- scrapy/core/downloader/handlers/data.py | 99 ++----------------------- 1 file changed, 8 insertions(+), 91 deletions(-) diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.py index 30c39865f..d102f2b73 100644 --- a/scrapy/core/downloader/handlers/data.py +++ b/scrapy/core/downloader/handlers/data.py @@ -1,106 +1,23 @@ -import base64 -import re - -import six - -if six.PY2: - from urllib import unquote -else: - from urllib.parse import unquote_to_bytes as unquote +from w3lib.url import parse_data_uri from scrapy.http import TextResponse from scrapy.responsetypes import responsetypes -from scrapy.utils.datatypes import CaselessDict from scrapy.utils.decorators import defers -# ASCII characters. -_char = set(map(chr, range(127))) - -# RFC 2045 token. -_token = r'[{}]+'.format(re.escape(''.join(_char - - # Control characters. - set(map(chr, range(0, 32))) - - # tspecials and space. - set('()<>@,;:\\"/[]?= ')))) - -# RFC 822 quoted-string, without surrounding quotation marks. -_quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format( - re.escape(''.join(_char - {'"', '\\', '\r'})), - re.escape(''.join(_char)) -) - -# Encode the regular expression strings to make them into bytes, as Python 3 -# bytes have no format() method, but bytes must be passed to re.compile() in -# order to make a pattern object that can be used to match on bytes. - -# RFC 2397 mediatype. -_mediatype_pattern = re.compile( - r'{token}/{token}'.format(token=_token).encode() -) - -_mediatype_parameter_pattern = re.compile( - r';({token})=(?:({token})|"({quoted})")'.format(token=_token, - quoted=_quoted_string - ).encode() -) - - class DataURIDownloadHandler(object): def __init__(self, settings): super(DataURIDownloadHandler, self).__init__() @defers def download_request(self, request, spider): - url = request.url - - scheme, url = url.split(':', 1) - if scheme != 'data': - raise ValueError("not a data URI") - - # RFC 3986 section 2.1 allows percent encoding to escape characters - # that would be interpreted as delimiters, implying that actual - # delimiters should not be percent-encoded. - # Decoding before parsing will allow malformed URIs with - # percent-encoded delimiters, but it makes parsing easier and should - # not affect well-formed URIs, as the delimiters used in this URI - # scheme are not allowed, percent-encoded or not, in tokens. - url = unquote(url) - - media_type = "text/plain" - media_type_params = CaselessDict() - - m = _mediatype_pattern.match(url) - if m: - media_type = m.group().decode() - url = url[m.end():] - else: - media_type_params['charset'] = "US-ASCII" - - while True: - m = _mediatype_parameter_pattern.match(url) - if m: - attribute, value, value_quoted = m.groups() - if value_quoted: - value = re.sub(br'\\(.)', r'\1', value_quoted) - media_type_params[attribute.decode()] = value.decode() - url = url[m.end():] - else: - break - - is_base64, data = url.split(b',', 1) - if is_base64: - if is_base64 != b";base64": - raise ValueError("invalid data URI") - data = base64.b64decode(data) - - respcls = responsetypes.from_mimetype(media_type) + uri = parse_data_uri(request.url) + respcls = responsetypes.from_mimetype(uri.media_type) resp_kwargs = {} + if (issubclass(respcls, TextResponse) and + uri.media_type.split('/')[0] == 'text'): + charset = uri.media_type_parameters.get('charset') + resp_kwargs['encoding'] = charset - if media_type: - media_type = media_type.split('/') - if issubclass(respcls, TextResponse) and media_type[0] == 'text': - resp_kwargs['encoding'] = media_type_params.get('charset') - - return respcls(url=request.url, body=data, **resp_kwargs) + return respcls(url=request.url, body=uri.data, **resp_kwargs) From 121a668a479c3b681581a36c7da090518fbc2120 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Sun, 12 Feb 2017 11:23:21 -0200 Subject: [PATCH 5/7] Rename data URI downloader module. --- scrapy/core/downloader/handlers/{data.py => datauri.py} | 0 tests/test_downloader_handlers.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename scrapy/core/downloader/handlers/{data.py => datauri.py} (100%) diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/datauri.py similarity index 100% rename from scrapy/core/downloader/handlers/data.py rename to scrapy/core/downloader/handlers/datauri.py diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index cfcdcd8f8..b27245a36 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -19,7 +19,7 @@ from twisted.cred import portal, checkers, credentials from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers -from scrapy.core.downloader.handlers.data import DataURIDownloadHandler +from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler From 3139f4a5f700a100b6ab6890677dac808b01835f Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Sun, 12 Feb 2017 11:23:56 -0200 Subject: [PATCH 6/7] Add data URI download handler to settings. --- scrapy/settings/default_settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index e0e39120c..2251d3db5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -67,6 +67,7 @@ DOWNLOAD_DELAY = 0 DOWNLOAD_HANDLERS = {} DOWNLOAD_HANDLERS_BASE = { + 'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler', 'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler', 'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', From b50d0370f4ebdb485f4625ef6d71398ac8538c79 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Thu, 2 Mar 2017 14:46:33 -0300 Subject: [PATCH 7/7] Test response attributes in data URI download handler. --- tests/test_downloader_handlers.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index b27245a36..74203dbfe 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -838,6 +838,16 @@ class DataURITestCase(unittest.TestCase): self.download_request = self.download_handler.download_request self.spider = Spider('foo') + def test_response_attrs(self): + uri = "data:,A%20brief%20note" + + def _test(response): + self.assertEquals(response.url, uri) + self.assertFalse(response.headers) + + request = Request(uri) + return self.download_request(request, self.spider).addCallback(_test) + def test_default_mediatype_encoding(self): def _test(response): self.assertEquals(response.text, 'A brief note')