From 9d17d594712fdfd0da9e240c79901f7cb8d1441f Mon Sep 17 00:00:00 2001 From: Gregory Vigo Torres Date: Wed, 29 Jul 2015 20:05:45 +0200 Subject: [PATCH 1/6] from_content_disposition --- scrapy/responsetypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index c212f5706..5b0bc50a5 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -11,7 +11,7 @@ import six from scrapy.http import Response from scrapy.utils.misc import load_object -from scrapy.utils.python import isbinarytext +from scrapy.utils.python import isbinarytext, to_bytes, to_native_str class ResponseTypes(object): @@ -59,7 +59,7 @@ class ResponseTypes(object): def from_content_disposition(self, content_disposition): try: - filename = content_disposition.split(';')[1].split('=')[1] + filename = to_native_str(content_disposition).split(';')[1].split('=')[1] filename = filename.strip('"\'') return self.from_filename(filename) except IndexError: From 36ae635fbe47d68025b475762cece2a6c9e8257f Mon Sep 17 00:00:00 2001 From: Gregory Vigo Torres Date: Wed, 29 Jul 2015 20:07:26 +0200 Subject: [PATCH 2/6] from_content_type --- scrapy/responsetypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 5b0bc50a5..3e3518e79 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -54,7 +54,7 @@ class ResponseTypes(object): header """ if content_encoding: return Response - mimetype = content_type.split(';')[0].strip().lower() + mimetype = to_native_str(content_type).split(';')[0].strip().lower() return self.from_mimetype(mimetype) def from_content_disposition(self, content_disposition): From 42b8988eb2ca7bc31c0a3ab2346b584bed162e18 Mon Sep 17 00:00:00 2001 From: Gregory Vigo Torres Date: Wed, 29 Jul 2015 20:09:06 +0200 Subject: [PATCH 3/6] PY3 port responsetypes from_body --- scrapy/responsetypes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 3e3518e79..b519c56d3 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -90,6 +90,7 @@ class ResponseTypes(object): it's not meant to be used except for special cases where response types cannot be guess using more straightforward methods.""" chunk = body[:5000] + chunk = to_bytes(chunk) if isbinarytext(chunk): return self.from_mimetype('application/octet-stream') elif b"" in chunk.lower(): From 5d75d44f2c1779d7176ea3e122478cb55cd98ee1 Mon Sep 17 00:00:00 2001 From: Gregory Vigo Torres Date: Thu, 30 Jul 2015 13:07:42 +0200 Subject: [PATCH 4/6] removed test_responsetypes from py3ignores --- tests/py3-ignores.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 84bd01c0e..47abd8004 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -30,7 +30,7 @@ tests/test_mail.py tests/test_pipeline_files.py tests/test_pipeline_images.py tests/test_proxy_connect.py -tests/test_responsetypes.py + tests/test_selector_csstranslator.py tests/test_selector_lxmldocument.py tests/test_selector.py From 06b91da943c1355ddc8a1018b67ebd5f35bc833e Mon Sep 17 00:00:00 2001 From: Gregory Vigo Torres Date: Fri, 31 Jul 2015 14:31:11 +0200 Subject: [PATCH 5/6] using bytes for body constant --- tests/test_responsetypes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 1d78d0976..2374d518f 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -43,10 +43,10 @@ class ResponseTypesTest(unittest.TestCase): def test_from_body(self): mappings = [ - ('\x03\x02\xdf\xdd\x23', Response), - ('Some plain text\ndata with tabs\t and null bytes\0', TextResponse), - ('Hello', HtmlResponse), - ('Hello', HtmlResponse), + (b' Date: Fri, 31 Jul 2015 23:48:49 +0500 Subject: [PATCH 6/6] small ResponseTypes cleanup --- scrapy/responsetypes.py | 19 +++++++++++-------- tests/py3-ignores.txt | 1 - 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index b519c56d3..4880cc7b9 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -1,9 +1,8 @@ """ This module implements a class which returns the appropriate Response class based on different criteria. - """ - +from __future__ import absolute_import from mimetypes import MimeTypes from pkgutil import get_data from io import StringIO @@ -13,6 +12,7 @@ from scrapy.http import Response from scrapy.utils.misc import load_object from scrapy.utils.python import isbinarytext, to_bytes, to_native_str + class ResponseTypes(object): CLASSES = { @@ -69,11 +69,13 @@ class ResponseTypes(object): """Return the most appropriate Response class by looking at the HTTP headers""" cls = Response - if 'Content-Type' in headers: - cls = self.from_content_type(headers['Content-type'], \ - headers.get('Content-Encoding')) - if cls is Response and 'Content-Disposition' in headers: - cls = self.from_content_disposition(headers['Content-Disposition']) + if b'Content-Type' in headers: + cls = self.from_content_type( + content_type=headers[b'Content-type'], + content_encoding=headers.get(b'Content-Encoding') + ) + if cls is Response and b'Content-Disposition' in headers: + cls = self.from_content_disposition(headers[b'Content-Disposition']) return cls def from_filename(self, filename): @@ -101,7 +103,8 @@ class ResponseTypes(object): return self.from_mimetype('text') def from_args(self, headers=None, url=None, filename=None, body=None): - """Guess the most appropriate Response class based on the given arguments""" + """Guess the most appropriate Response class based on + the given arguments.""" cls = Response if headers is not None: cls = self.from_headers(headers) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 47abd8004..c51bc5981 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -30,7 +30,6 @@ tests/test_mail.py tests/test_pipeline_files.py tests/test_pipeline_images.py tests/test_proxy_connect.py - tests/test_selector_csstranslator.py tests/test_selector_lxmldocument.py tests/test_selector.py