From d4872940dbb450fcd0fa688766af0aa6dfccc861 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 6 Jan 2016 21:21:21 +0100 Subject: [PATCH 1/2] PY3: port utils/iterators --- scrapy/utils/iterators.py | 14 ++++++++++---- tests/py3-ignores.txt | 1 - tests/test_utils_iterators.py | 21 +++++++++++---------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c0d93f7a9..ed286f5c5 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -8,6 +8,8 @@ except ImportError: from io import BytesIO import six +if six.PY3: + from io import StringIO from scrapy.http import TextResponse, Response from scrapy.selector import Selector @@ -65,7 +67,7 @@ class _StreamReader(object): self._text, self.encoding = obj.body, obj.encoding else: self._text, self.encoding = obj, 'utf-8' - self._is_unicode = isinstance(self._text, unicode) + self._is_unicode = isinstance(self._text, six.text_type) def read(self, n=65535): self.read = self._read_unicode if self._is_unicode else self._read_string @@ -94,7 +96,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): headers is an iterable that when provided offers the keys for the returned dictionaries, if not the first row is used. - + quotechar is the character used to enclosure fields on the given obj. """ @@ -102,7 +104,11 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _getrow(csv_r): return [to_unicode(field, encoding) for field in next(csv_r)] - lines = BytesIO(_body_or_str(obj, unicode=False)) + # Python 3 csv reader input object needs to return strings + if six.PY3: + lines = StringIO(_body_or_str(obj, unicode=True)) + else: + lines = BytesIO(_body_or_str(obj, unicode=False)) kwargs = {} if delimiter: kwargs["delimiter"] = delimiter @@ -125,7 +131,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, six.string_types)), \ + assert isinstance(obj, (Response, six.string_types, six.binary_type)), \ "obj must be Response or basestring, not %s" % type(obj).__name__ if isinstance(obj, Response): if not unicode: diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 55ed75c92..015578e1e 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -16,7 +16,6 @@ tests/test_pipeline_files.py tests/test_pipeline_images.py tests/test_proxy_connect.py tests/test_spidermiddleware_httperror.py -tests/test_utils_iterators.py tests/test_utils_template.py tests/test_webclient.py diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 590c53302..d42ed2c91 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,4 +1,5 @@ import os +import six from twisted.trial import unittest from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml @@ -99,7 +100,7 @@ class XmliterTestCase(unittest.TestCase): body = b'\n\n Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6\n\n\n' response = XmlResponse('http://www.example.com', body=body) self.assertEqual( - self.xmliter(response, 'item').next().extract(), + next(self.xmliter(response, 'item')).extract(), u'Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6' ) @@ -189,11 +190,11 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assert_(all((isinstance(k, unicode) for k in result_row.keys()))) - self.assert_(all((isinstance(v, unicode) for v in result_row.values()))) + self.assert_(all((isinstance(k, six.text_type) for k in result_row.keys()))) + self.assert_(all((isinstance(v, six.text_type) for v in result_row.values()))) def test_csviter_delimiter(self): - body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t') + body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') response = TextResponse(url="http://example.com/", body=body) csv = csviter(response, delimiter='\t') @@ -205,8 +206,8 @@ class UtilsCsvTestCase(unittest.TestCase): def test_csviter_quotechar(self): body1 = get_testdata('feeds', 'feed-sample6.csv') - body2 = get_testdata('feeds', 'feed-sample6.csv').replace(",", '|') - + body2 = get_testdata('feeds', 'feed-sample6.csv').replace(b',', b'|') + response1 = TextResponse(url="http://example.com/", body=body1) csv1 = csviter(response1, quotechar="'") @@ -237,7 +238,7 @@ class UtilsCsvTestCase(unittest.TestCase): {u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}]) def test_csviter_delimiter_binary_response_assume_utf8_encoding(self): - body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t') + body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') response = Response(url="http://example.com/", body=body) csv = csviter(response, delimiter='\t') @@ -249,10 +250,10 @@ class UtilsCsvTestCase(unittest.TestCase): def test_csviter_headers(self): sample = get_testdata('feeds', 'feed-sample3.csv').splitlines() - headers, body = sample[0].split(','), '\n'.join(sample[1:]) + headers, body = sample[0].split(b','), b'\n'.join(sample[1:]) response = TextResponse(url="http://example.com/", body=body) - csv = csviter(response, headers=headers) + csv = csviter(response, headers=[h.decode('utf-8') for h in headers]) self.assertEqual([row for row in csv], [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, @@ -262,7 +263,7 @@ class UtilsCsvTestCase(unittest.TestCase): def test_csviter_falserow(self): body = get_testdata('feeds', 'feed-sample3.csv') - body = '\n'.join((body, 'a,b', 'a,b,c,d')) + body = b'\n'.join((body, b'a,b', b'a,b,c,d')) response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) From d7d4ef67a697243143df969e32b8ed956394f4fb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 12 Jan 2016 11:08:49 +0100 Subject: [PATCH 2/2] Changes following comments --- scrapy/utils/iterators.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index ed286f5c5..ce59c9719 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,15 +1,12 @@ import re import csv import logging - try: from cStringIO import StringIO as BytesIO except ImportError: from io import BytesIO - +from io import StringIO import six -if six.PY3: - from io import StringIO from scrapy.http import TextResponse, Response from scrapy.selector import Selector @@ -131,7 +128,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, six.string_types, six.binary_type)), \ + assert isinstance(obj, (Response, six.string_types, bytes)), \ "obj must be Response or basestring, not %s" % type(obj).__name__ if isinstance(obj, Response): if not unicode: