diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index bec985062..c215a0bdd 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -1,12 +1,11 @@
import re
import csv
import logging
-
try:
from cStringIO import StringIO as BytesIO
except ImportError:
from io import BytesIO
-
+from io import StringIO
import six
from scrapy.http import TextResponse, Response
@@ -65,7 +64,7 @@ class _StreamReader(object):
self._text, self.encoding = obj.body, obj.encoding
else:
self._text, self.encoding = obj, 'utf-8'
- self._is_unicode = isinstance(self._text, unicode)
+ self._is_unicode = isinstance(self._text, six.text_type)
def read(self, n=65535):
self.read = self._read_unicode if self._is_unicode else self._read_string
@@ -102,7 +101,11 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
def _getrow(csv_r):
return [to_unicode(field, encoding) for field in next(csv_r)]
- lines = BytesIO(_body_or_str(obj, unicode=False))
+ # Python 3 csv reader input object needs to return strings
+ if six.PY3:
+ lines = StringIO(_body_or_str(obj, unicode=True))
+ else:
+ lines = BytesIO(_body_or_str(obj, unicode=False))
kwargs = {}
if delimiter: kwargs["delimiter"] = delimiter
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 55ed75c92..015578e1e 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -16,7 +16,6 @@ tests/test_pipeline_files.py
tests/test_pipeline_images.py
tests/test_proxy_connect.py
tests/test_spidermiddleware_httperror.py
-tests/test_utils_iterators.py
tests/test_utils_template.py
tests/test_webclient.py
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 8dceed7cc..de103fea5 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
import os
+import six
from twisted.trial import unittest
from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
@@ -148,7 +149,7 @@ class XmliterTestCase(unittest.TestCase):
body = b'\n\n - Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6
\n\n\n'
response = XmlResponse('http://www.example.com', body=body)
self.assertEqual(
- self.xmliter(response, 'item').next().extract(),
+ next(self.xmliter(response, 'item')).extract(),
u'- Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6
'
)
@@ -238,11 +239,11 @@ class UtilsCsvTestCase(unittest.TestCase):
# explicit type check cuz' we no like stinkin' autocasting! yarrr
for result_row in result:
- self.assert_(all((isinstance(k, unicode) for k in result_row.keys())))
- self.assert_(all((isinstance(v, unicode) for v in result_row.values())))
+ self.assert_(all((isinstance(k, six.text_type) for k in result_row.keys())))
+ self.assert_(all((isinstance(v, six.text_type) for v in result_row.values())))
def test_csviter_delimiter(self):
- body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t')
+ body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
response = TextResponse(url="http://example.com/", body=body)
csv = csviter(response, delimiter='\t')
@@ -254,7 +255,7 @@ class UtilsCsvTestCase(unittest.TestCase):
def test_csviter_quotechar(self):
body1 = get_testdata('feeds', 'feed-sample6.csv')
- body2 = get_testdata('feeds', 'feed-sample6.csv').replace(",", '|')
+ body2 = get_testdata('feeds', 'feed-sample6.csv').replace(b',', b'|')
response1 = TextResponse(url="http://example.com/", body=body1)
csv1 = csviter(response1, quotechar="'")
@@ -286,7 +287,7 @@ class UtilsCsvTestCase(unittest.TestCase):
{u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}])
def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
- body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t')
+ body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
response = Response(url="http://example.com/", body=body)
csv = csviter(response, delimiter='\t')
@@ -298,10 +299,10 @@ class UtilsCsvTestCase(unittest.TestCase):
def test_csviter_headers(self):
sample = get_testdata('feeds', 'feed-sample3.csv').splitlines()
- headers, body = sample[0].split(','), '\n'.join(sample[1:])
+ headers, body = sample[0].split(b','), b'\n'.join(sample[1:])
response = TextResponse(url="http://example.com/", body=body)
- csv = csviter(response, headers=headers)
+ csv = csviter(response, headers=[h.decode('utf-8') for h in headers])
self.assertEqual([row for row in csv],
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
@@ -311,7 +312,7 @@ class UtilsCsvTestCase(unittest.TestCase):
def test_csviter_falserow(self):
body = get_testdata('feeds', 'feed-sample3.csv')
- body = '\n'.join((body, 'a,b', 'a,b,c,d'))
+ body = b'\n'.join((body, b'a,b', b'a,b,c,d'))
response = TextResponse(url="http://example.com/", body=body)
csv = csviter(response)