diff --git a/scrapy/tests/test_utils_iterators.py b/scrapy/tests/test_utils_iterators.py index 0c77ded21..d32d1e658 100644 --- a/scrapy/tests/test_utils_iterators.py +++ b/scrapy/tests/test_utils_iterators.py @@ -3,7 +3,7 @@ from twisted.trial import unittest from scrapy.utils.iterators import csviter, xmliter from scrapy.contrib_exp.iterators import xmliter_lxml -from scrapy.http import XmlResponse, TextResponse +from scrapy.http import XmlResponse, TextResponse, Response from scrapy.tests import get_testdata @@ -132,6 +132,18 @@ class UtilsCsvTestCase(unittest.TestCase): {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'}, {u'id': u'4', u'name': u'empty', u'value': u''}]) + def test_csviter_delimiter_binary_response_assume_utf8_encoding(self): + body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t') + response = Response(url="http://example.com/", body=body) + csv = csviter(response, delimiter='\t') + + self.assertEqual([row for row in csv], + [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, + {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, + {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'}, + {u'id': u'4', u'name': u'empty', u'value': u''}]) + + def test_csviter_headers(self): sample = get_testdata('feeds', 'feed-sample3.csv').splitlines() headers, body = sample[0].split(','), '\n'.join(sample[1:]) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index d528a857d..3b078c25a 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,7 +1,7 @@ import re, csv from cStringIO import StringIO -from scrapy.http import Response +from scrapy.http import TextResponse from scrapy.selector import XmlXPathSelector from scrapy import log from scrapy.utils.python import re_rsearch, str_to_unicode @@ -45,7 +45,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None): headers is an iterable that when provided offers the keys for the returned dictionaries, if not the first row is used. """ - encoding = obj.encoding if isinstance(obj, Response) else encoding or 'utf-8' + encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8' def _getrow(csv_r): return [str_to_unicode(field, encoding) for field in csv_r.next()]