fixed csviter bug when called with a Response, not TextResponse

scrapy/tests/test_utils_iterators.py

@@ -3,7 +3,7 @@ from twisted.trial import unittest
 
 from scrapy.utils.iterators import csviter, xmliter
 from scrapy.contrib_exp.iterators import xmliter_lxml
-from scrapy.http import XmlResponse, TextResponse
+from scrapy.http import XmlResponse, TextResponse, Response
 from scrapy.tests import get_testdata
 
 
@@ -132,6 +132,18 @@ class UtilsCsvTestCase(unittest.TestCase):
                           {u'id': u'3', u'name': u'multi',   u'value': u'foo\nbar'},
                           {u'id': u'4', u'name': u'empty',   u'value': u''}])
 
+    def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
+        body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t')
+        response = Response(url="http://example.com/", body=body)
+        csv = csviter(response, delimiter='\t')
+
+        self.assertEqual([row for row in csv],
+                         [{u'id': u'1', u'name': u'alpha',   u'value': u'foobar'},
+                          {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
+                          {u'id': u'3', u'name': u'multi',   u'value': u'foo\nbar'},
+                          {u'id': u'4', u'name': u'empty',   u'value': u''}])
+
+
     def test_csviter_headers(self):
         sample = get_testdata('feeds', 'feed-sample3.csv').splitlines()
         headers, body = sample[0].split(','), '\n'.join(sample[1:])

scrapy/utils/iterators.py

@@ -1,7 +1,7 @@
 import re, csv
 from cStringIO import StringIO
 
-from scrapy.http import Response
+from scrapy.http import TextResponse
 from scrapy.selector import XmlXPathSelector
 from scrapy import log
 from scrapy.utils.python import re_rsearch, str_to_unicode
@@ -45,7 +45,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None):
     headers is an iterable that when provided offers the keys
     for the returned dictionaries, if not the first row is used.
     """
-    encoding = obj.encoding if isinstance(obj, Response) else encoding or 'utf-8'
+    encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8'
     def _getrow(csv_r):
         return [str_to_unicode(field, encoding) for field in csv_r.next()]