diff --git a/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample4.csv b/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample4.csv new file mode 100644 index 000000000..fc9980bef --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample4.csv @@ -0,0 +1,3 @@ +id,name,value +1,latin1,test +2,something,ραισ diff --git a/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample5.csv b/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample5.csv new file mode 100644 index 000000000..9cff1dfa4 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample5.csv @@ -0,0 +1,3 @@ +id,name,value +1,cp852,test +2,something,ΘΚΚΚΝΝ» diff --git a/scrapy/trunk/scrapy/tests/test_utils_iterators.py b/scrapy/trunk/scrapy/tests/test_utils_iterators.py index 80343ba9b..f9ee20769 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_iterators.py +++ b/scrapy/trunk/scrapy/tests/test_utils_iterators.py @@ -80,7 +80,10 @@ class UtilsXmlTestCase(unittest.TestCase): self.assertRaises(StopIteration, iter.next) class UtilsCsvTestCase(unittest.TestCase): - sample_feed_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds', 'feed-sample3.csv') + sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds') + sample_feed_path = os.path.join(sample_feeds_dir, 'feed-sample3.csv') + sample_feed2_path = os.path.join(sample_feeds_dir, 'feed-sample4.csv') + sample_feed3_path = os.path.join(sample_feeds_dir, 'feed-sample5.csv') def test_iterator_defaults(self): body = open(self.sample_feed_path).read() @@ -150,5 +153,22 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertRaises(StopIteration, iter.next) + def test_iterator_encoding(self): + body1 = open(self.sample_feed2_path).read() + body2 = open(self.sample_feed3_path).read() + + response = TextResponse(url="http://example.com/", body=body1, encoding='latin1') + csv = csviter(response) + self.assertEqual([row for row in csv], + [{u'id': u'1', u'name': u'latin1', u'value': u'test'}, + {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'}]) + + response = TextResponse(url="http://example.com/", body=body2, encoding='cp852') + csv = csviter(response) + self.assertEqual([row for row in csv], + [{u'id': u'1', u'name': u'cp852', u'value': u'test'}, + {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}]) + + if __name__ == "__main__": unittest.main()