From 85cd7ea140f3bd52ef3481f0996e9a432816c0e3 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 14 May 2009 20:21:02 -0300 Subject: [PATCH] fixed encoding bug in xmliter (thanks Atamert!), added unittests and updated utils.iterator unittest names for consistency --- scrapy/tests/test_utils_iterators.py | 30 ++++++++++++++++++---------- scrapy/xpath/selector.py | 2 +- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/scrapy/tests/test_utils_iterators.py b/scrapy/tests/test_utils_iterators.py index f9ee20769..3f5b02c06 100644 --- a/scrapy/tests/test_utils_iterators.py +++ b/scrapy/tests/test_utils_iterators.py @@ -5,11 +5,11 @@ import libxml2 from scrapy.utils.iterators import csviter, xmliter from scrapy.http import XmlResponse, TextResponse -class UtilsXmlTestCase(unittest.TestCase): +class UtilsIteratorsTestCase(unittest.TestCase): ### NOTE: Encoding issues have been found with BeautifulSoup for utf-16 files, utf-16 test removed ### # pablo: Tests shouldn't be removed, but commented with proper steps on how # to reproduce the missing functionality - def test_iterator(self): + def test_xmliter(self): body = """\ \ \ @@ -30,13 +30,13 @@ class UtilsXmlTestCase(unittest.TestCase): self.assertEqual(attrs, [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])]) - def test_iterator_text(self): + def test_xmliter_text(self): body = u"""onetwo""" self.assertEqual([x.x("text()").extract() for x in xmliter(body, 'product')], [[u'one'], [u'two']]) - def test_iterator_namespaces(self): + def test_xmliter_namespaces(self): body = """\ @@ -70,7 +70,7 @@ class UtilsXmlTestCase(unittest.TestCase): self.assertEqual(node.x('id/text()').extract(), []) self.assertEqual(node.x('price/text()').extract(), []) - def test_iterator_exception(self): + def test_xmliter_exception(self): body = u"""onetwo""" iter = xmliter(body, 'product') @@ -79,13 +79,21 @@ class UtilsXmlTestCase(unittest.TestCase): self.assertRaises(StopIteration, iter.next) + def test_xmliter_encoding(self): + body = '\n\n Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6\n\n\n' + response = XmlResponse('http://www.example.com', body=body) + self.assertEqual( + xmliter(response, 'item').next().extract(), + u'Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6' + ) + class UtilsCsvTestCase(unittest.TestCase): sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds') sample_feed_path = os.path.join(sample_feeds_dir, 'feed-sample3.csv') sample_feed2_path = os.path.join(sample_feeds_dir, 'feed-sample4.csv') sample_feed3_path = os.path.join(sample_feeds_dir, 'feed-sample5.csv') - def test_iterator_defaults(self): + def test_csviter_defaults(self): body = open(self.sample_feed_path).read() response = TextResponse(url="http://example.com/", body=body) @@ -103,7 +111,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assert_(all((isinstance(k, unicode) for k in result_row.keys()))) self.assert_(all((isinstance(v, unicode) for v in result_row.values()))) - def test_iterator_delimiter(self): + def test_csviter_delimiter(self): body = open(self.sample_feed_path).read().replace(',', '\t') response = TextResponse(url="http://example.com/", body=body) @@ -115,7 +123,7 @@ class UtilsCsvTestCase(unittest.TestCase): {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'}, {u'id': u'4', u'name': u'empty', u'value': u''}]) - def test_iterator_headers(self): + def test_csviter_headers(self): sample = open(self.sample_feed_path).read().splitlines() headers, body = sample[0].split(','), '\n'.join(sample[1:]) @@ -128,7 +136,7 @@ class UtilsCsvTestCase(unittest.TestCase): {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'}, {u'id': u'4', u'name': u'empty', u'value': u''}]) - def test_iterator_falserow(self): + def test_csviter_falserow(self): body = open(self.sample_feed_path).read() body = '\n'.join((body, 'a,b', 'a,b,c,d')) @@ -141,7 +149,7 @@ class UtilsCsvTestCase(unittest.TestCase): {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'}, {u'id': u'4', u'name': u'empty', u'value': u''}]) - def test_iterator_exception(self): + def test_csviter_exception(self): body = open(self.sample_feed_path).read() response = TextResponse(url="http://example.com/", body=body) @@ -153,7 +161,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertRaises(StopIteration, iter.next) - def test_iterator_encoding(self): + def test_csviter_encoding(self): body1 = open(self.sample_feed2_path).read() body2 = open(self.sample_feed3_path).read() diff --git a/scrapy/xpath/selector.py b/scrapy/xpath/selector.py index 8c49c8f97..c0a1a5b5e 100644 --- a/scrapy/xpath/selector.py +++ b/scrapy/xpath/selector.py @@ -34,7 +34,7 @@ class XPathSelector(object): self.doc = Libxml2Document(response, constructor=constructor) self.xmlNode = self.doc.xmlDoc elif text: - response = TextResponse(url=None, body=unicode_to_str(text)) + response = TextResponse(url=None, body=unicode_to_str(text), encoding='utf-8') self.doc = Libxml2Document(response, constructor=constructor) self.xmlNode = self.doc.xmlDoc self.expr = expr