diff --git a/scrapy/trunk/scrapy/tests/test_utils_xml.py b/scrapy/trunk/scrapy/tests/test_utils_xml.py index e2bf4b4ef..d12bffd99 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_xml.py +++ b/scrapy/trunk/scrapy/tests/test_utils_xml.py @@ -1,3 +1,4 @@ +import os import unittest from scrapy.utils.xml import xpathselector_iternodes @@ -26,5 +27,13 @@ class UtilsXmlTestCase(unittest.TestCase): self.assertEqual(attrs, [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])]) + def test_iterator_utf16(self): + samplefile = os.path.join(os.path.abspath(os.path.dirname(__file__)), "sample_data", "feeds", "feed-sample3.xml") + body = open(samplefile).read() + response = Response(domain="example.com", url="http://example.com", headers={"Content-type": "text/xml; encoding=UTF-16"}, body=body) + self.assertEqual([x.x("@id").extract() for x in xpathselector_iternodes(response, 'product')], + [['34017532'], ['34017557'], ['34017563'], ['34018057'], ['34018313'], ['34018599']]) + + if __name__ == "__main__": unittest.main() diff --git a/scrapy/trunk/scrapy/utils/xml.py b/scrapy/trunk/scrapy/utils/xml.py index 50c5c66fb..2d43c8b3a 100644 --- a/scrapy/trunk/scrapy/utils/xml.py +++ b/scrapy/trunk/scrapy/utils/xml.py @@ -16,15 +16,11 @@ def xpathselector_iternodes(obj, nodename): assert isinstance(obj, (Response, basestring)), "obj must be Response or basestring, not %s" % type(obj).__name__ if isinstance(obj, Response): - text = obj.body.to_string() - enc = obj.body.get_real_encoding() - else: - text = obj - enc = 'utf-8' - + text = obj.body.to_unicode() + elif isinstance(obj, str): + text = obj.decode('utf-8') r = re.compile(r"<%s[\s>].*?" % (nodename, nodename), re.DOTALL) - for match in r.finditer(text): - nodetext = match.group().decode(enc) + nodetext = match.group() yield XmlXPathSelector(text=nodetext).x('/' + nodename)[0]