added utf-16 (and other encodings) support to xpathselector_iternodes

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4061

scrapy/trunk/scrapy/tests/test_utils_xml.py

@@ -1,3 +1,4 @@
+import os
 import unittest
 
 from scrapy.utils.xml import xpathselector_iternodes
@@ -26,5 +27,13 @@ class UtilsXmlTestCase(unittest.TestCase):
         self.assertEqual(attrs, 
                          [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])])
 
+    def test_iterator_utf16(self):
+        samplefile = os.path.join(os.path.abspath(os.path.dirname(__file__)), "sample_data", "feeds", "feed-sample3.xml")
+        body = open(samplefile).read()
+        response = Response(domain="example.com", url="http://example.com", headers={"Content-type": "text/xml; encoding=UTF-16"}, body=body)
+        self.assertEqual([x.x("@id").extract() for x in xpathselector_iternodes(response, 'product')],
+                         [['34017532'], ['34017557'], ['34017563'], ['34018057'], ['34018313'], ['34018599']])
+
+
 if __name__ == "__main__":
     unittest.main()   

scrapy/trunk/scrapy/utils/xml.py

@@ -16,15 +16,11 @@ def xpathselector_iternodes(obj, nodename):
     assert isinstance(obj, (Response, basestring)), "obj must be Response or basestring, not %s" % type(obj).__name__
 
     if isinstance(obj, Response):
-        text = obj.body.to_string()
-        enc = obj.body.get_real_encoding()
-    else:
-        text = obj
-        enc = 'utf-8'
-
+        text = obj.body.to_unicode()
+    elif isinstance(obj, str):
+        text = obj.decode('utf-8')
 
     r = re.compile(r"<%s[\s>].*?</%s>" % (nodename, nodename), re.DOTALL)
-
     for match in r.finditer(text):
-        nodetext = match.group().decode(enc)
+        nodetext = match.group()
         yield XmlXPathSelector(text=nodetext).x('/' + nodename)[0]