Use explicit Unicode and bytes for XML body in tests

scrapy/utils/iterators.py

@@ -48,7 +48,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
     iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
     selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename)
     for _, node in iterable:
-        nodetext = etree.tostring(node, encoding='unicode')
+        nodetext = etree.tostring(node, encoding=six.text_type)
         node.clear()
         xs = Selector(text=nodetext, type='xml')
         if namespace:
@@ -128,8 +128,11 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
 
 def _body_or_str(obj, unicode=True):
-    assert isinstance(obj, (Response, six.string_types, bytes)), \
-        "obj must be Response or basestring, not %s" % type(obj).__name__
+    expected_types = (Response, six.text_type, six.binary_type)
+    assert isinstance(obj, expected_types), \
+        "obj must be %s, not %s" % (
+            " or ".join(t.__name__ for t in expected_types),
+            type(obj).__name__)
     if isinstance(obj, Response):
         if not unicode:
             return obj.body

tests/test_utils_iterators.py

@@ -49,7 +49,7 @@ class XmliterTestCase(unittest.TestCase):
 
     def test_xmliter_unicode(self):
         # example taken from https://github.com/scrapy/scrapy/issues/1665
-        body = """<?xml version="1.0" encoding="UTF-8"?>
+        body = u"""<?xml version="1.0" encoding="UTF-8"?>
             <þingflokkar>
                <þingflokkur id="26">
                   <heiti />
@@ -84,7 +84,22 @@ class XmliterTestCase(unittest.TestCase):
                   </tímabil>
                </þingflokkur>
             </þingflokkar>"""
-        response = XmlResponse(url="http://example.com", body=body)
+
+        # with bytes
+        response = XmlResponse(url="http://example.com", body=body.encode('utf-8'))
+        attrs = []
+        for x in self.xmliter(response, u'þingflokkur'):
+            attrs.append((x.xpath('@id').extract(),
+                          x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(),
+                          x.xpath(u'./tímabil/fyrstaþing/text()').extract()))
+
+        self.assertEqual(attrs,
+                         [([u'26'], [u'-'], [u'80']),
+                          ([u'21'], [u'Ab'], [u'76']),
+                          ([u'27'], [u'A'], [u'27'])])
+
+        # Unicode body needs encoding information
+        response = XmlResponse(url="http://example.com", body=body, encoding='utf-8')
         attrs = []
         for x in self.xmliter(response, u'þingflokkur'):
             attrs.append((x.xpath('@id').extract(),