do specialized xmliter public functions and add tests

scrapy/tests/test_utils_iterators.py

@@ -1,15 +1,16 @@
 import os
-import unittest
 import libxml2
+from twisted.trial import unittest
 
-from scrapy.utils.iterators import csviter, xmliter
+from scrapy.utils.iterators import csviter, xmliter, xmliter_lxml, xmliter_regex
 from scrapy.http import XmlResponse, TextResponse
 from scrapy.tests import get_testdata
 
-class UtilsIteratorsTestCase(unittest.TestCase):
-    ### NOTE: Encoding issues have been found with BeautifulSoup for utf-16 files, utf-16 test removed ###
-    # pablo: Tests shouldn't be removed, but commented with proper steps on how
-    # to reproduce the missing functionality
+
+class XmliterTestCase(unittest.TestCase):
+
+    xmliter = staticmethod(xmliter)
+
     def test_xmliter(self):
         body = """<?xml version="1.0" encoding="UTF-8"?>\
             <products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="someschmea.xsd">\
@@ -25,7 +26,7 @@ class UtilsIteratorsTestCase(unittest.TestCase):
 
         response = XmlResponse(url="http://example.com", body=body)
         attrs = []
-        for x in xmliter(response, 'product'):
+        for x in self.xmliter(response, 'product'):
             attrs.append((x.select("@id").extract(), x.select("name/text()").extract(), x.select("./type/text()").extract()))
 
         self.assertEqual(attrs, 
@@ -34,7 +35,7 @@ class UtilsIteratorsTestCase(unittest.TestCase):
     def test_xmliter_text(self):
         body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
         
-        self.assertEqual([x.select("text()").extract() for x in xmliter(body, 'product')],
+        self.assertEqual([x.select("text()").extract() for x in self.xmliter(body, 'product')],
                          [[u'one'], [u'two']])
 
     def test_xmliter_namespaces(self):
@@ -57,7 +58,7 @@ class UtilsIteratorsTestCase(unittest.TestCase):
             </rss>
         """
         response = XmlResponse(url='http://mydummycompany.com', body=body)
-        my_iter = xmliter(response, 'item')
+        my_iter = self.xmliter(response, 'item')
 
         node = my_iter.next()
         node.register_namespace('g', 'http://base.google.com/ns/1.0')
@@ -74,7 +75,7 @@ class UtilsIteratorsTestCase(unittest.TestCase):
     def test_xmliter_exception(self):
         body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
         
-        iter = xmliter(body, 'product')
+        iter = self.xmliter(body, 'product')
         iter.next()
         iter.next()
 
@@ -84,10 +85,23 @@ class UtilsIteratorsTestCase(unittest.TestCase):
         body = '<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n    <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
         response = XmlResponse('http://www.example.com', body=body)
         self.assertEqual(
-            xmliter(response, 'item').next().extract(),
+            self.xmliter(response, 'item').next().extract(),
             u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
         )
 
+
+class RegexXmliterTestCase(XmliterTestCase):
+    xmliter = staticmethod(xmliter_regex)
+
+
+class LxmlXmliterTestCase(XmliterTestCase):
+    xmliter = staticmethod(xmliter_lxml)
+    try:
+        import lxml
+    except ImportError:
+        skip = True
+
+
 class UtilsCsvTestCase(unittest.TestCase):
     sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')
     sample_feed_path = os.path.join(sample_feeds_dir, 'feed-sample3.csv')

scrapy/utils/iterators.py

@@ -7,7 +7,7 @@ from scrapy.utils.python import re_rsearch, str_to_unicode
 from scrapy.utils.response import body_or_str
 
 
-def _xmliter_regex(obj, nodename):
+def xmliter_regex(obj, nodename):
     """Return a iterator of XPathSelector's over all nodes of a XML document,
        given tha name of the node to iterate. Useful for parsing XML feeds.
 
@@ -31,7 +31,7 @@ def _xmliter_regex(obj, nodename):
         yield XmlXPathSelector(text=nodetext).select('//' + nodename)[0]
 
 
-def _xmliter_lxml(obj, nodename, encoding='utf-8'):
+def xmliter_lxml(obj, nodename):
     reader = _StreamReader(obj)
     iterable = etree.iterparse(reader, tag=nodename, encoding=reader.encoding)
     for _, node in iterable:
@@ -66,9 +66,9 @@ class _StreamReader(object):
 
 try:
     from lxml import etree
-    xmliter = _xmliter_lxml
+    xmliter = xmliter_lxml
 except ImportError:
-    xmliter = _xmliter_regex
+    xmliter = xmliter_regex
 
 
 def csviter(obj, delimiter=None, headers=None, encoding=None):