diff --git a/scrapy/tests/test_utils_iterators.py b/scrapy/tests/test_utils_iterators.py index 264197333..554b6e364 100644 --- a/scrapy/tests/test_utils_iterators.py +++ b/scrapy/tests/test_utils_iterators.py @@ -2,7 +2,8 @@ import os import libxml2 from twisted.trial import unittest -from scrapy.utils.iterators import csviter, xmliter, _xmliter_lxml, _xmliter_regex +from scrapy.utils.iterators import csviter, xmliter +from scrapy.contrib_exp.iterators import xmliter_lxml from scrapy.http import XmlResponse, TextResponse from scrapy.tests import get_testdata @@ -90,12 +91,8 @@ class XmliterTestCase(unittest.TestCase): ) -class RegexXmliterTestCase(XmliterTestCase): - xmliter = staticmethod(_xmliter_regex) - - class LxmlXmliterTestCase(XmliterTestCase): - xmliter = staticmethod(_xmliter_lxml) + xmliter = staticmethod(xmliter_lxml) try: import lxml except ImportError: diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 22a95606b..9f745503f 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -7,7 +7,7 @@ from scrapy.utils.python import re_rsearch, str_to_unicode from scrapy.utils.response import body_or_str -def _xmliter_regex(obj, nodename): +def xmliter(obj, nodename): """Return a iterator of XPathSelector's over all nodes of a XML document, given tha name of the node to iterate. Useful for parsing XML feeds. @@ -31,46 +31,6 @@ def _xmliter_regex(obj, nodename): yield XmlXPathSelector(text=nodetext).select('//' + nodename)[0] -def _xmliter_lxml(obj, nodename): - reader = _StreamReader(obj) - iterable = etree.iterparse(reader, tag=nodename, encoding=reader.encoding) - for _, node in iterable: - nodetext = etree.tostring(node) - node.clear() - yield XmlXPathSelector(text=nodetext).select('//' + nodename)[0] - -class _StreamReader(object): - - def __init__(self, obj): - self._ptr = 0 - if isinstance(obj, Response): - self._text, self.encoding = obj.body, obj.encoding - else: - self._text, self.encoding = obj, 'utf-8' - self._is_unicode = isinstance(self._text, unicode) - - def read(self, n=65535): - self.read = self._read_unicode if self._is_unicode else self._read_string - return self.read(n).lstrip() - - def _read_string(self, n=65535): - s, e = self._ptr, self._ptr + n - self._ptr = e - return self._text[s:e] - - def _read_unicode(self, n=65535): - s, e = self._ptr, self._ptr + n - self._ptr = e - return self._text[s:e].encode('utf-8') - - -try: - from lxml import etree - xmliter = _xmliter_lxml -except ImportError: - xmliter = _xmliter_regex - - def csviter(obj, delimiter=None, headers=None, encoding=None): """ Returns an iterator of dictionaries from the given csv object