diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 021bc43db..436f89792 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -6,7 +6,8 @@ from scrapy import log from scrapy.utils.python import re_rsearch, str_to_unicode from scrapy.utils.response import body_or_str -def xmliter(obj, nodename): + +def _xmliter_regex(obj, nodename): """Return a iterator of XPathSelector's over all nodes of a XML document, given tha name of the node to iterate. Useful for parsing XML feeds. @@ -29,6 +30,47 @@ def xmliter(obj, nodename): nodetext = header_start + match.group() + header_end yield XmlXPathSelector(text=nodetext).select('//' + nodename)[0] + +def _xmliter_lxml(obj, nodename, encoding='utf-8'): + reader = _StreamReader(obj) + iterable = etree.iterparse(reader, tag=nodename, encoding=reader.encoding) + for _, node in iterable: + nodetext = etree.tostring(node) + node.clear() + yield XmlXPathSelector(text=nodetext).select('//' + nodename)[0] + +class _StreamReader(object): + + def __init__(self, obj): + self._ptr = 0 + if isinstance(obj, Response): + self._text, self.encoding = obj.body, obj.encoding + else: + self._text, self.encoding = obj, 'utf-8' + self._is_unicode = isinstance(self._text, unicode) + + def read(self, n=65535): + self.read = self._read_unicode if self._is_unicode else self._read_string + return self.read(n).lstrip() + + def _read_string(self, n=65535): + s, e = self._ptr, self._ptr + n + self._ptr = e + return self._text[s:e] + + def _read_unicode(self, n=65535): + s, e = self._ptr, self._ptr + n + self._ptr = e + return self._text[s:e].encode('utf-8') + + +try: + from lxml import etree + xmliter = _xmliter_lxml +except ImportError: + xmliter = _xmliter_regex + + def csviter(obj, delimiter=None, headers=None, encoding=None): """ Returns an iterator of dictionaries from the given csv object