diff --git a/scrapy/trunk/scrapy/tests/test_utils_iterators.py b/scrapy/trunk/scrapy/tests/test_utils_iterators.py
index 3a575ce1e..c1f0e4bcf 100644
--- a/scrapy/trunk/scrapy/tests/test_utils_iterators.py
+++ b/scrapy/trunk/scrapy/tests/test_utils_iterators.py
@@ -1,5 +1,6 @@
import os
import unittest
+import libxml2
from scrapy.utils.iterators import csviter, xmliter
from scrapy.http import Response
@@ -33,6 +34,53 @@ class UtilsXmlTestCase(unittest.TestCase):
self.assertEqual([x.x("text()").extract() for x in xmliter(body, 'product')],
[[u'one'], [u'two']])
+ def test_iterator_namespaces(self):
+ body = """
+
+
+
+ My Dummy Company
+ http://www.mydummycompany.com
+ This is a dummy company. We do nothing.
+
+ Item 1
+ This is item 1
+ http://www.mydummycompany.com/items/1
+ http://www.mydummycompany.com/images/item1.jpg
+ ITEM_1
+ 400
+
+
+ Item 2
+ This is item 2
+ http://www.mydummycompany.com/items/2
+ http://www.mydummycompany.com/images/item2.jpg
+ ITEM_2
+ 100
+
+
+
+ """
+ response = Response(domain='mydummycompany.com', url='http://mydummycompany.com', body=body)
+ my_iter = xmliter(response, 'item')
+
+ node = my_iter.next()
+ node.register_namespace('g', 'http://base.google.com/ns/1.0')
+ self.assertEqual(node.x('title/text()').extract(), ['Item 1'])
+ self.assertEqual(node.x('description/text()').extract(), ['This is item 1'])
+ self.assertEqual(node.x('link/text()').extract(), ['http://www.mydummycompany.com/items/1'])
+ self.assertEqual(node.x('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
+ self.assertEqual(node.x('g:id/text()').extract(), ['ITEM_1'])
+ self.assertEqual(node.x('g:price/text()').extract(), ['400'])
+
+ node = my_iter.next()
+ self.assertEqual(node.x('title/text()').extract(), ['Item 2'])
+ self.assertEqual(node.x('description/text()').extract(), ['This is item 2'])
+ self.assertEqual(node.x('link/text()').extract(), ['http://www.mydummycompany.com/items/2'])
+ self.assertRaises(libxml2.xpathError, node.x, 'g:image_link/text()')
+ self.assertRaises(libxml2.xpathError, node.x, 'g:id/text()')
+ self.assertRaises(libxml2.xpathError, node.x, 'g:price/text()')
+
def test_iterator_exception(self):
body = u"""onetwo"""
diff --git a/scrapy/trunk/scrapy/utils/iterators.py b/scrapy/trunk/scrapy/utils/iterators.py
index 86a2ea19b..d585dcc06 100644
--- a/scrapy/trunk/scrapy/utils/iterators.py
+++ b/scrapy/trunk/scrapy/utils/iterators.py
@@ -3,6 +3,7 @@ import re, csv
from scrapy.xpath import XmlXPathSelector
from scrapy.http import Response
from scrapy import log
+from scrapy.utils.python import re_rsearch
def _normalize_input(obj):
assert isinstance(obj, (Response, basestring)), "obj must be Response or basestring, not %s" % type(obj).__name__
@@ -22,12 +23,19 @@ def xmliter(obj, nodename):
- a unicode string
- a string encoded as utf-8
"""
+ HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename, re.S)
+ HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S)
text = _normalize_input(obj)
+ header_start = re.search(HEADER_START_RE, text)
+ header_start = header_start.group(1).strip() if header_start else ''
+ header_end = re_rsearch(HEADER_END_RE, text)
+ header_end = text[header_end[1]:].strip() if header_end else ''
+
r = re.compile(r"<%s[\s>].*?%s>" % (nodename, nodename), re.DOTALL)
for match in r.finditer(text):
- nodetext = match.group()
- yield XmlXPathSelector(text=nodetext).x('/' + nodename)[0]
+ nodetext = header_start + match.group() + header_end
+ yield XmlXPathSelector(text=nodetext).x('//' + nodename)[0]
def csviter(obj, delimiter=None, headers=None):
""" Returns an iterator of dictionaries from the given csv object
diff --git a/scrapy/trunk/scrapy/utils/python.py b/scrapy/trunk/scrapy/utils/python.py
index 23e08075a..0420ff9c2 100644
--- a/scrapy/trunk/scrapy/utils/python.py
+++ b/scrapy/trunk/scrapy/utils/python.py
@@ -1,7 +1,7 @@
"""
This module contains essential stuff that should've come with Python itself ;)
"""
-
+import re
from sgmllib import SGMLParser
class FixedSGMLParser(SGMLParser):
@@ -53,3 +53,32 @@ def unique(list_):
return result
+def re_rsearch(pattern, text, chunk_size=1024):
+ """
+ This function does a reverse search in a text using a regular expression
+ given in the attribute 'pattern'.
+ Since the re module does not provide this functionality, we have to find for
+ the expression into chunks of text extracted from the end (for the sake of efficiency).
+ At first, a chunk of 'chunk_size' kilobytes is extracted from the end, and searched for
+ the pattern. If the pattern is not found, another chunk is extracted, and another
+ search is performed.
+ This process continues until a match is found, or until the whole file is read.
+ In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing
+ the start position of the match, and the ending (regarding the entire text).
+ """
+ def _chunk_iter():
+ offset = text_len = len(text)
+ while True:
+ offset -= (chunk_size * 1024)
+ if offset <= 0:
+ break
+ yield (text[offset:], offset)
+ yield (text, 0)
+
+ pattern = re.compile(pattern) if isinstance(pattern, basestring) else pattern
+ for chunk, offset in _chunk_iter():
+ matches = [match for match in pattern.finditer(chunk)]
+ if matches:
+ return (offset + matches[-1].span()[0], offset + matches[-1].span()[1])
+ return None
+