Fixed bug in xmliter, which didnt add headers to the nodes it returned

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40536

scrapy/trunk/scrapy/tests/test_utils_iterators.py

@@ -1,5 +1,6 @@
 import os
 import unittest
+import libxml2
 
 from scrapy.utils.iterators import csviter, xmliter
 from scrapy.http import Response
@@ -33,6 +34,53 @@ class UtilsXmlTestCase(unittest.TestCase):
         self.assertEqual([x.x("text()").extract() for x in xmliter(body, 'product')],
                          [[u'one'], [u'two']])
 
+    def test_iterator_namespaces(self):
+        body = """
+            <?xml version="1.0" encoding="UTF-8"?>
+            <rss version="2.0" xmlns:g="http://base.google.com/ns/1.0">
+                <channel>
+                <title>My Dummy Company</title>
+                <link>http://www.mydummycompany.com</link>
+                <description>This is a dummy company. We do nothing.</description>
+                <item>
+                    <title>Item 1</title>
+                    <description>This is item 1</description>
+                    <link>http://www.mydummycompany.com/items/1</link>
+                    <g:image_link>http://www.mydummycompany.com/images/item1.jpg</g:image_link>
+                    <g:id>ITEM_1</g:id>
+                    <g:price>400</g:price>
+                </item>
+                <item>
+                    <title>Item 2</title>
+                    <description>This is item 2</description>
+                    <link>http://www.mydummycompany.com/items/2</link>
+                    <g:image_link>http://www.mydummycompany.com/images/item2.jpg</g:image_link>
+                    <g:id>ITEM_2</g:id>
+                    <g:price>100</g:price>
+                </item>
+                </channel>
+            </rss>
+        """
+        response = Response(domain='mydummycompany.com', url='http://mydummycompany.com', body=body)
+        my_iter = xmliter(response, 'item')
+
+        node = my_iter.next()
+        node.register_namespace('g', 'http://base.google.com/ns/1.0')
+        self.assertEqual(node.x('title/text()').extract(), ['Item 1'])
+        self.assertEqual(node.x('description/text()').extract(), ['This is item 1'])
+        self.assertEqual(node.x('link/text()').extract(), ['http://www.mydummycompany.com/items/1'])
+        self.assertEqual(node.x('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
+        self.assertEqual(node.x('g:id/text()').extract(), ['ITEM_1'])
+        self.assertEqual(node.x('g:price/text()').extract(), ['400'])
+
+        node = my_iter.next()
+        self.assertEqual(node.x('title/text()').extract(), ['Item 2'])
+        self.assertEqual(node.x('description/text()').extract(), ['This is item 2'])
+        self.assertEqual(node.x('link/text()').extract(), ['http://www.mydummycompany.com/items/2'])
+        self.assertRaises(libxml2.xpathError, node.x, 'g:image_link/text()')
+        self.assertRaises(libxml2.xpathError, node.x, 'g:id/text()')
+        self.assertRaises(libxml2.xpathError, node.x, 'g:price/text()')
+
     def test_iterator_exception(self):
         body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
         

scrapy/trunk/scrapy/utils/iterators.py

@@ -3,6 +3,7 @@ import re, csv
 from scrapy.xpath import XmlXPathSelector
 from scrapy.http import Response
 from scrapy import log
+from scrapy.utils.python import re_rsearch
 
 def _normalize_input(obj):
     assert isinstance(obj, (Response, basestring)), "obj must be Response or basestring, not %s" % type(obj).__name__
@@ -22,12 +23,19 @@ def xmliter(obj, nodename):
     - a unicode string
     - a string encoded as utf-8
     """
+    HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename, re.S)
+    HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S)
     text = _normalize_input(obj)
 
+    header_start = re.search(HEADER_START_RE, text)
+    header_start = header_start.group(1).strip() if header_start else ''
+    header_end = re_rsearch(HEADER_END_RE, text)
+    header_end = text[header_end[1]:].strip() if header_end else ''
+
     r = re.compile(r"<%s[\s>].*?</%s>" % (nodename, nodename), re.DOTALL)
     for match in r.finditer(text):
-        nodetext = match.group()
-        yield XmlXPathSelector(text=nodetext).x('/' + nodename)[0]
+        nodetext = header_start + match.group() + header_end
+        yield XmlXPathSelector(text=nodetext).x('//' + nodename)[0]
 
 def csviter(obj, delimiter=None, headers=None):
     """ Returns an iterator of dictionaries from the given csv object

scrapy/trunk/scrapy/utils/python.py

@@ -1,7 +1,7 @@
 """
 This module contains essential stuff that should've come with Python itself ;)
 """
-
+import re
 from sgmllib import SGMLParser
 
 class FixedSGMLParser(SGMLParser):
@@ -53,3 +53,32 @@ def unique(list_):
     return result
 
 
+def re_rsearch(pattern, text, chunk_size=1024):
+    """
+    This function does a reverse search in a text using a regular expression
+    given in the attribute 'pattern'.
+    Since the re module does not provide this functionality, we have to find for
+    the expression into chunks of text extracted from the end (for the sake of efficiency).
+    At first, a chunk of 'chunk_size' kilobytes is extracted from the end, and searched for
+    the pattern. If the pattern is not found, another chunk is extracted, and another
+    search is performed.
+    This process continues until a match is found, or until the whole file is read.
+    In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing
+    the start position of the match, and the ending (regarding the entire text).
+    """
+    def _chunk_iter():
+        offset = text_len = len(text)
+        while True:
+            offset -= (chunk_size * 1024)
+            if offset <= 0:
+                break
+            yield (text[offset:], offset)
+        yield (text, 0)
+
+    pattern = re.compile(pattern) if isinstance(pattern, basestring) else pattern
+    for chunk, offset in _chunk_iter():
+        matches = [match for match in pattern.finditer(chunk)]
+        if matches:
+            return (offset + matches[-1].span()[0], offset + matches[-1].span()[1])
+    return None
+