From 9d76ff606b9dd182c19cbd728ac6a3067b249793 Mon Sep 17 00:00:00 2001 From: elpolilla Date: Fri, 19 Dec 2008 10:16:31 +0000 Subject: [PATCH] Fixed bug in xmliter, which didnt add headers to the nodes it returned --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40536 --- .../scrapy/tests/test_utils_iterators.py | 48 +++++++++++++++++++ scrapy/trunk/scrapy/utils/iterators.py | 12 ++++- scrapy/trunk/scrapy/utils/python.py | 31 +++++++++++- 3 files changed, 88 insertions(+), 3 deletions(-) diff --git a/scrapy/trunk/scrapy/tests/test_utils_iterators.py b/scrapy/trunk/scrapy/tests/test_utils_iterators.py index 3a575ce1e..c1f0e4bcf 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_iterators.py +++ b/scrapy/trunk/scrapy/tests/test_utils_iterators.py @@ -1,5 +1,6 @@ import os import unittest +import libxml2 from scrapy.utils.iterators import csviter, xmliter from scrapy.http import Response @@ -33,6 +34,53 @@ class UtilsXmlTestCase(unittest.TestCase): self.assertEqual([x.x("text()").extract() for x in xmliter(body, 'product')], [[u'one'], [u'two']]) + def test_iterator_namespaces(self): + body = """ + + + + My Dummy Company + http://www.mydummycompany.com + This is a dummy company. We do nothing. + + Item 1 + This is item 1 + http://www.mydummycompany.com/items/1 + http://www.mydummycompany.com/images/item1.jpg + ITEM_1 + 400 + + + Item 2 + This is item 2 + http://www.mydummycompany.com/items/2 + http://www.mydummycompany.com/images/item2.jpg + ITEM_2 + 100 + + + + """ + response = Response(domain='mydummycompany.com', url='http://mydummycompany.com', body=body) + my_iter = xmliter(response, 'item') + + node = my_iter.next() + node.register_namespace('g', 'http://base.google.com/ns/1.0') + self.assertEqual(node.x('title/text()').extract(), ['Item 1']) + self.assertEqual(node.x('description/text()').extract(), ['This is item 1']) + self.assertEqual(node.x('link/text()').extract(), ['http://www.mydummycompany.com/items/1']) + self.assertEqual(node.x('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + self.assertEqual(node.x('g:id/text()').extract(), ['ITEM_1']) + self.assertEqual(node.x('g:price/text()').extract(), ['400']) + + node = my_iter.next() + self.assertEqual(node.x('title/text()').extract(), ['Item 2']) + self.assertEqual(node.x('description/text()').extract(), ['This is item 2']) + self.assertEqual(node.x('link/text()').extract(), ['http://www.mydummycompany.com/items/2']) + self.assertRaises(libxml2.xpathError, node.x, 'g:image_link/text()') + self.assertRaises(libxml2.xpathError, node.x, 'g:id/text()') + self.assertRaises(libxml2.xpathError, node.x, 'g:price/text()') + def test_iterator_exception(self): body = u"""onetwo""" diff --git a/scrapy/trunk/scrapy/utils/iterators.py b/scrapy/trunk/scrapy/utils/iterators.py index 86a2ea19b..d585dcc06 100644 --- a/scrapy/trunk/scrapy/utils/iterators.py +++ b/scrapy/trunk/scrapy/utils/iterators.py @@ -3,6 +3,7 @@ import re, csv from scrapy.xpath import XmlXPathSelector from scrapy.http import Response from scrapy import log +from scrapy.utils.python import re_rsearch def _normalize_input(obj): assert isinstance(obj, (Response, basestring)), "obj must be Response or basestring, not %s" % type(obj).__name__ @@ -22,12 +23,19 @@ def xmliter(obj, nodename): - a unicode string - a string encoded as utf-8 """ + HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename, re.S) + HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S) text = _normalize_input(obj) + header_start = re.search(HEADER_START_RE, text) + header_start = header_start.group(1).strip() if header_start else '' + header_end = re_rsearch(HEADER_END_RE, text) + header_end = text[header_end[1]:].strip() if header_end else '' + r = re.compile(r"<%s[\s>].*?" % (nodename, nodename), re.DOTALL) for match in r.finditer(text): - nodetext = match.group() - yield XmlXPathSelector(text=nodetext).x('/' + nodename)[0] + nodetext = header_start + match.group() + header_end + yield XmlXPathSelector(text=nodetext).x('//' + nodename)[0] def csviter(obj, delimiter=None, headers=None): """ Returns an iterator of dictionaries from the given csv object diff --git a/scrapy/trunk/scrapy/utils/python.py b/scrapy/trunk/scrapy/utils/python.py index 23e08075a..0420ff9c2 100644 --- a/scrapy/trunk/scrapy/utils/python.py +++ b/scrapy/trunk/scrapy/utils/python.py @@ -1,7 +1,7 @@ """ This module contains essential stuff that should've come with Python itself ;) """ - +import re from sgmllib import SGMLParser class FixedSGMLParser(SGMLParser): @@ -53,3 +53,32 @@ def unique(list_): return result +def re_rsearch(pattern, text, chunk_size=1024): + """ + This function does a reverse search in a text using a regular expression + given in the attribute 'pattern'. + Since the re module does not provide this functionality, we have to find for + the expression into chunks of text extracted from the end (for the sake of efficiency). + At first, a chunk of 'chunk_size' kilobytes is extracted from the end, and searched for + the pattern. If the pattern is not found, another chunk is extracted, and another + search is performed. + This process continues until a match is found, or until the whole file is read. + In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing + the start position of the match, and the ending (regarding the entire text). + """ + def _chunk_iter(): + offset = text_len = len(text) + while True: + offset -= (chunk_size * 1024) + if offset <= 0: + break + yield (text[offset:], offset) + yield (text, 0) + + pattern = re.compile(pattern) if isinstance(pattern, basestring) else pattern + for chunk, offset in _chunk_iter(): + matches = [match for match in pattern.finditer(chunk)] + if matches: + return (offset + matches[-1].span()[0], offset + matches[-1].span()[1]) + return None +