diff --git a/scrapy/contrib_exp/iterators.py b/scrapy/contrib_exp/iterators.py index 0fc73e194..d42a25c3f 100644 --- a/scrapy/contrib_exp/iterators.py +++ b/scrapy/contrib_exp/iterators.py @@ -2,14 +2,19 @@ from scrapy.http import Response from scrapy.selector import XmlXPathSelector -def xmliter_lxml(obj, nodename): +def xmliter_lxml(obj, nodename, namespace=None): from lxml import etree reader = _StreamReader(obj) - iterable = etree.iterparse(reader, tag=nodename, encoding=reader.encoding) + tag = '{%s}%s' % (namespace, nodename) if namespace else nodename + iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) for _, node in iterable: nodetext = etree.tostring(node) node.clear() - yield XmlXPathSelector(text=nodetext).select('//' + nodename)[0] + xs = XmlXPathSelector(text=nodetext) + if namespace: + xs.register_namespace('x', namespace) + nodename = 'x:%s' % nodename + yield xs.select('//' + nodename)[0] class _StreamReader(object): diff --git a/scrapy/tests/test_utils_iterators.py b/scrapy/tests/test_utils_iterators.py index d32d1e658..79435d615 100644 --- a/scrapy/tests/test_utils_iterators.py +++ b/scrapy/tests/test_utils_iterators.py @@ -29,12 +29,12 @@ class XmliterTestCase(unittest.TestCase): for x in self.xmliter(response, 'product'): attrs.append((x.select("@id").extract(), x.select("name/text()").extract(), x.select("./type/text()").extract())) - self.assertEqual(attrs, + self.assertEqual(attrs, [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])]) def test_xmliter_text(self): body = u"""onetwo""" - + self.assertEqual([x.select("text()").extract() for x in self.xmliter(body, 'product')], [[u'one'], [u'two']]) @@ -74,7 +74,7 @@ class XmliterTestCase(unittest.TestCase): def test_xmliter_exception(self): body = u"""onetwo""" - + iter = self.xmliter(body, 'product') iter.next() iter.next() @@ -97,6 +97,32 @@ class LxmlXmliterTestCase(XmliterTestCase): except ImportError: skip = "lxml not available" + def test_xmliter_iterate_namespace(self): + body = """\ + + + + My Dummy Company + http://www.mydummycompany.com + This is a dummy company. We do nothing. + + Item 1 + This is item 1 + http://www.mydummycompany.com/items/1 + http://www.mydummycompany.com/images/item1.jpg + + + + """ + response = XmlResponse(url='http://mydummycompany.com', body=body) + + no_namespace_iter = self.xmliter(response, 'image_link') + self.assertEqual(len(list(no_namespace_iter)), 0) + + namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0') + node = namespace_iter.next() + self.assertEqual(node.select('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + class UtilsCsvTestCase(unittest.TestCase): sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')