diff --git a/scrapy/contrib_exp/iterators.py b/scrapy/contrib_exp/iterators.py
index 0fc73e194..d42a25c3f 100644
--- a/scrapy/contrib_exp/iterators.py
+++ b/scrapy/contrib_exp/iterators.py
@@ -2,14 +2,19 @@ from scrapy.http import Response
from scrapy.selector import XmlXPathSelector
-def xmliter_lxml(obj, nodename):
+def xmliter_lxml(obj, nodename, namespace=None):
from lxml import etree
reader = _StreamReader(obj)
- iterable = etree.iterparse(reader, tag=nodename, encoding=reader.encoding)
+ tag = '{%s}%s' % (namespace, nodename) if namespace else nodename
+ iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
for _, node in iterable:
nodetext = etree.tostring(node)
node.clear()
- yield XmlXPathSelector(text=nodetext).select('//' + nodename)[0]
+ xs = XmlXPathSelector(text=nodetext)
+ if namespace:
+ xs.register_namespace('x', namespace)
+ nodename = 'x:%s' % nodename
+ yield xs.select('//' + nodename)[0]
class _StreamReader(object):
diff --git a/scrapy/tests/test_utils_iterators.py b/scrapy/tests/test_utils_iterators.py
index d32d1e658..79435d615 100644
--- a/scrapy/tests/test_utils_iterators.py
+++ b/scrapy/tests/test_utils_iterators.py
@@ -29,12 +29,12 @@ class XmliterTestCase(unittest.TestCase):
for x in self.xmliter(response, 'product'):
attrs.append((x.select("@id").extract(), x.select("name/text()").extract(), x.select("./type/text()").extract()))
- self.assertEqual(attrs,
+ self.assertEqual(attrs,
[(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])])
def test_xmliter_text(self):
body = u"""onetwo"""
-
+
self.assertEqual([x.select("text()").extract() for x in self.xmliter(body, 'product')],
[[u'one'], [u'two']])
@@ -74,7 +74,7 @@ class XmliterTestCase(unittest.TestCase):
def test_xmliter_exception(self):
body = u"""onetwo"""
-
+
iter = self.xmliter(body, 'product')
iter.next()
iter.next()
@@ -97,6 +97,32 @@ class LxmlXmliterTestCase(XmliterTestCase):
except ImportError:
skip = "lxml not available"
+ def test_xmliter_iterate_namespace(self):
+ body = """\
+
+
+
+ My Dummy Company
+ http://www.mydummycompany.com
+ This is a dummy company. We do nothing.
+ -
+ Item 1
+ This is item 1
+ http://www.mydummycompany.com/items/1
+ http://www.mydummycompany.com/images/item1.jpg
+
+
+
+ """
+ response = XmlResponse(url='http://mydummycompany.com', body=body)
+
+ no_namespace_iter = self.xmliter(response, 'image_link')
+ self.assertEqual(len(list(no_namespace_iter)), 0)
+
+ namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0')
+ node = namespace_iter.next()
+ self.assertEqual(node.select('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
+
class UtilsCsvTestCase(unittest.TestCase):
sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')