diff --git a/docs/news.rst b/docs/news.rst index 1296666c4..59c7c849c 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,6 +6,7 @@ Release notes 0.18 (unreleased) ----------------- +- added :meth:`XmlXPathSelector.remove_namespaces` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. - several improvements to spider contracts - New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections, MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62 diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 4211ea025..54e9a81fc 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -261,6 +261,11 @@ XPathSelector objects Without registering namespaces you can't select or extract data from non-standard namespaces. See examples below. + .. method:: remove_namespaces() + + Remove all namespaces, allowing to traverse the document using + namespace-less xpaths. See example below. + .. method:: __nonzero__() Returns ``True`` if there is any real content selected by this @@ -377,4 +382,24 @@ instantiated with a :class:`~scrapy.http.Response` object like this:: x.register_namespace("g", "http://base.google.com/ns/1.0") x.select("//g:price").extract() +Removing namespaces +~~~~~~~~~~~~~~~~~~~ + +When dealing with scraping projects, it is often quite convenient to get rid of +namespaces altogether and just work with element names. You can use the +:meth:`XmlXPathSelector.remove_namespaces` method for that. + +Here's an example that illustrates this to parse the Github blog atom feed:: + + $ scrapy shell https://github.com/blog.atom + # ... + >>> xxs.select("//link") + [] + >>> xxs.remove_namespaces() + >>> xxs.select("//link") + [, + ... + ] + .. _Google Base XML feed: http://base.google.com/support/bin/answer.py?hl=en&answer=59461 diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py index 6e5fda556..8708144bd 100644 --- a/scrapy/selector/lxmlsel.py +++ b/scrapy/selector/lxmlsel.py @@ -73,6 +73,11 @@ class XPathSelector(object_ref): self.namespaces = {} self.namespaces[prefix] = uri + def remove_namespaces(self): + for el in self._root.iter('*'): + if el.tag.startswith('{'): + el.tag = el.tag.split('}', 1)[1] + def __nonzero__(self): return bool(self.extract()) diff --git a/scrapy/tests/test_selector_lxml.py b/scrapy/tests/test_selector_lxml.py index c8eca9679..f9c9ec2ac 100644 --- a/scrapy/tests/test_selector_lxml.py +++ b/scrapy/tests/test_selector_lxml.py @@ -4,7 +4,7 @@ Selectors tests, specific for lxml backend import unittest from scrapy.tests import test_selector -from scrapy.http import TextResponse, HtmlResponse +from scrapy.http import TextResponse, HtmlResponse, XmlResponse from scrapy.selector.lxmldocument import LxmlDocument from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector @@ -15,6 +15,17 @@ class LxmlXPathSelectorTestCase(test_selector.XPathSelectorTestCase): hxs_cls = HtmlXPathSelector xxs_cls = XmlXPathSelector + def test_remove_namespaces(self): + xml = """ + + + + +""" + xxs = XmlXPathSelector(XmlResponse("http://example.com/feed.atom", body=xml)) + self.assertEqual(len(xxs.select("//link")), 0) + xxs.remove_namespaces() + self.assertEqual(len(xxs.select("//link")), 2) class Libxml2DocumentTest(unittest.TestCase):