added remove_namespaces() method to XmlXPathSelector objects

docs/news.rst

@@ -6,6 +6,7 @@ Release notes
 0.18 (unreleased)
 -----------------
 
+- added :meth:`XmlXPathSelector.remove_namespaces` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
 - several improvements to spider contracts
 - New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections,
   MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62

docs/topics/selectors.rst

@@ -261,6 +261,11 @@ XPathSelector objects
        Without registering namespaces you can't select or extract data from
        non-standard namespaces. See examples below.
 
+   .. method:: remove_namespaces()
+
+       Remove all namespaces, allowing to traverse the document using
+       namespace-less xpaths. See example below.
+
    .. method:: __nonzero__()
 
        Returns ``True`` if there is any real content selected by this
@@ -377,4 +382,24 @@ instantiated with a :class:`~scrapy.http.Response` object like this::
       x.register_namespace("g", "http://base.google.com/ns/1.0")
       x.select("//g:price").extract()
 
+Removing namespaces
+~~~~~~~~~~~~~~~~~~~
+
+When dealing with scraping projects, it is often quite convenient to get rid of
+namespaces altogether and just work with element names. You can use the
+:meth:`XmlXPathSelector.remove_namespaces` method for that.
+
+Here's an example that illustrates this to parse the Github blog atom feed::
+
+    $ scrapy shell https://github.com/blog.atom
+    # ...
+    >>> xxs.select("//link")
+    []
+    >>> xxs.remove_namespaces()
+    >>> xxs.select("//link")
+    [<XmlXPathSelector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>,
+     <XmlXPathSelector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>,
+     ...
+    ]
+
 .. _Google Base XML feed: http://base.google.com/support/bin/answer.py?hl=en&answer=59461

scrapy/selector/lxmlsel.py

@@ -73,6 +73,11 @@ class XPathSelector(object_ref):
             self.namespaces = {}
         self.namespaces[prefix] = uri
 
+    def remove_namespaces(self):
+        for el in self._root.iter('*'):
+            if el.tag.startswith('{'):
+                el.tag = el.tag.split('}', 1)[1]
+
     def __nonzero__(self):
         return bool(self.extract())
 

scrapy/tests/test_selector_lxml.py

@@ -4,7 +4,7 @@ Selectors tests, specific for lxml backend
 
 import unittest
 from scrapy.tests import test_selector
-from scrapy.http import TextResponse, HtmlResponse
+from scrapy.http import TextResponse, HtmlResponse, XmlResponse
 from scrapy.selector.lxmldocument import LxmlDocument
 from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector
 
@@ -15,6 +15,17 @@ class LxmlXPathSelectorTestCase(test_selector.XPathSelectorTestCase):
     hxs_cls = HtmlXPathSelector
     xxs_cls = XmlXPathSelector
 
+    def test_remove_namespaces(self):
+        xml = """<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en-US" xmlns:media="http://search.yahoo.com/mrss/">
+  <link type="text/html">
+  <link type="application/atom+xml">
+</feed>
+"""
+        xxs = XmlXPathSelector(XmlResponse("http://example.com/feed.atom", body=xml))
+        self.assertEqual(len(xxs.select("//link")), 0)
+        xxs.remove_namespaces()
+        self.assertEqual(len(xxs.select("//link")), 2)
 
 class Libxml2DocumentTest(unittest.TestCase):