Register EXSLT namespaces by default (resolves #470)

scrapy/selector/unified.py

@@ -3,6 +3,7 @@ XPath selectors based on lxml
 """
 
 from lxml import etree
+import copy
 
 from scrapy.utils.misc import extract_regex
 from scrapy.utils.trackref import object_ref
@@ -46,6 +47,45 @@ class Selector(object_ref):
                  '__weakref__', '_parser', '_csstranslator', '_tostring_method']
 
     _default_type = None
+    _default_namespaces = {
+        "regexp": "http://exslt.org/regular-expressions",
+
+        # supported in libxslt:
+        # set:difference
+        # set:has-same-node
+        # set:intersection
+        # set:leading
+        # set:trailing
+        "set": "http://exslt.org/sets",
+
+        # supported in libxslt:
+        # math:abs()
+        # math:acos()
+        # math:asin()
+        # math:atan()
+        # math:atan2()
+        # math:constant()
+        # math:cos()
+        # math:exp()
+        # math:highest()
+        # math:log()
+        # math:lowest()
+        # math:max()
+        # math:min()
+        # math:power()
+        # math:random()
+        # math:sin()
+        # math:sqrt()
+        # math:tan()
+        "math": "http://exslt.org/math",
+
+        # supported in libxslt:
+        # str:align
+        # str:concat
+        # str:padding
+        # str:tokenize
+        "str": "http://exslt.org/strings",
+    }
 
     def __init__(self, response=None, text=None, type=None, namespaces=None,
                  _root=None, _expr=None):
@@ -61,7 +101,9 @@ class Selector(object_ref):
             _root = LxmlDocument(response, self._parser)
 
         self.response = response
-        self.namespaces = namespaces
+        ns = copy.copy(self._default_namespaces)
+        ns.update(namespaces or {})
+        self.namespaces = ns
         self._root = _root
         self._expr = _expr
 

scrapy/tests/test_selector.py

@@ -333,3 +333,52 @@ class DeprecatedXpathSelectorTest(unittest.TestCase):
             self.assertEqual(xs.select("//div").extract(),
                              [u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
             self.assertRaises(RuntimeError, xs.css, 'div')
+
+
+
+class ExsltTestCase(unittest.TestCase):
+
+    sscls = Selector
+
+    def test_regexp(self):
+        """EXSLT regular expression tests"""
+        body = """
+        <p><input name='a' value='1'/><input name='b' value='2'/></p>
+        <div class="links">
+        <a href="/first.html">first link</a>
+        <a href="/second.html">second link</a>
+        <a href="http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml">EXSLT match example</a>
+        </div>
+        """
+        response = TextResponse(url="http://example.com", body=body)
+        sel = self.sscls(response)
+
+        # regexp:test()
+        self.assertEqual(sel.xpath('//input[regexp:test(@name, "[A-Z]+", "i")]').extract(),
+                         [x.extract() for x in sel.xpath('//input[regexp:test(@name, "[A-Z]+", "i")]')])
+        self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "\.html$")]/text()')],
+                         [u'first link', u'second link'])
+        self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "first")]/text()')],
+                         [u'first link'])
+        self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "second")]/text()')],
+                         [u'second link'])
+
+        # regexp:match() is rather special: it returns a node-set of <match> nodes
+        #[u'<match>http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml</match>',
+        #u'<match>http</match>',
+        #u'<match>www.bayes.co.uk</match>',
+        #u'<match></match>',
+        #u'<match>/xml/index.xml?/xml/utils/rechecker.xml</match>']
+        self.assertEqual(sel.xpath(''
+            'regexp:match(//a[regexp:test(@href, "\.xml$")]/@href,'
+            '"(\w+):\/\/([^/:]+)(:\d*)?([^# ]*)")/text()').extract(),
+                         [u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml',
+                             u'http',
+                             u'www.bayes.co.uk',
+                             u'',
+                             u'/xml/index.xml?/xml/utils/rechecker.xml'])
+
+        # regexp:replace()
+        self.assertEqual(sel.xpath('regexp:replace(//a[regexp:test(@href, "\.xml$")]/@href,'
+            '"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(),
+                         [u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html'])