diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index a316c20cf..1646bb6c6 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -3,6 +3,7 @@ XPath selectors based on lxml """ from lxml import etree +import copy from scrapy.utils.misc import extract_regex from scrapy.utils.trackref import object_ref @@ -46,6 +47,45 @@ class Selector(object_ref): '__weakref__', '_parser', '_csstranslator', '_tostring_method'] _default_type = None + _default_namespaces = { + "regexp": "http://exslt.org/regular-expressions", + + # supported in libxslt: + # set:difference + # set:has-same-node + # set:intersection + # set:leading + # set:trailing + "set": "http://exslt.org/sets", + + # supported in libxslt: + # math:abs() + # math:acos() + # math:asin() + # math:atan() + # math:atan2() + # math:constant() + # math:cos() + # math:exp() + # math:highest() + # math:log() + # math:lowest() + # math:max() + # math:min() + # math:power() + # math:random() + # math:sin() + # math:sqrt() + # math:tan() + "math": "http://exslt.org/math", + + # supported in libxslt: + # str:align + # str:concat + # str:padding + # str:tokenize + "str": "http://exslt.org/strings", + } def __init__(self, response=None, text=None, type=None, namespaces=None, _root=None, _expr=None): @@ -61,7 +101,9 @@ class Selector(object_ref): _root = LxmlDocument(response, self._parser) self.response = response - self.namespaces = namespaces + ns = copy.copy(self._default_namespaces) + ns.update(namespaces or {}) + self.namespaces = ns self._root = _root self._expr = _expr diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index 5a14031fe..a2eaa0e58 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -333,3 +333,52 @@ class DeprecatedXpathSelectorTest(unittest.TestCase): self.assertEqual(xs.select("//div").extract(), [u'

Hello

']) self.assertRaises(RuntimeError, xs.css, 'div') + + + +class ExsltTestCase(unittest.TestCase): + + sscls = Selector + + def test_regexp(self): + """EXSLT regular expression tests""" + body = """ +

+ + """ + response = TextResponse(url="http://example.com", body=body) + sel = self.sscls(response) + + # regexp:test() + self.assertEqual(sel.xpath('//input[regexp:test(@name, "[A-Z]+", "i")]').extract(), + [x.extract() for x in sel.xpath('//input[regexp:test(@name, "[A-Z]+", "i")]')]) + self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "\.html$")]/text()')], + [u'first link', u'second link']) + self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "first")]/text()')], + [u'first link']) + self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "second")]/text()')], + [u'second link']) + + # regexp:match() is rather special: it returns a node-set of nodes + #[u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml', + #u'http', + #u'www.bayes.co.uk', + #u'', + #u'/xml/index.xml?/xml/utils/rechecker.xml'] + self.assertEqual(sel.xpath('' + 'regexp:match(//a[regexp:test(@href, "\.xml$")]/@href,' + '"(\w+):\/\/([^/:]+)(:\d*)?([^# ]*)")/text()').extract(), + [u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml', + u'http', + u'www.bayes.co.uk', + u'', + u'/xml/index.xml?/xml/utils/rechecker.xml']) + + # regexp:replace() + self.assertEqual(sel.xpath('regexp:replace(//a[regexp:test(@href, "\.xml$")]/@href,' + '"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(), + [u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html'])