diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 373d25080..deff0d8dc 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -252,8 +252,6 @@ prefix namespace usage ====== ==================================== ======================= regexp http://exslt.org/regular-expressions `regular expressions`_ set http://exslt.org/sets `set manipulation`_ -str http://exslt.org/strings `string manipulations`_ -math http://exslt.org/math `mathematical operations`_ ====== ==================================== ======================= Regular expressions @@ -379,22 +377,9 @@ Here we first iterate over ``itemscope`` elements, and for each one, we look for all ``itemprops`` elements and exclude those that are themselves inside another ``itemscope``. -Maths -~~~~~ - -Not that useful in practice, but you never know. - -String manipulation -~~~~~~~~~~~~~~~~~~~ - -In practive, Python's string manipulation outside XPath is usually more -powerful. - .. _EXSLT: http://www.exslt.org/ .. _regular expressions: http://www.exslt.org/regexp/index.html .. _set manipulation: http://www.exslt.org/set/index.html -.. _mathematical operations: http://www.exslt.org/math/index.html -.. _string manipulations: http://www.exslt.org/str/index.html .. _topics-selectors-ref: diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 4939c596f..6f502ce56 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -55,35 +55,7 @@ class Selector(object_ref): # set:intersection # set:leading # set:trailing - "set": "http://exslt.org/sets", - - # supported in libxslt: - # math:abs() - # math:acos() - # math:asin() - # math:atan() - # math:atan2() - # math:constant() - # math:cos() - # math:exp() - # math:highest() - # math:log() - # math:lowest() - # math:max() - # math:min() - # math:power() - # math:random() - # math:sin() - # math:sqrt() - # math:tan() - "math": "http://exslt.org/math", - - # supported in libxslt: - # str:align - # str:concat - # str:padding - # str:tokenize - "str": "http://exslt.org/strings", + "set": "http://exslt.org/sets" } def __init__(self, response=None, text=None, type=None, namespaces=None, diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index 8823520d1..e5035688d 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -335,7 +335,6 @@ class DeprecatedXpathSelectorTest(unittest.TestCase): self.assertRaises(RuntimeError, xs.css, 'div') - class ExsltTestCase(unittest.TestCase): sscls = Selector @@ -354,14 +353,26 @@ class ExsltTestCase(unittest.TestCase): sel = self.sscls(response) # regexp:test() - self.assertEqual(sel.xpath('//input[regexp:test(@name, "[A-Z]+", "i")]').extract(), - [x.extract() for x in sel.xpath('//input[regexp:test(@name, "[A-Z]+", "i")]')]) - self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "\.html$")]/text()')], - [u'first link', u'second link']) - self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "first")]/text()')], - [u'first link']) - self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "second")]/text()')], - [u'second link']) + self.assertEqual( + sel.xpath( + '//input[regexp:test(@name, "[A-Z]+", "i")]').extract(), + [x.extract() for x in sel.xpath('//input[regexp:test(@name, "[A-Z]+", "i")]')]) + self.assertEqual( + [x.extract() + for x in sel.xpath( + '//a[regexp:test(@href, "\.html$")]/text()')], + [u'first link', u'second link']) + self.assertEqual( + [x.extract() + for x in sel.xpath( + '//a[regexp:test(@href, "first")]/text()')], + [u'first link']) + self.assertEqual( + [x.extract() + for x in sel.xpath( + '//a[regexp:test(@href, "second")]/text()')], + [u'second link']) + # regexp:match() is rather special: it returns a node-set of nodes #[u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml', @@ -369,19 +380,22 @@ class ExsltTestCase(unittest.TestCase): #u'www.bayes.co.uk', #u'', #u'/xml/index.xml?/xml/utils/rechecker.xml'] - self.assertEqual(sel.xpath('' - 'regexp:match(//a[regexp:test(@href, "\.xml$")]/@href,' - '"(\w+):\/\/([^/:]+)(:\d*)?([^# ]*)")/text()').extract(), - [u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml', - u'http', - u'www.bayes.co.uk', - u'', - u'/xml/index.xml?/xml/utils/rechecker.xml']) + self.assertEqual( + sel.xpath('regexp:match(//a[regexp:test(@href, "\.xml$")]/@href,' + '"(\w+):\/\/([^/:]+)(:\d*)?([^# ]*)")/text()').extract(), + [u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml', + u'http', + u'www.bayes.co.uk', + u'', + u'/xml/index.xml?/xml/utils/rechecker.xml']) + + # regexp:replace() - self.assertEqual(sel.xpath('regexp:replace(//a[regexp:test(@href, "\.xml$")]/@href,' - '"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(), - [u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html']) + self.assertEqual( + sel.xpath('regexp:replace(//a[regexp:test(@href, "\.xml$")]/@href,' + '"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(), + [u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html']) def test_set(self): """EXSLT set manipulation tests""" @@ -430,10 +444,11 @@ class ExsltTestCase(unittest.TestCase): u'offers', u'lowPrice', u'offerCount'] -) + ) + self.assertEqual(sel.xpath(''' set:difference(//div[@itemtype="http://schema.org/Event"] //@itemprop, //div[@itemtype="http://schema.org/Event"] //*[@itemscope]/*/@itemprop)''').extract(), - [u'url', u'name', u'startDate', u'location', u'offers']) + [u'url', u'name', u'startDate', u'location', u'offers'])