RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Drop EXSLT strings and math extensions

This commit is contained in:
Paul Tremberth 2014-01-15 12:28:25 +01:00
parent 2cc26e6f56
commit a3eba68aca
3 changed files with 38 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
docs/topics/selectors.rst
View File

@ -252,8 +252,6 @@ prefix namespace usage
====== ==================================== =======================
regexp http://exslt.org/regular-expressions `regular expressions`_
set http://exslt.org/sets `set manipulation`_
str http://exslt.org/strings `string manipulations`_
math http://exslt.org/math `mathematical operations`_
====== ==================================== =======================
Regular expressions
@ -379,22 +377,9 @@ Here we first iterate over ``itemscope`` elements, and for each one,
we look for all ``itemprops`` elements and exclude those that are themselves
inside another ``itemscope``.
Maths
~~~~~
Not that useful in practice, but you never know.
String manipulation
~~~~~~~~~~~~~~~~~~~
In practive, Python's string manipulation outside XPath is usually more
powerful.
.. _EXSLT: http://www.exslt.org/
.. _regular expressions: http://www.exslt.org/regexp/index.html
.. _set manipulation: http://www.exslt.org/set/index.html
.. _mathematical operations: http://www.exslt.org/math/index.html
.. _string manipulations: http://www.exslt.org/str/index.html
.. _topics-selectors-ref:

30
scrapy/selector/unified.py
View File

@ -55,35 +55,7 @@ class Selector(object_ref):
# set:intersection
# set:leading
# set:trailing
"set": "http://exslt.org/sets",
# supported in libxslt:
# math:abs()
# math:acos()
# math:asin()
# math:atan()
# math:atan2()
# math:constant()
# math:cos()
# math:exp()
# math:highest()
# math:log()
# math:lowest()
# math:max()
# math:min()
# math:power()
# math:random()
# math:sin()
# math:sqrt()
# math:tan()
"math": "http://exslt.org/math",
# supported in libxslt:
# str:align
# str:concat
# str:padding
# str:tokenize
"str": "http://exslt.org/strings",
"set": "http://exslt.org/sets"
}
def __init__(self, response=None, text=None, type=None, namespaces=None,

59
scrapy/tests/test_selector.py
View File

@ -335,7 +335,6 @@ class DeprecatedXpathSelectorTest(unittest.TestCase):
self.assertRaises(RuntimeError, xs.css, 'div')
class ExsltTestCase(unittest.TestCase):
sscls = Selector
@ -354,14 +353,26 @@ class ExsltTestCase(unittest.TestCase):
sel = self.sscls(response)
# regexp:test()
self.assertEqual(sel.xpath('//input[regexp:test(@name, "[A-Z]+", "i")]').extract(),
[x.extract() for x in sel.xpath('//input[regexp:test(@name, "[A-Z]+", "i")]')])
self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "\.html$")]/text()')],
[u'first link', u'second link'])
self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "first")]/text()')],
[u'first link'])
self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "second")]/text()')],
[u'second link'])
self.assertEqual(
sel.xpath(
'//input[regexp:test(@name, "[A-Z]+", "i")]').extract(),
[x.extract() for x in sel.xpath('//input[regexp:test(@name, "[A-Z]+", "i")]')])
self.assertEqual(
[x.extract()
for x in sel.xpath(
'//a[regexp:test(@href, "\.html$")]/text()')],
[u'first link', u'second link'])
self.assertEqual(
[x.extract()
for x in sel.xpath(
'//a[regexp:test(@href, "first")]/text()')],
[u'first link'])
self.assertEqual(
[x.extract()
for x in sel.xpath(
'//a[regexp:test(@href, "second")]/text()')],
[u'second link'])
# regexp:match() is rather special: it returns a node-set of <match> nodes
#[u'<match>http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml</match>',
@ -369,19 +380,22 @@ class ExsltTestCase(unittest.TestCase):
#u'<match>www.bayes.co.uk</match>',
#u'<match></match>',
#u'<match>/xml/index.xml?/xml/utils/rechecker.xml</match>']
self.assertEqual(sel.xpath(''
'regexp:match(//a[regexp:test(@href, "\.xml$")]/@href,'
'"(\w+):\/\/([^/:]+)(:\d*)?([^# ]*)")/text()').extract(),
[u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml',
u'http',
u'www.bayes.co.uk',
u'',
u'/xml/index.xml?/xml/utils/rechecker.xml'])
self.assertEqual(
sel.xpath('regexp:match(//a[regexp:test(@href, "\.xml$")]/@href,'
'"(\w+):\/\/([^/:]+)(:\d*)?([^# ]*)")/text()').extract(),
[u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml',
u'http',
u'www.bayes.co.uk',
u'',
u'/xml/index.xml?/xml/utils/rechecker.xml'])
# regexp:replace()
self.assertEqual(sel.xpath('regexp:replace(//a[regexp:test(@href, "\.xml$")]/@href,'
'"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(),
[u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html'])
self.assertEqual(
sel.xpath('regexp:replace(//a[regexp:test(@href, "\.xml$")]/@href,'
'"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(),
[u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html'])
def test_set(self):
"""EXSLT set manipulation tests"""
@ -430,10 +444,11 @@ class ExsltTestCase(unittest.TestCase):
u'offers',
u'lowPrice',
u'offerCount']
)
)
self.assertEqual(sel.xpath('''
set:difference(//div[@itemtype="http://schema.org/Event"]
//@itemprop,
//div[@itemtype="http://schema.org/Event"]
//*[@itemscope]/*/@itemprop)''').extract(),
[u'url', u'name', u'startDate', u'location', u'offers'])
[u'url', u'name', u'startDate', u'location', u'offers'])