diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 3872736ae..c64ebad62 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -240,6 +240,149 @@ XPath specification. .. _Location Paths: http://www.w3.org/TR/xpath#location-paths +Using EXSLT extensions +---------------------- + +Being built atop `lxml`_, Scrapy selectors also support some `EXSLT`_ extensions +and come with these pre-registered namespaces to use in XPath expressions: + + +====== ==================================== ======================= +prefix namespace usage +====== ==================================== ======================= +re http://exslt.org/regular-expressions `regular expressions`_ +set http://exslt.org/sets `set manipulation`_ +====== ==================================== ======================= + +Regular expressions +~~~~~~~~~~~~~~~~~~~ + +The ``test()`` function for example can prove quite useful when XPath's +``starts-with()`` or ``contains()`` are not sufficient. + +Example selecting links in list item with a "class" attribute ending with a digit:: + + >>> doc = """ + ...
+ ... + ...
+ ... """ + >>> sel = Selector(text=doc, type="html") + >>> sel.xpath('//li//@href').extract() + [u'link1.html', u'link2.html', u'link3.html', u'link4.html', u'link5.html'] + >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').extract() + [u'link1.html', u'link2.html', u'link4.html', u'link5.html'] + >>> + +.. warning:: C library ``libxslt`` doesn't natively support EXSLT regular + expressions so `lxml`_'s implementation uses hooks to Python's ``re`` module. + Thus, using regexp functions in your XPath expressions may add a small + performance penalty. + +Set operations +~~~~~~~~~~~~~~ + +These can be handy for excluding parts of a document tree before +extracting text elements for example. + +Example extracting microdata (sample content taken from http://schema.org/Product) +with groups of itemscopes and corresponding itemprops:: + + >>> doc = """ + ...
+ ... Kenmore White 17" Microwave + ... Kenmore 17" Microwave + ...
+ ... Rated 3.5/5 + ... based on 11 customer reviews + ...
+ ... + ...
+ ... $55.00 + ... In stock + ...
+ ... + ... Product description: + ... 0.7 cubic feet countertop microwave. + ... Has six preset cooking categories and convenience features like + ... Add-A-Minute and Child Lock. + ... + ... Customer reviews: + ... + ...
+ ... Not a happy camper - + ... by , + ... April 1, 2011 + ...
+ ... + ... 1/ + ... 5stars + ...
+ ... The lamp burned out and now I have to replace + ... it. + ...
+ ... + ...
+ ... Value purchase - + ... by , + ... March 25, 2011 + ...
+ ... + ... 4/ + ... 5stars + ...
+ ... Great microwave for the price. It is small and + ... fits in my apartment. + ...
+ ... ... + ...
+ ... """ + >>> + >>> for scope in sel.xpath('//div[@itemscope]'): + ... print "current scope:", scope.xpath('@itemtype').extract() + ... props = scope.xpath(''' + ... set:difference(./descendant::*/@itemprop, + ... .//*[@itemscope]/*/@itemprop)''') + ... print " properties:", props.extract() + ... print + ... + current scope: [u'http://schema.org/Product'] + properties: [u'name', u'aggregateRating', u'offers', u'description', u'review', u'review'] + + current scope: [u'http://schema.org/AggregateRating'] + properties: [u'ratingValue', u'reviewCount'] + + current scope: [u'http://schema.org/Offer'] + properties: [u'price', u'availability'] + + current scope: [u'http://schema.org/Review'] + properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description'] + + current scope: [u'http://schema.org/Rating'] + properties: [u'worstRating', u'ratingValue', u'bestRating'] + + current scope: [u'http://schema.org/Review'] + properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description'] + + current scope: [u'http://schema.org/Rating'] + properties: [u'worstRating', u'ratingValue', u'bestRating'] + + >>> + +Here we first iterate over ``itemscope`` elements, and for each one, +we look for all ``itemprops`` elements and exclude those that are themselves +inside another ``itemscope``. + +.. _EXSLT: http://www.exslt.org/ +.. _regular expressions: http://www.exslt.org/regexp/index.html +.. _set manipulation: http://www.exslt.org/set/index.html .. _topics-selectors-ref: diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index a316c20cf..790937af5 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -46,6 +46,17 @@ class Selector(object_ref): '__weakref__', '_parser', '_csstranslator', '_tostring_method'] _default_type = None + _default_namespaces = { + "re": "http://exslt.org/regular-expressions", + + # supported in libxslt: + # set:difference + # set:has-same-node + # set:intersection + # set:leading + # set:trailing + "set": "http://exslt.org/sets" + } def __init__(self, response=None, text=None, type=None, namespaces=None, _root=None, _expr=None): @@ -61,7 +72,9 @@ class Selector(object_ref): _root = LxmlDocument(response, self._parser) self.response = response - self.namespaces = namespaces + self.namespaces = dict(self._default_namespaces) + if namespaces is not None: + self.namespaces.update(namespaces) self._root = _root self._expr = _expr diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index 72f5ebdd5..d84e4bd47 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -400,3 +400,122 @@ class DeprecatedXpathSelectorTest(unittest.TestCase): self.assertEqual(xs.select("//div").extract(), [u'

Hello

']) self.assertRaises(RuntimeError, xs.css, 'div') + + +class ExsltTestCase(unittest.TestCase): + + sscls = Selector + + def test_regexp(self): + """EXSLT regular expression tests""" + body = """ +

+ + """ + response = TextResponse(url="http://example.com", body=body) + sel = self.sscls(response) + + # re:test() + self.assertEqual( + sel.xpath( + '//input[re:test(@name, "[A-Z]+", "i")]').extract(), + [x.extract() for x in sel.xpath('//input[re:test(@name, "[A-Z]+", "i")]')]) + self.assertEqual( + [x.extract() + for x in sel.xpath( + '//a[re:test(@href, "\.html$")]/text()')], + [u'first link', u'second link']) + self.assertEqual( + [x.extract() + for x in sel.xpath( + '//a[re:test(@href, "first")]/text()')], + [u'first link']) + self.assertEqual( + [x.extract() + for x in sel.xpath( + '//a[re:test(@href, "second")]/text()')], + [u'second link']) + + + # re:match() is rather special: it returns a node-set of nodes + #[u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml', + #u'http', + #u'www.bayes.co.uk', + #u'', + #u'/xml/index.xml?/xml/utils/rechecker.xml'] + self.assertEqual( + sel.xpath('re:match(//a[re:test(@href, "\.xml$")]/@href,' + '"(\w+):\/\/([^/:]+)(:\d*)?([^# ]*)")/text()').extract(), + [u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml', + u'http', + u'www.bayes.co.uk', + u'', + u'/xml/index.xml?/xml/utils/rechecker.xml']) + + + + # re:replace() + self.assertEqual( + sel.xpath('re:replace(//a[re:test(@href, "\.xml$")]/@href,' + '"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(), + [u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html']) + + def test_set(self): + """EXSLT set manipulation tests""" + # microdata example from http://schema.org/Event + body=""" +
+ + + + Thu, 04/21/16 + 8:00 p.m. + +
+ +
+ Philadelphia, + PA +
+
+ +
+ Priced from: $35 + 1938 tickets left +
+
+ """ + response = TextResponse(url="http://example.com", body=body) + sel = self.sscls(response) + + self.assertEqual( + sel.xpath('''//div[@itemtype="http://schema.org/Event"] + //@itemprop''').extract(), + [u'url', + u'name', + u'startDate', + u'location', + u'url', + u'address', + u'addressLocality', + u'addressRegion', + u'offers', + u'lowPrice', + u'offerCount'] + ) + + self.assertEqual(sel.xpath(''' + set:difference(//div[@itemtype="http://schema.org/Event"] + //@itemprop, + //div[@itemtype="http://schema.org/Event"] + //*[@itemscope]/*/@itemprop)''').extract(), + [u'url', u'name', u'startDate', u'location', u'offers'])