diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 3872736ae..d192498f3 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -240,6 +240,161 @@ XPath specification. .. _Location Paths: http://www.w3.org/TR/xpath#location-paths +Using EXSLT extensions +---------------------- + +Being built atop `lxml`_, Scrapy selectors also support some `EXSLT`_ extensions +and come with these pre-registered namespaces to use in XPath expressions: + + +====== ==================================== ======================= +prefix namespace usage +====== ==================================== ======================= +regexp http://exslt.org/regular-expressions `regular expressions`_ +set http://exslt.org/sets `set manipulation`_ +str http://exslt.org/strings `string manipulations`_ +math http://exslt.org/math `mathematical operations`_ +====== ==================================== ======================= + +Regular expressions +~~~~~~~~~~~~~~~~~~~ + +The ``test()`` function for example can prove quite useful when XPath's +``starts-with()`` or ``contains()`` are not sufficient. + +Example selecting links in list item with a "class" attribute ending with a digit:: + + >>> doc = """ + ...
+ ... + ...
+ ... """ + >>> sel = Selector(text=doc, type="html") + >>> sel.xpath('//li//@href').extract() + [u'link1.html', u'link2.html', u'link3.html', u'link4.html', u'link5.html'] + >>> sel.xpath('//li[regexp:test(@class, "item-\d$")]//@href').extract() + [u'link1.html', u'link2.html', u'link4.html', u'link5.html'] + >>> + + + +Set operations +~~~~~~~~~~~~~~ + +These can be handy for excluding parts of a document tree before +extracting text elements for example. + +Example extracting midrodata (sample content taken from http://schema.org/Product) +with groups of itemscopes and corresponding itemprops:: + + >>> doc = """ + ...
+ ... Kenmore White 17" Microwave + ... Kenmore 17" Microwave + ...
+ ... Rated 3.5/5 + ... based on 11 customer reviews + ...
+ ... + ...
+ ... $55.00 + ... In stock + ...
+ ... + ... Product description: + ... 0.7 cubic feet countertop microwave. + ... Has six preset cooking categories and convenience features like + ... Add-A-Minute and Child Lock. + ... + ... Customer reviews: + ... + ...
+ ... Not a happy camper - + ... by , + ... April 1, 2011 + ...
+ ... + ... 1/ + ... 5stars + ...
+ ... The lamp burned out and now I have to replace + ... it. + ...
+ ... + ...
+ ... Value purchase - + ... by , + ... March 25, 2011 + ...
+ ... + ... 4/ + ... 5stars + ...
+ ... Great microwave for the price. It is small and + ... fits in my apartment. + ...
+ ... ... + ...
+ ... """ + >>> + >>> for scope in sel.xpath('//div[@itemscope]'): + ... print "current scope:", scope.xpath('@itemtype').extract() + ... props = scope.xpath(''' + ... set:difference(./descendant::*/@itemprop, + ... .//*[@itemscope]/*/@itemprop)''') + ... print " properties:", props.extract() + ... print + ... + current scope: [u'http://schema.org/Product'] + properties: [u'name', u'aggregateRating', u'offers', u'description', u'review', u'review'] + + current scope: [u'http://schema.org/AggregateRating'] + properties: [u'ratingValue', u'reviewCount'] + + current scope: [u'http://schema.org/Offer'] + properties: [u'price', u'availability'] + + current scope: [u'http://schema.org/Review'] + properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description'] + + current scope: [u'http://schema.org/Rating'] + properties: [u'worstRating', u'ratingValue', u'bestRating'] + + current scope: [u'http://schema.org/Review'] + properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description'] + + current scope: [u'http://schema.org/Rating'] + properties: [u'worstRating', u'ratingValue', u'bestRating'] + + >>> + +Here we first iterate over ``itemscope`` elements, and for each one, +we look for all ``itemprops`` elements and exclude those that are themselves +inside another ``itemscope``. + +Maths +~~~~~ + +Not that useful in practice, but you never know. + +String manipulation +~~~~~~~~~~~~~~~~~~~ + +In practive, Python's string manipulation outside XPath is usually more +powerful. + +.. _EXSLT: http://www.exslt.org/ +.. _regular expressions: http://www.exslt.org/regexp/index.html +.. _set manipulation: http://www.exslt.org/set/index.html +.. _mathematical operations: http://www.exslt.org/math/index.html +.. _string manipulations: http://www.exslt.org/str/index.html .. _topics-selectors-ref: diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index a2eaa0e58..8823520d1 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -382,3 +382,58 @@ class ExsltTestCase(unittest.TestCase): self.assertEqual(sel.xpath('regexp:replace(//a[regexp:test(@href, "\.xml$")]/@href,' '"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(), [u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html']) + + def test_set(self): + """EXSLT set manipulation tests""" + # microdata example from http://schema.org/Event + body=""" +
+ + + + Thu, 04/21/16 + 8:00 p.m. + +
+ +
+ Philadelphia, + PA +
+
+ +
+ Priced from: $35 + 1938 tickets left +
+
+ """ + response = TextResponse(url="http://example.com", body=body) + sel = self.sscls(response) + + self.assertEqual( + sel.xpath('''//div[@itemtype="http://schema.org/Event"] + //@itemprop''').extract(), + [u'url', + u'name', + u'startDate', + u'location', + u'url', + u'address', + u'addressLocality', + u'addressRegion', + u'offers', + u'lowPrice', + u'offerCount'] +) + self.assertEqual(sel.xpath(''' + set:difference(//div[@itemtype="http://schema.org/Event"] + //@itemprop, + //div[@itemtype="http://schema.org/Event"] + //*[@itemscope]/*/@itemprop)''').extract(), + [u'url', u'name', u'startDate', u'location', u'offers'])