diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 3872736ae..c64ebad62 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -240,6 +240,149 @@ XPath specification.
.. _Location Paths: http://www.w3.org/TR/xpath#location-paths
+Using EXSLT extensions
+----------------------
+
+Being built atop `lxml`_, Scrapy selectors also support some `EXSLT`_ extensions
+and come with these pre-registered namespaces to use in XPath expressions:
+
+
+====== ==================================== =======================
+prefix namespace usage
+====== ==================================== =======================
+re http://exslt.org/regular-expressions `regular expressions`_
+set http://exslt.org/sets `set manipulation`_
+====== ==================================== =======================
+
+Regular expressions
+~~~~~~~~~~~~~~~~~~~
+
+The ``test()`` function for example can prove quite useful when XPath's
+``starts-with()`` or ``contains()`` are not sufficient.
+
+Example selecting links in list item with a "class" attribute ending with a digit::
+
+ >>> doc = """
+ ...
+ ... """
+ >>> sel = Selector(text=doc, type="html")
+ >>> sel.xpath('//li//@href').extract()
+ [u'link1.html', u'link2.html', u'link3.html', u'link4.html', u'link5.html']
+ >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').extract()
+ [u'link1.html', u'link2.html', u'link4.html', u'link5.html']
+ >>>
+
+.. warning:: C library ``libxslt`` doesn't natively support EXSLT regular
+ expressions so `lxml`_'s implementation uses hooks to Python's ``re`` module.
+ Thus, using regexp functions in your XPath expressions may add a small
+ performance penalty.
+
+Set operations
+~~~~~~~~~~~~~~
+
+These can be handy for excluding parts of a document tree before
+extracting text elements for example.
+
+Example extracting microdata (sample content taken from http://schema.org/Product)
+with groups of itemscopes and corresponding itemprops::
+
+ >>> doc = """
+ ...
+ ... Kenmore White 17" Microwave
+ ...
+ ...
+ ... Rated 3.5/5
+ ... based on 11 customer reviews
+ ...
+ ...
+ ...
+ ... $55.00
+ ... In stock
+ ...
+ ...
+ ... Product description:
+ ... 0.7 cubic feet countertop microwave.
+ ... Has six preset cooking categories and convenience features like
+ ... Add-A-Minute and Child Lock.
+ ...
+ ... Customer reviews:
+ ...
+ ...
+ ... Not a happy camper -
+ ... by Ellie,
+ ... April 1, 2011
+ ...
+ ...
+ ... 1/
+ ... 5stars
+ ...
+ ... The lamp burned out and now I have to replace
+ ... it.
+ ...
+ ...
+ ...
+ ... Value purchase -
+ ... by Lucas,
+ ... March 25, 2011
+ ...
+ ...
+ ... 4/
+ ... 5stars
+ ...
+ ... Great microwave for the price. It is small and
+ ... fits in my apartment.
+ ...
+ ... ...
+ ...
+ ... """
+ >>>
+ >>> for scope in sel.xpath('//div[@itemscope]'):
+ ... print "current scope:", scope.xpath('@itemtype').extract()
+ ... props = scope.xpath('''
+ ... set:difference(./descendant::*/@itemprop,
+ ... .//*[@itemscope]/*/@itemprop)''')
+ ... print " properties:", props.extract()
+ ... print
+ ...
+ current scope: [u'http://schema.org/Product']
+ properties: [u'name', u'aggregateRating', u'offers', u'description', u'review', u'review']
+
+ current scope: [u'http://schema.org/AggregateRating']
+ properties: [u'ratingValue', u'reviewCount']
+
+ current scope: [u'http://schema.org/Offer']
+ properties: [u'price', u'availability']
+
+ current scope: [u'http://schema.org/Review']
+ properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description']
+
+ current scope: [u'http://schema.org/Rating']
+ properties: [u'worstRating', u'ratingValue', u'bestRating']
+
+ current scope: [u'http://schema.org/Review']
+ properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description']
+
+ current scope: [u'http://schema.org/Rating']
+ properties: [u'worstRating', u'ratingValue', u'bestRating']
+
+ >>>
+
+Here we first iterate over ``itemscope`` elements, and for each one,
+we look for all ``itemprops`` elements and exclude those that are themselves
+inside another ``itemscope``.
+
+.. _EXSLT: http://www.exslt.org/
+.. _regular expressions: http://www.exslt.org/regexp/index.html
+.. _set manipulation: http://www.exslt.org/set/index.html
.. _topics-selectors-ref:
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index a316c20cf..790937af5 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -46,6 +46,17 @@ class Selector(object_ref):
'__weakref__', '_parser', '_csstranslator', '_tostring_method']
_default_type = None
+ _default_namespaces = {
+ "re": "http://exslt.org/regular-expressions",
+
+ # supported in libxslt:
+ # set:difference
+ # set:has-same-node
+ # set:intersection
+ # set:leading
+ # set:trailing
+ "set": "http://exslt.org/sets"
+ }
def __init__(self, response=None, text=None, type=None, namespaces=None,
_root=None, _expr=None):
@@ -61,7 +72,9 @@ class Selector(object_ref):
_root = LxmlDocument(response, self._parser)
self.response = response
- self.namespaces = namespaces
+ self.namespaces = dict(self._default_namespaces)
+ if namespaces is not None:
+ self.namespaces.update(namespaces)
self._root = _root
self._expr = _expr
diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py
index 72f5ebdd5..d84e4bd47 100644
--- a/scrapy/tests/test_selector.py
+++ b/scrapy/tests/test_selector.py
@@ -400,3 +400,122 @@ class DeprecatedXpathSelectorTest(unittest.TestCase):
self.assertEqual(xs.select("//div").extract(),
[u'