From d46534cc547b4da79f9b47b4ccec40a16f1ca3c2 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Sun, 24 Nov 2013 04:41:14 +0100
Subject: [PATCH 1/7] Register EXSLT namespaces by default (resolves #470)

---
 scrapy/selector/unified.py    | 44 ++++++++++++++++++++++++++++++-
 scrapy/tests/test_selector.py | 49 +++++++++++++++++++++++++++++++++++
 2 files changed, 92 insertions(+), 1 deletion(-)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index a316c20cf..1646bb6c6 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -3,6 +3,7 @@ XPath selectors based on lxml
 """
 
 from lxml import etree
+import copy
 
 from scrapy.utils.misc import extract_regex
 from scrapy.utils.trackref import object_ref
@@ -46,6 +47,45 @@ class Selector(object_ref):
                  '__weakref__', '_parser', '_csstranslator', '_tostring_method']
 
     _default_type = None
+    _default_namespaces = {
+        "regexp": "http://exslt.org/regular-expressions",
+
+        # supported in libxslt:
+        # set:difference
+        # set:has-same-node
+        # set:intersection
+        # set:leading
+        # set:trailing
+        "set": "http://exslt.org/sets",
+
+        # supported in libxslt:
+        # math:abs()
+        # math:acos()
+        # math:asin()
+        # math:atan()
+        # math:atan2()
+        # math:constant()
+        # math:cos()
+        # math:exp()
+        # math:highest()
+        # math:log()
+        # math:lowest()
+        # math:max()
+        # math:min()
+        # math:power()
+        # math:random()
+        # math:sin()
+        # math:sqrt()
+        # math:tan()
+        "math": "http://exslt.org/math",
+
+        # supported in libxslt:
+        # str:align
+        # str:concat
+        # str:padding
+        # str:tokenize
+        "str": "http://exslt.org/strings",
+    }
 
     def __init__(self, response=None, text=None, type=None, namespaces=None,
                  _root=None, _expr=None):
@@ -61,7 +101,9 @@ class Selector(object_ref):
             _root = LxmlDocument(response, self._parser)
 
         self.response = response
-        self.namespaces = namespaces
+        ns = copy.copy(self._default_namespaces)
+        ns.update(namespaces or {})
+        self.namespaces = ns
         self._root = _root
         self._expr = _expr
 
diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py
index 5a14031fe..a2eaa0e58 100644
--- a/scrapy/tests/test_selector.py
+++ b/scrapy/tests/test_selector.py
@@ -333,3 +333,52 @@ class DeprecatedXpathSelectorTest(unittest.TestCase):
             self.assertEqual(xs.select("//div").extract(),
                              [u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
             self.assertRaises(RuntimeError, xs.css, 'div')
+
+
+
+class ExsltTestCase(unittest.TestCase):
+
+    sscls = Selector
+
+    def test_regexp(self):
+        """EXSLT regular expression tests"""
+        body = """
+        <p><input name='a' value='1'/><input name='b' value='2'/></p>
+        <div class="links">
+        <a href="/first.html">first link</a>
+        <a href="/second.html">second link</a>
+        <a href="http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml">EXSLT match example</a>
+        </div>
+        """
+        response = TextResponse(url="http://example.com", body=body)
+        sel = self.sscls(response)
+
+        # regexp:test()
+        self.assertEqual(sel.xpath('//input[regexp:test(@name, "[A-Z]+", "i")]').extract(),
+                         [x.extract() for x in sel.xpath('//input[regexp:test(@name, "[A-Z]+", "i")]')])
+        self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "\.html$")]/text()')],
+                         [u'first link', u'second link'])
+        self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "first")]/text()')],
+                         [u'first link'])
+        self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "second")]/text()')],
+                         [u'second link'])
+
+        # regexp:match() is rather special: it returns a node-set of <match> nodes
+        #[u'<match>http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml</match>',
+        #u'<match>http</match>',
+        #u'<match>www.bayes.co.uk</match>',
+        #u'<match></match>',
+        #u'<match>/xml/index.xml?/xml/utils/rechecker.xml</match>']
+        self.assertEqual(sel.xpath(''
+            'regexp:match(//a[regexp:test(@href, "\.xml$")]/@href,'
+            '"(\w+):\/\/([^/:]+)(:\d*)?([^# ]*)")/text()').extract(),
+                         [u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml',
+                             u'http',
+                             u'www.bayes.co.uk',
+                             u'',
+                             u'/xml/index.xml?/xml/utils/rechecker.xml'])
+
+        # regexp:replace()
+        self.assertEqual(sel.xpath('regexp:replace(//a[regexp:test(@href, "\.xml$")]/@href,'
+            '"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(),
+                         [u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html'])

From 5df56a975884ed0680283ff6e301f8a4d32f4712 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 8 Jan 2014 18:48:20 +0100
Subject: [PATCH 2/7] Applying suggestions from previous comments

---
 scrapy/selector/unified.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 1646bb6c6..4939c596f 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -3,7 +3,6 @@ XPath selectors based on lxml
 """
 
 from lxml import etree
-import copy
 
 from scrapy.utils.misc import extract_regex
 from scrapy.utils.trackref import object_ref
@@ -101,9 +100,9 @@ class Selector(object_ref):
             _root = LxmlDocument(response, self._parser)
 
         self.response = response
-        ns = copy.copy(self._default_namespaces)
-        ns.update(namespaces or {})
-        self.namespaces = ns
+        self.namespaces = dict(self._default_namespaces)
+        if namespaces is not None:
+            self.namespaces.update(namespaces)
         self._root = _root
         self._expr = _expr
 

From 29fc9f3466a0ca50012f253fff27573a929d7275 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Thu, 9 Jan 2014 19:08:34 +0100
Subject: [PATCH 3/7] Update selectors documentation and tests

---
 docs/topics/selectors.rst     | 155 ++++++++++++++++++++++++++++++++++
 scrapy/tests/test_selector.py |  55 ++++++++++++
 2 files changed, 210 insertions(+)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 3872736ae..d192498f3 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -240,6 +240,161 @@ XPath specification.
 
 .. _Location Paths: http://www.w3.org/TR/xpath#location-paths
 
+Using EXSLT extensions
+----------------------
+
+Being built atop `lxml`_, Scrapy selectors also support some `EXSLT`_ extensions
+and come with these pre-registered namespaces to use in XPath expressions:
+
+
+======  ====================================    =======================
+prefix  namespace                               usage
+======  ====================================    =======================
+regexp  http://exslt.org/regular-expressions    `regular expressions`_
+set     http://exslt.org/sets                   `set manipulation`_
+str     http://exslt.org/strings                `string manipulations`_
+math    http://exslt.org/math                   `mathematical operations`_
+======  ====================================    =======================
+
+Regular expressions
+~~~~~~~~~~~~~~~~~~~
+
+The ``test()`` function for example can prove quite useful when XPath's
+``starts-with()`` or ``contains()`` are not sufficient.
+
+Example selecting links in list item with a "class" attribute ending with a digit::
+
+    >>> doc = """
+    ... <div>
+    ...     <ul>
+    ...         <li class="item-0"><a href="link1.html">first item</a></li>
+    ...         <li class="item-1"><a href="link2.html">second item</a></li>
+    ...         <li class="item-inactive"><a href="link3.html">third item</a></li>
+    ...         <li class="item-1"><a href="link4.html">fourth item</a></li>
+    ...         <li class="item-0"><a href="link5.html">fifth item</a></li>
+    ...     </ul>
+    ... </div>
+    ... """
+    >>> sel = Selector(text=doc, type="html")
+    >>> sel.xpath('//li//@href').extract()
+    [u'link1.html', u'link2.html', u'link3.html', u'link4.html', u'link5.html']
+    >>> sel.xpath('//li[regexp:test(@class, "item-\d$")]//@href').extract()
+    [u'link1.html', u'link2.html', u'link4.html', u'link5.html']
+    >>>
+
+
+
+Set operations
+~~~~~~~~~~~~~~
+
+These can be handy for excluding parts of a document tree before
+extracting text elements for example.
+
+Example extracting midrodata (sample content taken from http://schema.org/Product)
+with groups of itemscopes and corresponding itemprops::
+
+    >>> doc = """
+    ... <div itemscope itemtype="http://schema.org/Product">
+    ...   <span itemprop="name">Kenmore White 17" Microwave</span>
+    ...   <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />
+    ...   <div itemprop="aggregateRating"
+    ...     itemscope itemtype="http://schema.org/AggregateRating">
+    ...    Rated <span itemprop="ratingValue">3.5</span>/5
+    ...    based on <span itemprop="reviewCount">11</span> customer reviews
+    ...   </div>
+    ...
+    ...   <div itemprop="offers" itemscope itemtype="http://schema.org/Offer">
+    ...     <span itemprop="price">$55.00</span>
+    ...     <link itemprop="availability" href="http://schema.org/InStock" />In stock
+    ...   </div>
+    ...
+    ...   Product description:
+    ...   <span itemprop="description">0.7 cubic feet countertop microwave.
+    ...   Has six preset cooking categories and convenience features like
+    ...   Add-A-Minute and Child Lock.</span>
+    ...
+    ...   Customer reviews:
+    ...
+    ...   <div itemprop="review" itemscope itemtype="http://schema.org/Review">
+    ...     <span itemprop="name">Not a happy camper</span> -
+    ...     by <span itemprop="author">Ellie</span>,
+    ...     <meta itemprop="datePublished" content="2011-04-01">April 1, 2011
+    ...     <div itemprop="reviewRating" itemscope itemtype="http://schema.org/Rating">
+    ...       <meta itemprop="worstRating" content = "1">
+    ...       <span itemprop="ratingValue">1</span>/
+    ...       <span itemprop="bestRating">5</span>stars
+    ...     </div>
+    ...     <span itemprop="description">The lamp burned out and now I have to replace
+    ...     it. </span>
+    ...   </div>
+    ...
+    ...   <div itemprop="review" itemscope itemtype="http://schema.org/Review">
+    ...     <span itemprop="name">Value purchase</span> -
+    ...     by <span itemprop="author">Lucas</span>,
+    ...     <meta itemprop="datePublished" content="2011-03-25">March 25, 2011
+    ...     <div itemprop="reviewRating" itemscope itemtype="http://schema.org/Rating">
+    ...       <meta itemprop="worstRating" content = "1"/>
+    ...       <span itemprop="ratingValue">4</span>/
+    ...       <span itemprop="bestRating">5</span>stars
+    ...     </div>
+    ...     <span itemprop="description">Great microwave for the price. It is small and
+    ...     fits in my apartment.</span>
+    ...   </div>
+    ...   ...
+    ... </div>
+    ... """
+    >>>
+    >>> for scope in sel.xpath('//div[@itemscope]'):
+    ...     print "current scope:", scope.xpath('@itemtype').extract()
+    ...     props = scope.xpath('''
+    ...                 set:difference(./descendant::*/@itemprop,
+    ...                                .//*[@itemscope]/*/@itemprop)''')
+    ...     print "    properties:", props.extract()
+    ...     print
+    ...
+    current scope: [u'http://schema.org/Product']
+        properties: [u'name', u'aggregateRating', u'offers', u'description', u'review', u'review']
+
+    current scope: [u'http://schema.org/AggregateRating']
+        properties: [u'ratingValue', u'reviewCount']
+
+    current scope: [u'http://schema.org/Offer']
+        properties: [u'price', u'availability']
+
+    current scope: [u'http://schema.org/Review']
+        properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description']
+
+    current scope: [u'http://schema.org/Rating']
+        properties: [u'worstRating', u'ratingValue', u'bestRating']
+
+    current scope: [u'http://schema.org/Review']
+        properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description']
+
+    current scope: [u'http://schema.org/Rating']
+        properties: [u'worstRating', u'ratingValue', u'bestRating']
+
+    >>>
+
+Here we first iterate over ``itemscope`` elements, and for each one,
+we look for all ``itemprops`` elements and exclude those that are themselves
+inside another ``itemscope``.
+
+Maths
+~~~~~
+
+Not that useful in practice, but you never know.
+
+String manipulation
+~~~~~~~~~~~~~~~~~~~
+
+In practive, Python's string manipulation outside XPath is usually more
+powerful.
+
+.. _EXSLT: http://www.exslt.org/
+.. _regular expressions: http://www.exslt.org/regexp/index.html
+.. _set manipulation: http://www.exslt.org/set/index.html
+.. _mathematical operations: http://www.exslt.org/math/index.html
+.. _string manipulations: http://www.exslt.org/str/index.html
 
 .. _topics-selectors-ref:
 
diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py
index a2eaa0e58..8823520d1 100644
--- a/scrapy/tests/test_selector.py
+++ b/scrapy/tests/test_selector.py
@@ -382,3 +382,58 @@ class ExsltTestCase(unittest.TestCase):
         self.assertEqual(sel.xpath('regexp:replace(//a[regexp:test(@href, "\.xml$")]/@href,'
             '"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(),
                          [u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html'])
+
+    def test_set(self):
+        """EXSLT set manipulation tests"""
+        # microdata example from http://schema.org/Event
+        body="""
+        <div itemscope itemtype="http://schema.org/Event">
+          <a itemprop="url" href="nba-miami-philidelphia-game3.html">
+          NBA Eastern Conference First Round Playoff Tickets:
+          <span itemprop="name"> Miami Heat at Philadelphia 76ers - Game 3 (Home Game 1) </span>
+          </a>
+
+          <meta itemprop="startDate" content="2016-04-21T20:00">
+            Thu, 04/21/16
+            8:00 p.m.
+
+          <div itemprop="location" itemscope itemtype="http://schema.org/Place">
+            <a itemprop="url" href="wells-fargo-center.html">
+            Wells Fargo Center
+            </a>
+            <div itemprop="address" itemscope itemtype="http://schema.org/PostalAddress">
+              <span itemprop="addressLocality">Philadelphia</span>,
+              <span itemprop="addressRegion">PA</span>
+            </div>
+          </div>
+
+          <div itemprop="offers" itemscope itemtype="http://schema.org/AggregateOffer">
+            Priced from: <span itemprop="lowPrice">$35</span>
+            <span itemprop="offerCount">1938</span> tickets left
+          </div>
+        </div>
+        """
+        response = TextResponse(url="http://example.com", body=body)
+        sel = self.sscls(response)
+
+        self.assertEqual(
+            sel.xpath('''//div[@itemtype="http://schema.org/Event"]
+                            //@itemprop''').extract(),
+            [u'url',
+             u'name',
+             u'startDate',
+             u'location',
+             u'url',
+             u'address',
+             u'addressLocality',
+             u'addressRegion',
+             u'offers',
+             u'lowPrice',
+             u'offerCount']
+)
+        self.assertEqual(sel.xpath('''
+                set:difference(//div[@itemtype="http://schema.org/Event"]
+                                    //@itemprop,
+                               //div[@itemtype="http://schema.org/Event"]
+                                    //*[@itemscope]/*/@itemprop)''').extract(),
+            [u'url', u'name', u'startDate', u'location', u'offers'])

From 2cc26e6f561875ffc294f135369d23b6a295bfb8 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 14 Jan 2014 13:09:18 +0100
Subject: [PATCH 4/7] Fix typo error

---
 docs/topics/selectors.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index d192498f3..373d25080 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -290,7 +290,7 @@ Set operations
 These can be handy for excluding parts of a document tree before
 extracting text elements for example.
 
-Example extracting midrodata (sample content taken from http://schema.org/Product)
+Example extracting microdata (sample content taken from http://schema.org/Product)
 with groups of itemscopes and corresponding itemprops::
 
     >>> doc = """

From a3eba68aca804e67f9a083c4d9c490da9a243b59 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 15 Jan 2014 12:28:25 +0100
Subject: [PATCH 5/7] Drop EXSLT strings and math extensions

---
 docs/topics/selectors.rst     | 15 ---------
 scrapy/selector/unified.py    | 30 +-----------------
 scrapy/tests/test_selector.py | 59 ++++++++++++++++++++++-------------
 3 files changed, 38 insertions(+), 66 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 373d25080..deff0d8dc 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -252,8 +252,6 @@ prefix  namespace                               usage
 ======  ====================================    =======================
 regexp  http://exslt.org/regular-expressions    `regular expressions`_
 set     http://exslt.org/sets                   `set manipulation`_
-str     http://exslt.org/strings                `string manipulations`_
-math    http://exslt.org/math                   `mathematical operations`_
 ======  ====================================    =======================
 
 Regular expressions
@@ -379,22 +377,9 @@ Here we first iterate over ``itemscope`` elements, and for each one,
 we look for all ``itemprops`` elements and exclude those that are themselves
 inside another ``itemscope``.
 
-Maths
-~~~~~
-
-Not that useful in practice, but you never know.
-
-String manipulation
-~~~~~~~~~~~~~~~~~~~
-
-In practive, Python's string manipulation outside XPath is usually more
-powerful.
-
 .. _EXSLT: http://www.exslt.org/
 .. _regular expressions: http://www.exslt.org/regexp/index.html
 .. _set manipulation: http://www.exslt.org/set/index.html
-.. _mathematical operations: http://www.exslt.org/math/index.html
-.. _string manipulations: http://www.exslt.org/str/index.html
 
 .. _topics-selectors-ref:
 
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 4939c596f..6f502ce56 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -55,35 +55,7 @@ class Selector(object_ref):
         # set:intersection
         # set:leading
         # set:trailing
-        "set": "http://exslt.org/sets",
-
-        # supported in libxslt:
-        # math:abs()
-        # math:acos()
-        # math:asin()
-        # math:atan()
-        # math:atan2()
-        # math:constant()
-        # math:cos()
-        # math:exp()
-        # math:highest()
-        # math:log()
-        # math:lowest()
-        # math:max()
-        # math:min()
-        # math:power()
-        # math:random()
-        # math:sin()
-        # math:sqrt()
-        # math:tan()
-        "math": "http://exslt.org/math",
-
-        # supported in libxslt:
-        # str:align
-        # str:concat
-        # str:padding
-        # str:tokenize
-        "str": "http://exslt.org/strings",
+        "set": "http://exslt.org/sets"
     }
 
     def __init__(self, response=None, text=None, type=None, namespaces=None,
diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py
index 8823520d1..e5035688d 100644
--- a/scrapy/tests/test_selector.py
+++ b/scrapy/tests/test_selector.py
@@ -335,7 +335,6 @@ class DeprecatedXpathSelectorTest(unittest.TestCase):
             self.assertRaises(RuntimeError, xs.css, 'div')
 
 
-
 class ExsltTestCase(unittest.TestCase):
 
     sscls = Selector
@@ -354,14 +353,26 @@ class ExsltTestCase(unittest.TestCase):
         sel = self.sscls(response)
 
         # regexp:test()
-        self.assertEqual(sel.xpath('//input[regexp:test(@name, "[A-Z]+", "i")]').extract(),
-                         [x.extract() for x in sel.xpath('//input[regexp:test(@name, "[A-Z]+", "i")]')])
-        self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "\.html$")]/text()')],
-                         [u'first link', u'second link'])
-        self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "first")]/text()')],
-                         [u'first link'])
-        self.assertEqual([x.extract() for x in sel.xpath('//a[regexp:test(@href, "second")]/text()')],
-                         [u'second link'])
+        self.assertEqual(
+            sel.xpath(
+                '//input[regexp:test(@name, "[A-Z]+", "i")]').extract(),
+            [x.extract() for x in sel.xpath('//input[regexp:test(@name, "[A-Z]+", "i")]')])
+        self.assertEqual(
+            [x.extract()
+             for x in sel.xpath(
+                 '//a[regexp:test(@href, "\.html$")]/text()')],
+            [u'first link', u'second link'])
+        self.assertEqual(
+            [x.extract()
+             for x in sel.xpath(
+                 '//a[regexp:test(@href, "first")]/text()')],
+            [u'first link'])
+        self.assertEqual(
+            [x.extract()
+             for x in sel.xpath(
+                 '//a[regexp:test(@href, "second")]/text()')],
+            [u'second link'])
+
 
         # regexp:match() is rather special: it returns a node-set of <match> nodes
         #[u'<match>http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml</match>',
@@ -369,19 +380,22 @@ class ExsltTestCase(unittest.TestCase):
         #u'<match>www.bayes.co.uk</match>',
         #u'<match></match>',
         #u'<match>/xml/index.xml?/xml/utils/rechecker.xml</match>']
-        self.assertEqual(sel.xpath(''
-            'regexp:match(//a[regexp:test(@href, "\.xml$")]/@href,'
-            '"(\w+):\/\/([^/:]+)(:\d*)?([^# ]*)")/text()').extract(),
-                         [u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml',
-                             u'http',
-                             u'www.bayes.co.uk',
-                             u'',
-                             u'/xml/index.xml?/xml/utils/rechecker.xml'])
+        self.assertEqual(
+            sel.xpath('regexp:match(//a[regexp:test(@href, "\.xml$")]/@href,'
+                      '"(\w+):\/\/([^/:]+)(:\d*)?([^# ]*)")/text()').extract(),
+            [u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml',
+             u'http',
+             u'www.bayes.co.uk',
+             u'',
+             u'/xml/index.xml?/xml/utils/rechecker.xml'])
+
+
 
         # regexp:replace()
-        self.assertEqual(sel.xpath('regexp:replace(//a[regexp:test(@href, "\.xml$")]/@href,'
-            '"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(),
-                         [u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html'])
+        self.assertEqual(
+            sel.xpath('regexp:replace(//a[regexp:test(@href, "\.xml$")]/@href,'
+                      '"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(),
+            [u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html'])
 
     def test_set(self):
         """EXSLT set manipulation tests"""
@@ -430,10 +444,11 @@ class ExsltTestCase(unittest.TestCase):
              u'offers',
              u'lowPrice',
              u'offerCount']
-)
+        )
+
         self.assertEqual(sel.xpath('''
                 set:difference(//div[@itemtype="http://schema.org/Event"]
                                     //@itemprop,
                                //div[@itemtype="http://schema.org/Event"]
                                     //*[@itemscope]/*/@itemprop)''').extract(),
-            [u'url', u'name', u'startDate', u'location', u'offers'])
+                         [u'url', u'name', u'startDate', u'location', u'offers'])

From 88c8a523a7653a80d5f2f88b33e9d455a42402b7 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 15 Jan 2014 12:52:10 +0100
Subject: [PATCH 6/7] Add warning in docs on performance when using EXSLT
 regexp functions

---
 docs/topics/selectors.rst | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index deff0d8dc..af48a1b07 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -280,7 +280,10 @@ Example selecting links in list item with a "class" attribute ending with a digi
     [u'link1.html', u'link2.html', u'link4.html', u'link5.html']
     >>>
 
-
+.. warning:: C library ``libxslt`` doesn't natively support EXSLT regular
+    expressions so `lxml`_'s implementation uses hooks to Python's ``re`` module.
+    Thus, using regexp functions in your XPath expressions may add a small
+    performance penalty.
 
 Set operations
 ~~~~~~~~~~~~~~

From 827c0cf51f47ab87b1de1c517ee7d5095fdb9c32 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 15 Jan 2014 15:00:25 +0100
Subject: [PATCH 7/7] Rename "regexp" prefix to "re"

---
 docs/topics/selectors.rst     |  4 ++--
 scrapy/selector/unified.py    |  2 +-
 scrapy/tests/test_selector.py | 20 ++++++++++----------
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index af48a1b07..c64ebad62 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -250,7 +250,7 @@ and come with these pre-registered namespaces to use in XPath expressions:
 ======  ====================================    =======================
 prefix  namespace                               usage
 ======  ====================================    =======================
-regexp  http://exslt.org/regular-expressions    `regular expressions`_
+re      http://exslt.org/regular-expressions    `regular expressions`_
 set     http://exslt.org/sets                   `set manipulation`_
 ======  ====================================    =======================
 
@@ -276,7 +276,7 @@ Example selecting links in list item with a "class" attribute ending with a digi
     >>> sel = Selector(text=doc, type="html")
     >>> sel.xpath('//li//@href').extract()
     [u'link1.html', u'link2.html', u'link3.html', u'link4.html', u'link5.html']
-    >>> sel.xpath('//li[regexp:test(@class, "item-\d$")]//@href').extract()
+    >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').extract()
     [u'link1.html', u'link2.html', u'link4.html', u'link5.html']
     >>>
 
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 6f502ce56..790937af5 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -47,7 +47,7 @@ class Selector(object_ref):
 
     _default_type = None
     _default_namespaces = {
-        "regexp": "http://exslt.org/regular-expressions",
+        "re": "http://exslt.org/regular-expressions",
 
         # supported in libxslt:
         # set:difference
diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py
index e5035688d..f340c73d8 100644
--- a/scrapy/tests/test_selector.py
+++ b/scrapy/tests/test_selector.py
@@ -352,36 +352,36 @@ class ExsltTestCase(unittest.TestCase):
         response = TextResponse(url="http://example.com", body=body)
         sel = self.sscls(response)
 
-        # regexp:test()
+        # re:test()
         self.assertEqual(
             sel.xpath(
-                '//input[regexp:test(@name, "[A-Z]+", "i")]').extract(),
-            [x.extract() for x in sel.xpath('//input[regexp:test(@name, "[A-Z]+", "i")]')])
+                '//input[re:test(@name, "[A-Z]+", "i")]').extract(),
+            [x.extract() for x in sel.xpath('//input[re:test(@name, "[A-Z]+", "i")]')])
         self.assertEqual(
             [x.extract()
              for x in sel.xpath(
-                 '//a[regexp:test(@href, "\.html$")]/text()')],
+                 '//a[re:test(@href, "\.html$")]/text()')],
             [u'first link', u'second link'])
         self.assertEqual(
             [x.extract()
              for x in sel.xpath(
-                 '//a[regexp:test(@href, "first")]/text()')],
+                 '//a[re:test(@href, "first")]/text()')],
             [u'first link'])
         self.assertEqual(
             [x.extract()
              for x in sel.xpath(
-                 '//a[regexp:test(@href, "second")]/text()')],
+                 '//a[re:test(@href, "second")]/text()')],
             [u'second link'])
 
 
-        # regexp:match() is rather special: it returns a node-set of <match> nodes
+        # re:match() is rather special: it returns a node-set of <match> nodes
         #[u'<match>http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml</match>',
         #u'<match>http</match>',
         #u'<match>www.bayes.co.uk</match>',
         #u'<match></match>',
         #u'<match>/xml/index.xml?/xml/utils/rechecker.xml</match>']
         self.assertEqual(
-            sel.xpath('regexp:match(//a[regexp:test(@href, "\.xml$")]/@href,'
+            sel.xpath('re:match(//a[re:test(@href, "\.xml$")]/@href,'
                       '"(\w+):\/\/([^/:]+)(:\d*)?([^# ]*)")/text()').extract(),
             [u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml',
              u'http',
@@ -391,9 +391,9 @@ class ExsltTestCase(unittest.TestCase):
 
 
 
-        # regexp:replace()
+        # re:replace()
         self.assertEqual(
-            sel.xpath('regexp:replace(//a[regexp:test(@href, "\.xml$")]/@href,'
+            sel.xpath('re:replace(//a[re:test(@href, "\.xml$")]/@href,'
                       '"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(),
             [u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html'])