diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 39ec9b73c..43370d479 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -283,6 +283,40 @@ XPath specification. .. _Location Paths: https://www.w3.org/TR/xpath#location-paths +.. _topics-selectors-xpath-variables: + +Variables in XPath expressions +------------------------------ + +XPath allows you to reference variables in your XPath expressions, using +the ``$somevariable`` syntax. This is somewhat similar to parameterized +queries or prepared statements in the SQL world where you replace +some arguments in your queries with placeholders like ``?``, +which are then substituted with values passed with the query. + +Here's an example to match an element based on its "id" attribute value, +without hard-coding it (that was shown previously):: + + >>> # `$val` used in the expression, a `val` argument needs to be passed + >>> response.xpath('//div[@id=$val]/a/text()', val='images').extract_first() + u'Name: My image 1 ' + +Here's another example, to find the "id" attribute of a ``
`` tag containing +five ```` children (here we pass the value ``5`` as an integer):: + + >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).extract_first() + u'images' + +All variable references must have a binding value when calling ``.xpath()`` +(otherwise you'll get a ``ValueError: XPath error:`` exception). +This is done by passing as many named arguments as necessary. + +`parsel`_, the library powering Scrapy selectors, has more details and examples +on `XPath variables`_. + +.. _parsel: https://parsel.readthedocs.io/ +.. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions + Using EXSLT extensions ---------------------- diff --git a/requirements.txt b/requirements.txt index 64b6e771c..f92603d3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ queuelib six>=1.5.2 PyDispatcher>=2.0.5 service_identity -parsel>=0.9.5 +parsel>=1.1 diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index afa430329..5a6507aa8 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -111,8 +111,8 @@ class TextResponse(Response): self._cached_selector = Selector(self) return self._cached_selector - def xpath(self, query): - return self.selector.xpath(query) + def xpath(self, query, **kwargs): + return self.selector.xpath(query, **kwargs) def css(self, query): return self.selector.css(query) diff --git a/setup.py b/setup.py index 388cf0dec..a6e6f9615 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ setup( 'pyOpenSSL', 'cssselect>=0.9', 'six>=1.5.2', - 'parsel>=0.9.5', + 'parsel>=1.1', 'PyDispatcher>=2.0.5', 'service_identity', ], diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 7624aa4c4..9df3bf6e7 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -320,6 +320,20 @@ class TextResponseTest(BaseResponseTest): response.selector.css("title::text").extract(), ) + def test_selector_shortcuts_kwargs(self): + body = b"Some page

A nice paragraph.

" + response = self.response_class("http://www.example.com", body=body) + + self.assertEqual( + response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(), + response.xpath("normalize-space(//p[@class=\"content\"])").extract(), + ) + self.assertEqual( + response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()", + pclass="content", pcount=1).extract(), + response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(), + ) + def test_urljoin_with_base_url(self): """Test urljoin shortcut which also evaluates base-url through get_base_url().""" body = b'' @@ -428,3 +442,21 @@ class XmlResponseTest(TextResponseTest): response.xpath("//elem/text()").extract(), response.selector.xpath("//elem/text()").extract(), ) + + def test_selector_shortcuts_kwargs(self): + body = b''' + + value + ''' + response = self.response_class("http://www.example.com", body=body) + + self.assertEqual( + response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(), + response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(), + ) + + response.selector.register_namespace('s2', 'http://scrapy.org') + self.assertEqual( + response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(), + response.selector.xpath("//s2:elem/text()").extract(), + )