RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2457 from redapple/parsel-selector-kwargs 

[MRG] Support kwargs for response.xpath()
This commit is contained in:
Mikhail Korobov 2017-02-02 22:17:26 +05:00 committed by GitHub
parent c305c46254 1295c17a26
commit ae2a5292d8
5 changed files with 70 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
docs/topics/selectors.rst
View File

@ -283,6 +283,40 @@ XPath specification.
.. _Location Paths: https://www.w3.org/TR/xpath#location-paths
.. _topics-selectors-xpath-variables:
Variables in XPath expressions
------------------------------
XPath allows you to reference variables in your XPath expressions, using
the ``$somevariable`` syntax. This is somewhat similar to parameterized
queries or prepared statements in the SQL world where you replace
some arguments in your queries with placeholders like ``?``,
which are then substituted with values passed with the query.
Here's an example to match an element based on its "id" attribute value,
without hard-coding it (that was shown previously)::
>>> # `$val` used in the expression, a `val` argument needs to be passed
>>> response.xpath('//div[@id=$val]/a/text()', val='images').extract_first()
u'Name: My image 1 '
Here's another example, to find the "id" attribute of a ``<div>`` tag containing
five ``<a>`` children (here we pass the value ``5`` as an integer)::
>>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).extract_first()
u'images'
All variable references must have a binding value when calling ``.xpath()``
(otherwise you'll get a ``ValueError: XPath error:`` exception).
This is done by passing as many named arguments as necessary.
`parsel`_, the library powering Scrapy selectors, has more details and examples
on `XPath variables`_.
.. _parsel: https://parsel.readthedocs.io/
.. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions
Using EXSLT extensions
----------------------

2
requirements.txt
View File

@ -7,4 +7,4 @@ queuelib
six>=1.5.2
PyDispatcher>=2.0.5
service_identity
parsel>=0.9.5
parsel>=1.1

4
scrapy/http/response/text.py
View File

@ -111,8 +111,8 @@ class TextResponse(Response):
self._cached_selector = Selector(self)
return self._cached_selector
def xpath(self, query):
return self.selector.xpath(query)
def xpath(self, query, **kwargs):
return self.selector.xpath(query, **kwargs)
def css(self, query):
return self.selector.css(query)

2
setup.py
View File

@ -49,7 +49,7 @@ setup(
'pyOpenSSL',
'cssselect>=0.9',
'six>=1.5.2',
'parsel>=0.9.5',
'parsel>=1.1',
'PyDispatcher>=2.0.5',
'service_identity',
],

32
tests/test_http_response.py
View File

@ -320,6 +320,20 @@ class TextResponseTest(BaseResponseTest):
response.selector.css("title::text").extract(),
)
def test_selector_shortcuts_kwargs(self):
body = b"<html><head><title>Some page</title><body><p class=\"content\">A nice paragraph.</p></body></html>"
response = self.response_class("http://www.example.com", body=body)
self.assertEqual(
response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(),
response.xpath("normalize-space(//p[@class=\"content\"])").extract(),
)
self.assertEqual(
response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()",
pclass="content", pcount=1).extract(),
response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(),
)
def test_urljoin_with_base_url(self):
"""Test urljoin shortcut which also evaluates base-url through get_base_url()."""
body = b'<html><body><base href="https://example.net"></body></html>'
@ -428,3 +442,21 @@ class XmlResponseTest(TextResponseTest):
response.xpath("//elem/text()").extract(),
response.selector.xpath("//elem/text()").extract(),
)
def test_selector_shortcuts_kwargs(self):
body = b'''<?xml version="1.0" encoding="utf-8"?>
<xml xmlns:somens="http://scrapy.org">
<somens:elem>value</somens:elem>
</xml>'''
response = self.response_class("http://www.example.com", body=body)
self.assertEqual(
response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
)
response.selector.register_namespace('s2', 'http://scrapy.org')
self.assertEqual(
response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(),
response.selector.xpath("//s2:elem/text()").extract(),
)