RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5915 from jxlil/feature/support-parsel-jmespath 

Added support for the Parsel JMESPath feature
This commit is contained in:
Andrey Rakhmatullin 2023-05-03 13:18:39 +04:00 committed by GitHub
parent 6a169d4462 a038faf11c
commit ea15ff1d32
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 189 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docs/topics/request-response.rst
View File

@ -1281,6 +1281,12 @@ TextResponse objects
:class:`TextResponse` objects support the following methods in addition to
the standard :class:`Response` ones:
.. method:: TextResponse.jmespath(query)
A shortcut to ``TextResponse.selector.jmespath(query)``::
response.jmespath('object.[*]')
.. method:: TextResponse.xpath(query)
A shortcut to ``TextResponse.selector.xpath(query)``::

6
scrapy/http/response/__init__.py
View File

@ -142,6 +142,12 @@ class Response(object_ref):
"""
raise NotSupported("Response content isn't text")
def jmespath(self, *a, **kw):
"""Shortcut method implemented only by responses whose content
is text (subclasses of TextResponse).
"""
raise NotSupported("Response content isn't text")
def xpath(self, *a, **kw):
"""Shortcut method implemented only by responses whose content
is text (subclasses of TextResponse).

8
scrapy/http/response/text.py
View File

@ -141,6 +141,14 @@ class TextResponse(Response):
self._cached_selector = Selector(self)
return self._cached_selector
def jmespath(self, query, **kwargs):
if not hasattr(self.selector, "jmespath"): # type: ignore[attr-defined]
raise AttributeError(
"Please install parsel >= 1.8.1 to get jmespath support"
)
return self.selector.jmespath(query, **kwargs) # type: ignore[attr-defined]
def xpath(self, query, **kwargs):
return self.selector.xpath(query, **kwargs)

4
tests/test_http_response.py
View File

@ -32,6 +32,9 @@ class BaseResponseTest(unittest.TestCase):
isinstance(self.response_class("http://example.com/"), self.response_class)
)
self.assertRaises(TypeError, self.response_class, b"http://example.com")
self.assertRaises(
TypeError, self.response_class, url="http://example.com", body={}
)
# body can be str or None
self.assertTrue(
isinstance(
@ -192,6 +195,7 @@ class BaseResponseTest(unittest.TestCase):
self.assertRaisesRegex(AttributeError, msg, getattr, r, "text")
self.assertRaisesRegex(NotSupported, msg, r.css, "body")
self.assertRaisesRegex(NotSupported, msg, r.xpath, "//body")
self.assertRaisesRegex(NotSupported, msg, r.jmespath, "body")
else:
r.text
r.css("body")

165
tests/test_selector.py
View File

@ -1,10 +1,16 @@
import weakref
import parsel
import pytest
from packaging import version
from twisted.trial import unittest
from scrapy.http import HtmlResponse, TextResponse, XmlResponse
from scrapy.selector import Selector
PARSEL_VERSION = version.parse(getattr(parsel, "__version__", "0.0"))
PARSEL_18_PLUS = PARSEL_VERSION >= version.parse("1.8.0")
class SelectorTestCase(unittest.TestCase):
def test_simple_selection(self):
@ -108,3 +114,162 @@ class SelectorTestCase(unittest.TestCase):
def test_selector_bad_args(self):
with self.assertRaisesRegex(ValueError, "received both response and text"):
Selector(TextResponse(url="http://example.com", body=b""), text="")
class JMESPathTestCase(unittest.TestCase):
@pytest.mark.skipif(
not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath"
)
def test_json_has_html(self) -> None:
"""Sometimes the information is returned in a json wrapper"""
body = """
{
"content": [
{
"name": "A",
"value": "a"
},
{
"name": {
"age": 18
},
"value": "b"
},
{
"name": "C",
"value": "c"
},
{
"name": "<a>D</a>",
"value": "<div>d</div>"
}
],
"html": "<div><a>a<br>b</a>c</div><div><a>d</a>e<b>f</b></div>"
}
"""
resp = TextResponse(url="http://example.com", body=body, encoding="utf-8")
self.assertEqual(
resp.jmespath("html").get(),
"<div><a>a<br>b</a>c</div><div><a>d</a>e<b>f</b></div>",
)
self.assertEqual(
resp.jmespath("html").xpath("//div/a/text()").getall(),
["a", "b", "d"],
)
self.assertEqual(resp.jmespath("html").css("div > b").getall(), ["<b>f</b>"])
self.assertEqual(resp.jmespath("content").jmespath("name.age").get(), "18")
@pytest.mark.skipif(
not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath"
)
def test_html_has_json(self) -> None:
body = """
<div>
<h1>Information</h1>
<content>
{
"user": [
{
"name": "A",
"age": 18
},
{
"name": "B",
"age": 32
},
{
"name": "C",
"age": 22
},
{
"name": "D",
"age": 25
}
],
"total": 4,
"status": "ok"
}
</content>
</div>
"""
resp = TextResponse(url="http://example.com", body=body, encoding="utf-8")
self.assertEqual(
resp.xpath("//div/content/text()").jmespath("user[*].name").getall(),
["A", "B", "C", "D"],
)
self.assertEqual(
resp.xpath("//div/content").jmespath("user[*].name").getall(),
["A", "B", "C", "D"],
)
self.assertEqual(resp.xpath("//div/content").jmespath("total").get(), "4")
@pytest.mark.skipif(
not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath"
)
def test_jmestpath_with_re(self) -> None:
body = """
<div>
<h1>Information</h1>
<content>
{
"user": [
{
"name": "A",
"age": 18
},
{
"name": "B",
"age": 32
},
{
"name": "C",
"age": 22
},
{
"name": "D",
"age": 25
}
],
"total": 4,
"status": "ok"
}
</content>
</div>
"""
resp = TextResponse(url="http://example.com", body=body, encoding="utf-8")
self.assertEqual(
resp.xpath("//div/content/text()").jmespath("user[*].name").re(r"(\w+)"),
["A", "B", "C", "D"],
)
self.assertEqual(
resp.xpath("//div/content").jmespath("user[*].name").re(r"(\w+)"),
["A", "B", "C", "D"],
)
self.assertEqual(
resp.xpath("//div/content").jmespath("unavailable").re(r"(\d+)"), []
)
self.assertEqual(
resp.xpath("//div/content").jmespath("unavailable").re_first(r"(\d+)"),
None,
)
self.assertEqual(
resp.xpath("//div/content")
.jmespath("user[*].age.to_string(@)")
.re(r"(\d+)"),
["18", "32", "22", "25"],
)
@pytest.mark.skipif(PARSEL_18_PLUS, reason="parsel >= 1.8 supports jmespath")
def test_jmespath_not_available(my_json_page) -> None:
body = """
{
"website": {"name": "Example"}
}
"""
resp = TextResponse(url="http://example.com", body=body, encoding="utf-8")
with pytest.raises(AttributeError):
resp.jmespath("website.name").get()