From 9af596a6b806a0cd0ba7f0d3bcff9ea6e3a19519 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 25 Apr 2023 10:24:14 -0500 Subject: [PATCH 1/8] feat: Add support for the Parsel JMESPath --- scrapy/http/response/__init__.py | 6 ++++++ scrapy/http/response/text.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 4213d491d..a82ed834a 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -142,6 +142,12 @@ class Response(object_ref): """ raise NotSupported("Response content isn't text") + def jmespath(self, *a, **kw): + """Shortcut method implemented only by responses whose content + is text (subclasses of TextResponse). + """ + raise NotSupported("Response content isn't text") + def xpath(self, *a, **kw): """Shortcut method implemented only by responses whose content is text (subclasses of TextResponse). diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 73bb811de..360d6334e 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -139,6 +139,9 @@ class TextResponse(Response): self._cached_selector = Selector(self) return self._cached_selector + def jmespath(self, query, **kwargs): + return self.selector.jmespath(query, **kwargs) + def xpath(self, query, **kwargs): return self.selector.xpath(query, **kwargs) From 865c36bdbbd7af0e5dd9c5c333d2285e88dfbcfd Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Fri, 28 Apr 2023 08:56:11 -0600 Subject: [PATCH 2/8] update docs --- docs/topics/request-response.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 99c7915df..407df32d2 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -1281,6 +1281,12 @@ TextResponse objects :class:`TextResponse` objects support the following methods in addition to the standard :class:`Response` ones: + .. method:: TextResponse.jmespath(query) + + A shortcut to ``TextResponse.selector.jmespath(query)``:: + + response.jmespath('object.[*]') + .. method:: TextResponse.xpath(query) A shortcut to ``TextResponse.selector.xpath(query)``:: From 3d29f20fc2021efb80d96389dc02c5a2cac9ef62 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Fri, 28 Apr 2023 23:54:09 -0600 Subject: [PATCH 3/8] added tests for jmespath --- tests/test_selector.py | 138 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/tests/test_selector.py b/tests/test_selector.py index febae46ac..b0deb5c99 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -108,3 +108,141 @@ class SelectorTestCase(unittest.TestCase): def test_selector_bad_args(self): with self.assertRaisesRegex(ValueError, "received both response and text"): Selector(TextResponse(url="http://example.com", body=b""), text="") + + +class JMESPathTestCase(unittest.TestCase): + def test_json_has_html(self) -> None: + """Sometimes the information is returned in a json wrapper""" + body = """ + { + "content": [ + { + "name": "A", + "value": "a" + }, + { + "name": { + "age": 18 + }, + "value": "b" + }, + { + "name": "C", + "value": "c" + }, + { + "name": "D", + "value": "
d
" + } + ], + "html": "
a
bc
def
" + } + """ + resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") + self.assertEqual( + resp.jmespath("html").get(), + "
a
bc
def
", + ) + self.assertEqual( + resp.jmespath("html").xpath("//div/a/text()").getall(), + ["a", "b", "d"], + ) + self.assertEqual(resp.jmespath("html").css("div > b").getall(), ["f"]) + self.assertEqual(resp.jmespath("content").jmespath("name.age").get(), "18") + + def test_html_has_json(self) -> None: + body = """ +
+

Information

+ + { + "user": [ + { + "name": "A", + "age": 18 + }, + { + "name": "B", + "age": 32 + }, + { + "name": "C", + "age": 22 + }, + { + "name": "D", + "age": 25 + } + ], + "total": 4, + "status": "ok" + } + +
+ """ + resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") + self.assertEqual( + resp.xpath("//div/content/text()").jmespath("user[*].name").getall(), + ["A", "B", "C", "D"], + ) + self.assertEqual( + resp.xpath("//div/content").jmespath("user[*].name").getall(), + ["A", "B", "C", "D"], + ) + self.assertEqual(resp.xpath("//div/content").jmespath("total").get(), "4") + + def test_jmestpath_with_re(self) -> None: + body = """ +
+

Information

+ + { + "user": [ + { + "name": "A", + "age": 18 + }, + { + "name": "B", + "age": 32 + }, + { + "name": "C", + "age": 22 + }, + { + "name": "D", + "age": 25 + } + ], + "total": 4, + "status": "ok" + } + +
+ """ + resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") + self.assertEqual( + resp.xpath("//div/content/text()").jmespath("user[*].name").re(r"(\w+)"), + ["A", "B", "C", "D"], + ) + self.assertEqual( + resp.xpath("//div/content").jmespath("user[*].name").re(r"(\w+)"), + ["A", "B", "C", "D"], + ) + + self.assertEqual( + resp.xpath("//div/content").jmespath("unavailable").re(r"(\d+)"), [] + ) + + self.assertEqual( + resp.xpath("//div/content").jmespath("unavailable").re_first(r"(\d+)"), + None, + ) + + self.assertEqual( + resp.xpath("//div/content") + .jmespath("user[*].age.to_string(@)") + .re(r"(\d+)"), + ["18", "32", "22", "25"], + ) From 578606779d0f127a759f8b1623c1e4be341a17db Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Sat, 29 Apr 2023 00:52:39 -0600 Subject: [PATCH 4/8] update tests --- tests/test_http_response.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index dbc9f1fef..cefdb1709 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -32,6 +32,9 @@ class BaseResponseTest(unittest.TestCase): isinstance(self.response_class("http://example.com/"), self.response_class) ) self.assertRaises(TypeError, self.response_class, b"http://example.com") + self.assertRaises( + TypeError, self.response_class, url="http://example.com", body={} + ) # body can be str or None self.assertTrue( isinstance( @@ -192,6 +195,7 @@ class BaseResponseTest(unittest.TestCase): self.assertRaisesRegex(AttributeError, msg, getattr, r, "text") self.assertRaisesRegex(NotSupported, msg, r.css, "body") self.assertRaisesRegex(NotSupported, msg, r.xpath, "//body") + self.assertRaisesRegex(NotSupported, msg, r.jmespath, "body") else: r.text r.css("body") From 1eb44604853e24f7b526853d5e95414949fd88c6 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 2 May 2023 18:49:05 -0600 Subject: [PATCH 5/8] fix: jmespath --- scrapy/http/response/text.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 360d6334e..dd042a2bd 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -140,7 +140,12 @@ class TextResponse(Response): return self._cached_selector def jmespath(self, query, **kwargs): - return self.selector.jmespath(query, **kwargs) + if not hasattr(self.selector, "jmespath"): # type: ignore[attr-defined] + raise AttributeError( + "Please install parsel >= 1.8.1 to get jmespath support" + ) + + return self.selector.jmespath(query, **kwargs) # type: ignore[attr-defined] def xpath(self, query, **kwargs): return self.selector.xpath(query, **kwargs) From a604dfae5c12a55760d0b44d7da06b022cce3615 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 2 May 2023 19:19:00 -0600 Subject: [PATCH 6/8] update tests --- tests/test_selector.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_selector.py b/tests/test_selector.py index b0deb5c99..274d63d8d 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -1,10 +1,16 @@ import weakref +import packaging.version as version +import parsel +import pytest from twisted.trial import unittest from scrapy.http import HtmlResponse, TextResponse, XmlResponse from scrapy.selector import Selector +PARSEL_VERSION = version.parse(getattr(parsel, "__version__", "0.0")) +PARSEL_18_PLUS = PARSEL_VERSION >= version.parse("1.8.0") + class SelectorTestCase(unittest.TestCase): def test_simple_selection(self): @@ -111,8 +117,12 @@ class SelectorTestCase(unittest.TestCase): class JMESPathTestCase(unittest.TestCase): + @pytest.mark.skipif( + not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" + ) def test_json_has_html(self) -> None: """Sometimes the information is returned in a json wrapper""" + body = """ { "content": [ @@ -150,6 +160,9 @@ class JMESPathTestCase(unittest.TestCase): self.assertEqual(resp.jmespath("html").css("div > b").getall(), ["f"]) self.assertEqual(resp.jmespath("content").jmespath("name.age").get(), "18") + @pytest.mark.skipif( + not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" + ) def test_html_has_json(self) -> None: body = """
@@ -191,6 +204,9 @@ class JMESPathTestCase(unittest.TestCase): ) self.assertEqual(resp.xpath("//div/content").jmespath("total").get(), "4") + @pytest.mark.skipif( + not PARSEL_18_PLUS, reason="parsel < 1.8 doesn't support jmespath" + ) def test_jmestpath_with_re(self) -> None: body = """
@@ -246,3 +262,14 @@ class JMESPathTestCase(unittest.TestCase): .re(r"(\d+)"), ["18", "32", "22", "25"], ) + + @pytest.mark.skipif(PARSEL_18_PLUS, reason="parsel >= 1.8 supports jmespath") + def test_jmespath_not_available(my_json_page) -> None: + body = """ + { + "website": {"name": "Example"} + } + """ + resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") + with pytest.raises(AttributeError): + resp.jmespath("website.name").get() From 4bb99fd2f3a957958ed9ea8fa418b6127a6bebee Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 2 May 2023 19:26:20 -0600 Subject: [PATCH 7/8] fix: pylint --- tests/test_selector.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_selector.py b/tests/test_selector.py index 274d63d8d..311c09aba 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -1,8 +1,8 @@ import weakref -import packaging.version as version import parsel import pytest +from packaging import version from twisted.trial import unittest from scrapy.http import HtmlResponse, TextResponse, XmlResponse @@ -11,6 +11,9 @@ from scrapy.selector import Selector PARSEL_VERSION = version.parse(getattr(parsel, "__version__", "0.0")) PARSEL_18_PLUS = PARSEL_VERSION >= version.parse("1.8.0") +print(PARSEL_VERSION) +print(PARSEL_18_PLUS) + class SelectorTestCase(unittest.TestCase): def test_simple_selection(self): From a038faf11c2bc47921f57954a2405279d427f890 Mon Sep 17 00:00:00 2001 From: Jalil SA <61639983+jxlil@users.noreply.github.com> Date: Tue, 2 May 2023 19:40:04 -0600 Subject: [PATCH 8/8] fix: tests/tes_selector.py --- tests/test_selector.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_selector.py b/tests/test_selector.py index 311c09aba..85527bba9 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -11,9 +11,6 @@ from scrapy.selector import Selector PARSEL_VERSION = version.parse(getattr(parsel, "__version__", "0.0")) PARSEL_18_PLUS = PARSEL_VERSION >= version.parse("1.8.0") -print(PARSEL_VERSION) -print(PARSEL_18_PLUS) - class SelectorTestCase(unittest.TestCase): def test_simple_selection(self):