diff --git a/tests/test_selector.py b/tests/test_selector.py index febae46ac..b0deb5c99 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -108,3 +108,141 @@ class SelectorTestCase(unittest.TestCase): def test_selector_bad_args(self): with self.assertRaisesRegex(ValueError, "received both response and text"): Selector(TextResponse(url="http://example.com", body=b""), text="") + + +class JMESPathTestCase(unittest.TestCase): + def test_json_has_html(self) -> None: + """Sometimes the information is returned in a json wrapper""" + body = """ + { + "content": [ + { + "name": "A", + "value": "a" + }, + { + "name": { + "age": 18 + }, + "value": "b" + }, + { + "name": "C", + "value": "c" + }, + { + "name": "D", + "value": "
d
" + } + ], + "html": "
a
b
c
def
" + } + """ + resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") + self.assertEqual( + resp.jmespath("html").get(), + "
a
b
c
def
", + ) + self.assertEqual( + resp.jmespath("html").xpath("//div/a/text()").getall(), + ["a", "b", "d"], + ) + self.assertEqual(resp.jmespath("html").css("div > b").getall(), ["f"]) + self.assertEqual(resp.jmespath("content").jmespath("name.age").get(), "18") + + def test_html_has_json(self) -> None: + body = """ +
+

Information

+ + { + "user": [ + { + "name": "A", + "age": 18 + }, + { + "name": "B", + "age": 32 + }, + { + "name": "C", + "age": 22 + }, + { + "name": "D", + "age": 25 + } + ], + "total": 4, + "status": "ok" + } + +
+ """ + resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") + self.assertEqual( + resp.xpath("//div/content/text()").jmespath("user[*].name").getall(), + ["A", "B", "C", "D"], + ) + self.assertEqual( + resp.xpath("//div/content").jmespath("user[*].name").getall(), + ["A", "B", "C", "D"], + ) + self.assertEqual(resp.xpath("//div/content").jmespath("total").get(), "4") + + def test_jmestpath_with_re(self) -> None: + body = """ +
+

Information

+ + { + "user": [ + { + "name": "A", + "age": 18 + }, + { + "name": "B", + "age": 32 + }, + { + "name": "C", + "age": 22 + }, + { + "name": "D", + "age": 25 + } + ], + "total": 4, + "status": "ok" + } + +
+ """ + resp = TextResponse(url="http://example.com", body=body, encoding="utf-8") + self.assertEqual( + resp.xpath("//div/content/text()").jmespath("user[*].name").re(r"(\w+)"), + ["A", "B", "C", "D"], + ) + self.assertEqual( + resp.xpath("//div/content").jmespath("user[*].name").re(r"(\w+)"), + ["A", "B", "C", "D"], + ) + + self.assertEqual( + resp.xpath("//div/content").jmespath("unavailable").re(r"(\d+)"), [] + ) + + self.assertEqual( + resp.xpath("//div/content").jmespath("unavailable").re_first(r"(\d+)"), + None, + ) + + self.assertEqual( + resp.xpath("//div/content") + .jmespath("user[*].age.to_string(@)") + .re(r"(\d+)"), + ["18", "32", "22", "25"], + )