added tests for jmespath

tests/test_selector.py

@@ -108,3 +108,141 @@ class SelectorTestCase(unittest.TestCase):
     def test_selector_bad_args(self):
         with self.assertRaisesRegex(ValueError, "received both response and text"):
             Selector(TextResponse(url="http://example.com", body=b""), text="")
+
+
+class JMESPathTestCase(unittest.TestCase):
+    def test_json_has_html(self) -> None:
+        """Sometimes the information is returned in a json wrapper"""
+        body = """
+        {
+            "content": [
+                {
+                    "name": "A",
+                    "value": "a"
+                },
+                {
+                    "name": {
+                        "age": 18
+                    },
+                    "value": "b"
+                },
+                {
+                    "name": "C",
+                    "value": "c"
+                },
+                {
+                    "name": "<a>D</a>",
+                    "value": "<div>d</div>"
+                }
+            ],
+            "html": "<div><a>a<br>b</a>c</div><div><a>d</a>e<b>f</b></div>"
+        }
+        """
+        resp = TextResponse(url="http://example.com", body=body, encoding="utf-8")
+        self.assertEqual(
+            resp.jmespath("html").get(),
+            "<div><a>a<br>b</a>c</div><div><a>d</a>e<b>f</b></div>",
+        )
+        self.assertEqual(
+            resp.jmespath("html").xpath("//div/a/text()").getall(),
+            ["a", "b", "d"],
+        )
+        self.assertEqual(resp.jmespath("html").css("div > b").getall(), ["<b>f</b>"])
+        self.assertEqual(resp.jmespath("content").jmespath("name.age").get(), "18")
+
+    def test_html_has_json(self) -> None:
+        body = """
+        <div>
+            <h1>Information</h1>
+            <content>
+            {
+              "user": [
+                        {
+                                  "name": "A",
+                                  "age": 18
+                        },
+                        {
+                                  "name": "B",
+                                  "age": 32
+                        },
+                        {
+                                  "name": "C",
+                                  "age": 22
+                        },
+                        {
+                                  "name": "D",
+                                  "age": 25
+                        }
+              ],
+              "total": 4,
+              "status": "ok"
+            }
+            </content>
+        </div>
+        """
+        resp = TextResponse(url="http://example.com", body=body, encoding="utf-8")
+        self.assertEqual(
+            resp.xpath("//div/content/text()").jmespath("user[*].name").getall(),
+            ["A", "B", "C", "D"],
+        )
+        self.assertEqual(
+            resp.xpath("//div/content").jmespath("user[*].name").getall(),
+            ["A", "B", "C", "D"],
+        )
+        self.assertEqual(resp.xpath("//div/content").jmespath("total").get(), "4")
+
+    def test_jmestpath_with_re(self) -> None:
+        body = """
+            <div>
+                <h1>Information</h1>
+                <content>
+                {
+                  "user": [
+                            {
+                                      "name": "A",
+                                      "age": 18
+                            },
+                            {
+                                      "name": "B",
+                                      "age": 32
+                            },
+                            {
+                                      "name": "C",
+                                      "age": 22
+                            },
+                            {
+                                      "name": "D",
+                                      "age": 25
+                            }
+                  ],
+                  "total": 4,
+                  "status": "ok"
+                }
+                </content>
+            </div>
+            """
+        resp = TextResponse(url="http://example.com", body=body, encoding="utf-8")
+        self.assertEqual(
+            resp.xpath("//div/content/text()").jmespath("user[*].name").re(r"(\w+)"),
+            ["A", "B", "C", "D"],
+        )
+        self.assertEqual(
+            resp.xpath("//div/content").jmespath("user[*].name").re(r"(\w+)"),
+            ["A", "B", "C", "D"],
+        )
+
+        self.assertEqual(
+            resp.xpath("//div/content").jmespath("unavailable").re(r"(\d+)"), []
+        )
+
+        self.assertEqual(
+            resp.xpath("//div/content").jmespath("unavailable").re_first(r"(\d+)"),
+            None,
+        )
+
+        self.assertEqual(
+            resp.xpath("//div/content")
+            .jmespath("user[*].age.to_string(@)")
+            .re(r"(\d+)"),
+            ["18", "32", "22", "25"],
+        )