From 1f184ed7bf5cd32fb2b19bf42d4260d81d4d1cdd Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 15 Jan 2014 15:17:18 +0100 Subject: [PATCH 1/4] Disable smart strings in lxml XPath evaluations --- scrapy/selector/unified.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 790937af5..368222768 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -85,7 +85,8 @@ class Selector(object_ref): return SelectorList([]) try: - result = xpathev(query, namespaces=self.namespaces) + result = xpathev(query, namespaces=self.namespaces, + smart_strings=False) except etree.XPathError: raise ValueError("Invalid XPath: %s" % query) From 5eb336215c573ae49774cab332a882b3a8ca830f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 16 Jan 2014 10:44:17 +0100 Subject: [PATCH 2/4] Make lxml smart strings functionality customizable --- scrapy/selector/unified.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 368222768..babb65070 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -57,6 +57,7 @@ class Selector(object_ref): # set:trailing "set": "http://exslt.org/sets" } + _lxml_smart_strings = False def __init__(self, response=None, text=None, type=None, namespaces=None, _root=None, _expr=None): @@ -86,7 +87,7 @@ class Selector(object_ref): try: result = xpathev(query, namespaces=self.namespaces, - smart_strings=False) + smart_strings=self._lxml_smart_strings) except etree.XPathError: raise ValueError("Invalid XPath: %s" % query) From 001cf39ff436fb44da2ae5e8b60aaef73991310e Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 17 Jan 2014 00:04:20 +0100 Subject: [PATCH 3/4] Add testcase to check is default Selector doesnt return smart strings --- scrapy/tests/test_selector.py | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index d84e4bd47..27f10a2b5 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -297,6 +297,49 @@ class SelectorTestCase(unittest.TestCase): sel.remove_namespaces() self.assertEqual(len(sel.xpath("//link/@type")), 2) + def test_smart_strings(self): + """Lxml smart strings return values""" + + class SmartStringsSelector(Selector): + _lxml_smart_strings = True + + body = """ +
+
    +
  • one
  • two
    • +
+
+
+
    +
  • four
  • five
  • six
    • +
+
+ """ + + response = HtmlResponse(url="http://example.com", body=body) + + # .getparent() is available for text nodes and attributes + # only when smart_strings are on + x = self.sscls(response) + li_text = x.xpath('//li/text()') + self.assertIs( + any(map(lambda e: hasattr(e._root, 'getparent'), li_text)), + False) + div_class = x.xpath('//div/@class') + self.assertIs( + any(map(lambda e: hasattr(e._root, 'getparent'), div_class)), + False) + + x = SmartStringsSelector(response) + li_text = x.xpath('//li/text()') + self.assertIs( + all(map(lambda e: hasattr(e._root, 'getparent'), li_text)), + True) + div_class = x.xpath('//div/@class') + self.assertIs( + all(map(lambda e: hasattr(e._root, 'getparent'), div_class)), + True) + class DeprecatedXpathSelectorTest(unittest.TestCase): From a0e25aec007a36eaea8d5bc2ea7d43ade1293328 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 20 Jan 2014 17:29:16 +0100 Subject: [PATCH 4/4] Use assertTrue/False --- scrapy/tests/test_selector.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index 27f10a2b5..489a163a0 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -322,23 +322,15 @@ class SelectorTestCase(unittest.TestCase): # only when smart_strings are on x = self.sscls(response) li_text = x.xpath('//li/text()') - self.assertIs( - any(map(lambda e: hasattr(e._root, 'getparent'), li_text)), - False) + self.assertFalse(any(map(lambda e: hasattr(e._root, 'getparent'), li_text))) div_class = x.xpath('//div/@class') - self.assertIs( - any(map(lambda e: hasattr(e._root, 'getparent'), div_class)), - False) + self.assertFalse(any(map(lambda e: hasattr(e._root, 'getparent'), div_class))) x = SmartStringsSelector(response) li_text = x.xpath('//li/text()') - self.assertIs( - all(map(lambda e: hasattr(e._root, 'getparent'), li_text)), - True) + self.assertTrue(all(map(lambda e: hasattr(e._root, 'getparent'), li_text))) div_class = x.xpath('//div/@class') - self.assertIs( - all(map(lambda e: hasattr(e._root, 'getparent'), div_class)), - True) + self.assertTrue(all(map(lambda e: hasattr(e._root, 'getparent'), div_class))) class DeprecatedXpathSelectorTest(unittest.TestCase):