From b9efa5ee73867a67891dc84124221ffc0f076305 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?=
Date: Fri, 13 Apr 2012 00:21:48 -0300
Subject: [PATCH] more fixes to lxml selector incompatibilities
* Do not fail parsing empty bodies
* Do not fail parsing bodies with null bytes
* Recode to utf8 using response.body_as_unicode() to avoid decoding bugs
* Return empty results with unevaluable nodes like text or attribute nodes
* Return u'1' and u'0' for boolean xpaths
---
scrapy/selector/lxmlsel.py | 32 +++++++++++-----------
scrapy/tests/test_selector.py | 51 ++++++++++++++++++++++++++++++++---
2 files changed, 65 insertions(+), 18 deletions(-)
diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py
index cc32a81f4..301858890 100644
--- a/scrapy/selector/lxmlsel.py
+++ b/scrapy/selector/lxmlsel.py
@@ -35,27 +35,29 @@ class XPathSelector(object_ref):
@property
def root(self):
if self._root is None:
- parser = self._parser(encoding=self.response.encoding, recover=True)
- self._root = etree.fromstring(self.response.body, parser=parser, \
- base_url=self.response.url)
+ url = self.response.url
+ body = self.response.body_as_unicode().strip().encode('utf8') or ''
+ parser = self._parser(recover=True, encoding='utf8')
+ self._root = etree.fromstring(body, parser=parser, base_url=url)
+ assert self._root is not None, 'BUG lxml selector with None root'
return self._root
- @property
- def xpathev(self):
- if self._xpathev is None:
- self._xpathev = etree.XPathEvaluator(self.root, namespaces=self.namespaces)
- return self._xpathev
-
def select(self, xpath):
try:
- result = self.xpathev(xpath)
+ xpathev = self.root.xpath
+ except AttributeError:
+ return XPathSelectorList([])
+
+ try:
+ result = xpathev(xpath, namespaces=self.namespaces)
except etree.XPathError:
raise ValueError("Invalid XPath: %s" % xpath)
- if hasattr(result, '__iter__'):
- result = [self.__class__(root=x, expr=xpath, namespaces=self.namespaces) \
- for x in result]
- else:
- result = [self.__class__(root=result, expr=xpath, namespaces=self.namespaces)]
+
+ if type(result) is not list:
+ result = [result]
+
+ result = [self.__class__(root=x, expr=xpath, namespaces=self.namespaces)
+ for x in result]
return XPathSelectorList(result)
def re(self, regex):
diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py
index 73fe30216..20672098f 100644
--- a/scrapy/tests/test_selector.py
+++ b/scrapy/tests/test_selector.py
@@ -238,9 +238,55 @@ class XPathSelectorTestCase(unittest.TestCase):
@libxml2debug
def test_empty_bodies(self):
+ # shouldn't raise errors
r1 = TextResponse('http://www.example.com', body='')
- self.hxs_cls(r1) # shouldn't raise error
- self.xxs_cls(r1) # shouldn't raise error
+ self.hxs_cls(r1).select('//text()').extract()
+ self.xxs_cls(r1).select('//text()').extract()
+
+ @libxml2debug
+ def test_null_bytes(self):
+ # shouldn't raise errors
+ r1 = TextResponse('http://www.example.com', \
+ body='pre\x00post', \
+ encoding='utf-8')
+ self.hxs_cls(r1).select('//text()').extract()
+ self.xxs_cls(r1).select('//text()').extract()
+
+ @libxml2debug
+ def test_badly_encoded_body(self):
+ # \xe9 alone isn't valid utf8 sequence
+ r1 = TextResponse('http://www.example.com', \
+ body='an Jos\xe9 de
', \
+ encoding='utf-8')
+ self.hxs_cls(r1).select('//text()').extract()
+ self.xxs_cls(r1).select('//text()').extract()
+
+ @libxml2debug
+ def test_select_on_unevaluable_nodes(self):
+ r = self.hxs_cls(text=u'some text')
+ # Text node
+ x1 = r.select('//text()')
+ self.assertEquals(x1.extract(), [u'some text'])
+ self.assertEquals(x1.select('.//b').extract(), [])
+ # Tag attribute
+ x1 = r.select('//span/@class')
+ self.assertEquals(x1.extract(), [u'big'])
+ self.assertEquals(x1.select('.//text()').extract(), [])
+
+ @libxml2debug
+ def test_select_on_text_nodes(self):
+ # FIXME: can't get this to work with lxml backend
+ r = self.hxs_cls(text=u'Options:opt1
Otheropt2
')
+ x1 = r.select("//div/descendant::text()[preceding-sibling::b[contains(text(), 'Options')]]")
+ self.assertEquals(x1.extract(), [u'opt1'])
+
+ x1 = r.select("//div/descendant::text()/preceding-sibling::b[contains(text(), 'Options')]")
+ self.assertEquals(x1.extract(), [u'Options:'])
+
+ x1 = r.select("//div/descendant::text()")
+ x2 = x1.select("./preceding-sibling::b[contains(text(), 'Options')]")
+ self.assertEquals(x2.extract(), [u'Options:'])
+ test_select_on_text_nodes.skip = True
@libxml2debug
def test_weakref_slots(self):
@@ -250,4 +296,3 @@ class XPathSelectorTestCase(unittest.TestCase):
weakref.ref(x)
assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
x.__class__.__name__
-