RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

more fixes to lxml selector incompatibilities 

* Do not fail parsing empty bodies
* Do not fail parsing bodies with null bytes
* Recode to utf8 using response.body_as_unicode() to avoid decoding bugs
* Return empty results with unevaluable nodes like text or attribute nodes
* Return u'1' and u'0' for boolean xpaths
This commit is contained in:
Daniel Graña 2012-04-13 00:21:48 -03:00
parent 02833e3265
commit b9efa5ee73
2 changed files with 65 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
scrapy/selector/lxmlsel.py
View File

@ -35,27 +35,29 @@ class XPathSelector(object_ref):
@property
def root(self):
if self._root is None:
parser = self._parser(encoding=self.response.encoding, recover=True)
self._root = etree.fromstring(self.response.body, parser=parser, \
base_url=self.response.url)
url = self.response.url
body = self.response.body_as_unicode().strip().encode('utf8') or '<html/>'
parser = self._parser(recover=True, encoding='utf8')
self._root = etree.fromstring(body, parser=parser, base_url=url)
assert self._root is not None, 'BUG lxml selector with None root'
return self._root
@property
def xpathev(self):
if self._xpathev is None:
self._xpathev = etree.XPathEvaluator(self.root, namespaces=self.namespaces)
return self._xpathev
def select(self, xpath):
try:
result = self.xpathev(xpath)
xpathev = self.root.xpath
except AttributeError:
return XPathSelectorList([])
try:
result = xpathev(xpath, namespaces=self.namespaces)
except etree.XPathError:
raise ValueError("Invalid XPath: %s" % xpath)
if hasattr(result, '__iter__'):
result = [self.__class__(root=x, expr=xpath, namespaces=self.namespaces) \
for x in result]
else:
result = [self.__class__(root=result, expr=xpath, namespaces=self.namespaces)]
if type(result) is not list:
result = [result]
result = [self.__class__(root=x, expr=xpath, namespaces=self.namespaces)
for x in result]
return XPathSelectorList(result)
def re(self, regex):

51
scrapy/tests/test_selector.py
View File

@ -238,9 +238,55 @@ class XPathSelectorTestCase(unittest.TestCase):
@libxml2debug
def test_empty_bodies(self):
# shouldn't raise errors
r1 = TextResponse('http://www.example.com', body='')
self.hxs_cls(r1) # shouldn't raise error
self.xxs_cls(r1) # shouldn't raise error
self.hxs_cls(r1).select('//text()').extract()
self.xxs_cls(r1).select('//text()').extract()
@libxml2debug
def test_null_bytes(self):
# shouldn't raise errors
r1 = TextResponse('http://www.example.com', \
body='<root>pre\x00post</root>', \
encoding='utf-8')
self.hxs_cls(r1).select('//text()').extract()
self.xxs_cls(r1).select('//text()').extract()
@libxml2debug
def test_badly_encoded_body(self):
# \xe9 alone isn't valid utf8 sequence
r1 = TextResponse('http://www.example.com', \
body='<html><p>an Jos\xe9 de</p><html>', \
encoding='utf-8')
self.hxs_cls(r1).select('//text()').extract()
self.xxs_cls(r1).select('//text()').extract()
@libxml2debug
def test_select_on_unevaluable_nodes(self):
r = self.hxs_cls(text=u'<span class="big">some text</span>')
# Text node
x1 = r.select('//text()')
self.assertEquals(x1.extract(), [u'some text'])
self.assertEquals(x1.select('.//b').extract(), [])
# Tag attribute
x1 = r.select('//span/@class')
self.assertEquals(x1.extract(), [u'big'])
self.assertEquals(x1.select('.//text()').extract(), [])
@libxml2debug
def test_select_on_text_nodes(self):
# FIXME: can't get this to work with lxml backend
r = self.hxs_cls(text=u'<div><b>Options:</b>opt1</div><div><b>Other</b>opt2</div>')
x1 = r.select("//div/descendant::text()[preceding-sibling::b[contains(text(), 'Options')]]")
self.assertEquals(x1.extract(), [u'opt1'])
x1 = r.select("//div/descendant::text()/preceding-sibling::b[contains(text(), 'Options')]")
self.assertEquals(x1.extract(), [u'<b>Options:</b>'])
x1 = r.select("//div/descendant::text()")
x2 = x1.select("./preceding-sibling::b[contains(text(), 'Options')]")
self.assertEquals(x2.extract(), [u'<b>Options:</b>'])
test_select_on_text_nodes.skip = True
@libxml2debug
def test_weakref_slots(self):
@ -250,4 +296,3 @@ class XPathSelectorTestCase(unittest.TestCase):
weakref.ref(x)
assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
x.__class__.__name__