diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py index cc32a81f4..301858890 100644 --- a/scrapy/selector/lxmlsel.py +++ b/scrapy/selector/lxmlsel.py @@ -35,27 +35,29 @@ class XPathSelector(object_ref): @property def root(self): if self._root is None: - parser = self._parser(encoding=self.response.encoding, recover=True) - self._root = etree.fromstring(self.response.body, parser=parser, \ - base_url=self.response.url) + url = self.response.url + body = self.response.body_as_unicode().strip().encode('utf8') or '' + parser = self._parser(recover=True, encoding='utf8') + self._root = etree.fromstring(body, parser=parser, base_url=url) + assert self._root is not None, 'BUG lxml selector with None root' return self._root - @property - def xpathev(self): - if self._xpathev is None: - self._xpathev = etree.XPathEvaluator(self.root, namespaces=self.namespaces) - return self._xpathev - def select(self, xpath): try: - result = self.xpathev(xpath) + xpathev = self.root.xpath + except AttributeError: + return XPathSelectorList([]) + + try: + result = xpathev(xpath, namespaces=self.namespaces) except etree.XPathError: raise ValueError("Invalid XPath: %s" % xpath) - if hasattr(result, '__iter__'): - result = [self.__class__(root=x, expr=xpath, namespaces=self.namespaces) \ - for x in result] - else: - result = [self.__class__(root=result, expr=xpath, namespaces=self.namespaces)] + + if type(result) is not list: + result = [result] + + result = [self.__class__(root=x, expr=xpath, namespaces=self.namespaces) + for x in result] return XPathSelectorList(result) def re(self, regex): diff --git a/scrapy/tests/test_selector.py b/scrapy/tests/test_selector.py index 73fe30216..20672098f 100644 --- a/scrapy/tests/test_selector.py +++ b/scrapy/tests/test_selector.py @@ -238,9 +238,55 @@ class XPathSelectorTestCase(unittest.TestCase): @libxml2debug def test_empty_bodies(self): + # shouldn't raise errors r1 = TextResponse('http://www.example.com', body='') - self.hxs_cls(r1) # shouldn't raise error - self.xxs_cls(r1) # shouldn't raise error + self.hxs_cls(r1).select('//text()').extract() + self.xxs_cls(r1).select('//text()').extract() + + @libxml2debug + def test_null_bytes(self): + # shouldn't raise errors + r1 = TextResponse('http://www.example.com', \ + body='pre\x00post', \ + encoding='utf-8') + self.hxs_cls(r1).select('//text()').extract() + self.xxs_cls(r1).select('//text()').extract() + + @libxml2debug + def test_badly_encoded_body(self): + # \xe9 alone isn't valid utf8 sequence + r1 = TextResponse('http://www.example.com', \ + body='

an Jos\xe9 de

', \ + encoding='utf-8') + self.hxs_cls(r1).select('//text()').extract() + self.xxs_cls(r1).select('//text()').extract() + + @libxml2debug + def test_select_on_unevaluable_nodes(self): + r = self.hxs_cls(text=u'some text') + # Text node + x1 = r.select('//text()') + self.assertEquals(x1.extract(), [u'some text']) + self.assertEquals(x1.select('.//b').extract(), []) + # Tag attribute + x1 = r.select('//span/@class') + self.assertEquals(x1.extract(), [u'big']) + self.assertEquals(x1.select('.//text()').extract(), []) + + @libxml2debug + def test_select_on_text_nodes(self): + # FIXME: can't get this to work with lxml backend + r = self.hxs_cls(text=u'
Options:opt1
Otheropt2
') + x1 = r.select("//div/descendant::text()[preceding-sibling::b[contains(text(), 'Options')]]") + self.assertEquals(x1.extract(), [u'opt1']) + + x1 = r.select("//div/descendant::text()/preceding-sibling::b[contains(text(), 'Options')]") + self.assertEquals(x1.extract(), [u'Options:']) + + x1 = r.select("//div/descendant::text()") + x2 = x1.select("./preceding-sibling::b[contains(text(), 'Options')]") + self.assertEquals(x2.extract(), [u'Options:']) + test_select_on_text_nodes.skip = True @libxml2debug def test_weakref_slots(self): @@ -250,4 +296,3 @@ class XPathSelectorTestCase(unittest.TestCase): weakref.ref(x) assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ x.__class__.__name__ -