diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 2207763c9..18a0f4487 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -115,13 +115,15 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor): base_url = None if self.restrict_xpaths: hxs = HtmlXPathSelector(response) - html = ''.join(''.join(html_fragm for html_fragm in hxs.select(xpath_expr).extract()) \ - for xpath_expr in self.restrict_xpaths) base_url = get_base_url(response) + body = u''.join(f + for x in self.restrict_xpaths + for f in hxs.select(x).extract() + ).encode(response.encoding) else: - html = response.body + body = response.body - links = self._extract_links(html, response.url, response.encoding, base_url) + links = self._extract_links(body, response.url, response.encoding, base_url) links = self._process_links(links) return links diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index f49e90792..d1992b170 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -217,6 +217,15 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.extract_links(response), [Link(url='http://example.org/about.html', text=u'About us\xa3')]) + def test_restrict_xpaths_concat_in_handle_data(self): + """html entities cause SGMLParser to call handle_data hook twice""" + body = """
>\xbe\xa9<\xb6\xab""" + response = HtmlResponse("http://example.org", body=body, encoding='gb18030') + lx = SgmlLinkExtractor(restrict_xpaths="//div") + self.assertEqual(lx.extract_links(response), + [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c', + fragment='', nofollow=False)]) + def test_deny_extensions(self): html = """asd and """ response = HtmlResponse("http://example.org/", body=html)