From cc69b3aa4c88c60ca41a792a60f7e80d75f1b7f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 29 Jan 2013 17:56:34 -0200 Subject: [PATCH] Fix #199 encoding error concatenating link text --- scrapy/contrib/linkextractors/sgml.py | 10 ++++++---- scrapy/tests/test_contrib_linkextractors.py | 9 +++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 2207763c9..18a0f4487 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -115,13 +115,15 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor): base_url = None if self.restrict_xpaths: hxs = HtmlXPathSelector(response) - html = ''.join(''.join(html_fragm for html_fragm in hxs.select(xpath_expr).extract()) \ - for xpath_expr in self.restrict_xpaths) base_url = get_base_url(response) + body = u''.join(f + for x in self.restrict_xpaths + for f in hxs.select(x).extract() + ).encode(response.encoding) else: - html = response.body + body = response.body - links = self._extract_links(html, response.url, response.encoding, base_url) + links = self._extract_links(body, response.url, response.encoding, base_url) links = self._process_links(links) return links diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index f49e90792..d1992b170 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -217,6 +217,15 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.extract_links(response), [Link(url='http://example.org/about.html', text=u'About us\xa3')]) + def test_restrict_xpaths_concat_in_handle_data(self): + """html entities cause SGMLParser to call handle_data hook twice""" + body = """
>\xbe\xa9<\xb6\xab""" + response = HtmlResponse("http://example.org", body=body, encoding='gb18030') + lx = SgmlLinkExtractor(restrict_xpaths="//div") + self.assertEqual(lx.extract_links(response), + [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c', + fragment='', nofollow=False)]) + def test_deny_extensions(self): html = """asd and """ response = HtmlResponse("http://example.org/", body=html)