RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix #199 encoding error concatenating link text

This commit is contained in:
Daniel Graña 2013-01-29 17:56:34 -02:00
parent 8e77f27897
commit cc69b3aa4c
2 changed files with 15 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scrapy/contrib/linkextractors/sgml.py
View File

@ -115,13 +115,15 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor):
base_url = None
if self.restrict_xpaths:
hxs = HtmlXPathSelector(response)
html = ''.join(''.join(html_fragm for html_fragm in hxs.select(xpath_expr).extract()) \
for xpath_expr in self.restrict_xpaths)
base_url = get_base_url(response)
body = u''.join(f
for x in self.restrict_xpaths
for f in hxs.select(x).extract()
).encode(response.encoding)
else:
html = response.body
body = response.body
links = self._extract_links(html, response.url, response.encoding, base_url)
links = self._extract_links(body, response.url, response.encoding, base_url)
links = self._process_links(links)
return links

9
scrapy/tests/test_contrib_linkextractors.py
View File

@ -217,6 +217,15 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
self.assertEqual(lx.extract_links(response),
[Link(url='http://example.org/about.html', text=u'About us\xa3')])
def test_restrict_xpaths_concat_in_handle_data(self):
"""html entities cause SGMLParser to call handle_data hook twice"""
body = """<html><body><div><a href="/foo">&gt;\xbe\xa9&lt;\xb6\xab</a></body></html>"""
response = HtmlResponse("http://example.org", body=body, encoding='gb18030')
lx = SgmlLinkExtractor(restrict_xpaths="//div")
self.assertEqual(lx.extract_links(response),
[Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
fragment='', nofollow=False)])
def test_deny_extensions(self):
html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
response = HtmlResponse("http://example.org/", body=html)