RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #565 from dangra/562-sgmlinkextractor 

replace unencodeable codepoints with html entities
This commit is contained in:
Pablo Hoffman 2014-01-27 22:01:39 -08:00
parent 116a1df4b0 66829c962f
commit 21f0e4048b
2 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/contrib/linkextractors/sgml.py
View File

@ -121,7 +121,7 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor):
body = u''.join(f
for x in self.restrict_xpaths
for f in sel.xpath(x).extract()
).encode(response.encoding)
).encode(response.encoding, errors='xmlcharrefreplace')
else:
body = response.body

7
scrapy/tests/test_contrib_linkextractors.py
View File

@ -236,6 +236,13 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
self.assertEqual(lx.extract_links(response),
[Link(url='http://example.org/about.html', text=u'About us\xa3')])
def test_restrict_xpaths_with_html_entities(self):
html = '<html><body><p><a href="/&hearts;/you?c=&euro;">text</a></p></body></html>'
response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
links = SgmlLinkExtractor(restrict_xpaths='//p').extract_links(response)
self.assertEqual(links,
[Link(url='http://example.org/%E2%99%A5/you?c=%E2%82%AC', text=u'text')])
def test_restrict_xpaths_concat_in_handle_data(self):
"""html entities cause SGMLParser to call handle_data hook twice"""
body = """<html><body><div><a href="/foo">&gt;\xbe\xa9&lt;\xb6\xab</a></body></html>"""