RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SgmlLinkExtractor - fix for parsing <area> tag with Unicode present

This commit is contained in:
yakxxx 2014-08-28 18:47:49 +02:00 committed by Daniel Graña
parent 4677eedb06
commit 49b40f01ad
2 changed files with 12 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
scrapy/contrib/linkextractors/sgml.py
View File

@ -59,6 +59,7 @@ class BaseSgmlLinkExtractor(FixedSGMLParser):
FixedSGMLParser.reset(self)
self.links = []
self.base_url = None
self.current_link = None
def unknown_starttag(self, tag, attrs):
if tag == 'base':

11
scrapy/tests/test_contrib_linkextractors.py
View File

@ -284,6 +284,17 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
[Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c',
fragment='', nofollow=False)])
def test_area_tag_with_unicode_present(self):
body = """<html><body>\xbe\xa9<map><area href="http://example.org/foo" /></map></body></html>"""
response = HtmlResponse("http://example.org", body=body, encoding='utf-8')
lx = self.extractor_cls()
lx.extract_links(response)
lx.extract_links(response)
lx.extract_links(response)
self.assertEqual(lx.extract_links(response),
[Link(url='http://example.org/foo', text=u'',
fragment='', nofollow=False)])
def test_encoded_url(self):
body = """<html><body><div><a href="?page=2">BinB</a></body></html>"""
response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')