diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 9a55c1581..3eb5fd91f 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -67,6 +67,7 @@ class BaseSgmlLinkExtractor(SGMLParser): SGMLParser.reset(self) self.links = [] self.base_url = None + self.current_link = None def unknown_starttag(self, tag, attrs): if tag == 'base': diff --git a/tests/test_contrib_linkextractors.py b/tests/test_contrib_linkextractors.py index 3617cb810..3902d4c50 100644 --- a/tests/test_contrib_linkextractors.py +++ b/tests/test_contrib_linkextractors.py @@ -284,6 +284,17 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c', fragment='', nofollow=False)]) + def test_area_tag_with_unicode_present(self): + body = """\xbe\xa9""" + response = HtmlResponse("http://example.org", body=body, encoding='utf-8') + lx = self.extractor_cls() + lx.extract_links(response) + lx.extract_links(response) + lx.extract_links(response) + self.assertEqual(lx.extract_links(response), + [Link(url='http://example.org/foo', text=u'', + fragment='', nofollow=False)]) + def test_encoded_url(self): body = """
BinB""" response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8')