diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 9ec664bda..5510396c0 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -25,16 +25,16 @@ class BaseSgmlLinkExtractor(FixedSGMLParser): self.feed(response_text) self.close() - links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links - ret = [] base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url - for link in links: + for link in self.links: link.url = urljoin_rfc(base_url, link.url, response_encoding) link.url = safe_url_string(link.url, response_encoding) link.text = str_to_unicode(link.text, response_encoding) ret.append(link) + ret = unique_list(ret, key=lambda link: link.url) if self.unique else ret + return ret def extract_links(self, response): diff --git a/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html b/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html index 602ea04cd..fecd86563 100644 --- a/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html +++ b/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html @@ -9,7 +9,7 @@ sample 2 -sample 3 text +sample 3 text sample 3 repetition diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 65ffad714..a5e57bde0 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -110,6 +110,13 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.com/sample3.html', text=u'sample 3 text'), Link(url='http://example.com/sample3.html', text=u'sample 3 repetition') ]) + lx = SgmlLinkExtractor(allow=('sample', )) + self.assertEqual([link for link in lx.extract_links(self.response)], + [ Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + ]) + lx = SgmlLinkExtractor(allow=('sample', ), deny=('3', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''),