diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py
index 9ec664bda..5510396c0 100644
--- a/scrapy/contrib/linkextractors/sgml.py
+++ b/scrapy/contrib/linkextractors/sgml.py
@@ -25,16 +25,16 @@ class BaseSgmlLinkExtractor(FixedSGMLParser):
self.feed(response_text)
self.close()
- links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links
-
ret = []
base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url
- for link in links:
+ for link in self.links:
link.url = urljoin_rfc(base_url, link.url, response_encoding)
link.url = safe_url_string(link.url, response_encoding)
link.text = str_to_unicode(link.text, response_encoding)
ret.append(link)
+ ret = unique_list(ret, key=lambda link: link.url) if self.unique else ret
+
return ret
def extract_links(self, response):
diff --git a/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html b/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html
index 602ea04cd..fecd86563 100644
--- a/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html
+++ b/scrapy/tests/sample_data/link_extractor/sgml_linkextractor.html
@@ -9,7 +9,7 @@
sample 2
-sample 3 text
+sample 3 text
sample 3 repetition
diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py
index 65ffad714..a5e57bde0 100644
--- a/scrapy/tests/test_contrib_linkextractors.py
+++ b/scrapy/tests/test_contrib_linkextractors.py
@@ -110,6 +110,13 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition') ])
+ lx = SgmlLinkExtractor(allow=('sample', ))
+ self.assertEqual([link for link in lx.extract_links(self.response)],
+ [ Link(url='http://example.com/sample1.html', text=u''),
+ Link(url='http://example.com/sample2.html', text=u'sample 2'),
+ Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
+ ])
+
lx = SgmlLinkExtractor(allow=('sample', ), deny=('3', ))
self.assertEqual([link for link in lx.extract_links(self.response)],
[ Link(url='http://example.com/sample1.html', text=u''),