diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 016f2d8fd..2207763c9 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -71,7 +71,7 @@ class BaseSgmlLinkExtractor(FixedSGMLParser): if self.scan_attr(attr): url = self.process_value(value) if url is not None: - link = Link(url=url) + link = Link(url=url, nofollow=True if dict(attrs).get('rel') == 'nofollow' else False) self.links.append(link) self.current_link = link diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index f986a5bc7..f49e90792 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -85,6 +85,17 @@ class LinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.matches(url1), True) self.assertEqual(lx.matches(url2), True) + def test_link_nofollow(self): + html = """ + Printer-friendly page + About us + """ + response = HtmlResponse("http://example.org/page.html", body=html) + lx = SgmlLinkExtractor() + self.assertEqual([link for link in lx.extract_links(response)], + [ Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True), + Link(url='http://example.org/about.html', text=u'About us', nofollow=False) ]) + class SgmlLinkExtractorTestCase(unittest.TestCase): def setUp(self): body = get_testdata('link_extractor', 'sgml_linkextractor.html')