diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py index d7e42f6f8..7dc3dacd6 100644 --- a/scrapy/contrib/spiders/crawl.py +++ b/scrapy/contrib/spiders/crawl.py @@ -86,6 +86,13 @@ class CrawlSpider(Spider): rule.process_links = get_method(rule.process_links) rule.process_request = get_method(rule.process_request) + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs) + spider._follow_links = crawler.settings.getbool( + 'CRAWLSPIDER_FOLLOW_LINKS', True) + return spider + def set_crawler(self, crawler): super(CrawlSpider, self).set_crawler(crawler) self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True) diff --git a/tests/test_spider.py b/tests/test_spider.py index 53daf39fb..188bef6e4 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -220,6 +220,30 @@ class CrawlSpiderTest(SpiderTest): 'http://example.org/about.html', 'http://example.org/nofollow.html']) + def test_follow_links_attribute_population(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, 'example.com') + self.assertTrue(hasattr(spider, '_follow_links')) + self.assertTrue(spider._follow_links) + + crawler.settings.set('CRAWLSPIDER_FOLLOW_LINKS', False) + spider = self.spider_class.from_crawler(crawler, 'example.com') + self.assertTrue(hasattr(spider, '_follow_links')) + self.assertFalse(spider._follow_links) + + def test_follow_links_attribute_deprecated_population(self): + spider = self.spider_class('example.com') + self.assertFalse(hasattr(spider, '_follow_links')) + + spider.set_crawler(get_crawler()) + self.assertTrue(hasattr(spider, '_follow_links')) + self.assertTrue(spider._follow_links) + + spider = self.spider_class('example.com') + spider.set_crawler(get_crawler({'CRAWLSPIDER_FOLLOW_LINKS': False})) + self.assertTrue(hasattr(spider, '_follow_links')) + self.assertFalse(spider._follow_links) + class SitemapSpiderTest(SpiderTest):