RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update from_crawler method as well as set_crawler on CrawlSpider

This commit is contained in:
Julia Medina 2014-06-30 03:20:05 -03:00
parent 84fa004793
commit eb0253e530
2 changed files with 31 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scrapy/contrib/spiders/crawl.py
View File

@ -86,6 +86,13 @@ class CrawlSpider(Spider):
rule.process_links = get_method(rule.process_links)
rule.process_request = get_method(rule.process_request)
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs)
spider._follow_links = crawler.settings.getbool(
'CRAWLSPIDER_FOLLOW_LINKS', True)
return spider
def set_crawler(self, crawler):
super(CrawlSpider, self).set_crawler(crawler)
self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)

24
tests/test_spider.py
View File

@ -220,6 +220,30 @@ class CrawlSpiderTest(SpiderTest):
'http://example.org/about.html',
'http://example.org/nofollow.html'])
def test_follow_links_attribute_population(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, 'example.com')
self.assertTrue(hasattr(spider, '_follow_links'))
self.assertTrue(spider._follow_links)
crawler.settings.set('CRAWLSPIDER_FOLLOW_LINKS', False)
spider = self.spider_class.from_crawler(crawler, 'example.com')
self.assertTrue(hasattr(spider, '_follow_links'))
self.assertFalse(spider._follow_links)
def test_follow_links_attribute_deprecated_population(self):
spider = self.spider_class('example.com')
self.assertFalse(hasattr(spider, '_follow_links'))
spider.set_crawler(get_crawler())
self.assertTrue(hasattr(spider, '_follow_links'))
self.assertTrue(spider._follow_links)
spider = self.spider_class('example.com')
spider.set_crawler(get_crawler({'CRAWLSPIDER_FOLLOW_LINKS': False}))
self.assertTrue(hasattr(spider, '_follow_links'))
self.assertFalse(spider._follow_links)
class SitemapSpiderTest(SpiderTest):