mirror of https://github.com/scrapy/scrapy.git
106 lines
3.8 KiB
Python
106 lines
3.8 KiB
Python
import warnings
|
|
import unittest
|
|
|
|
import scrapy
|
|
from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
|
|
from scrapy.settings import Settings, default_settings
|
|
from scrapy.spiderloader import SpiderLoader
|
|
from scrapy.utils.spider import DefaultSpider
|
|
from scrapy.utils.misc import load_object
|
|
|
|
|
|
class BaseCrawlerTest(unittest.TestCase):
|
|
|
|
def assertOptionIsDefault(self, settings, key):
|
|
self.assertIsInstance(settings, Settings)
|
|
self.assertEqual(settings[key], getattr(default_settings, key))
|
|
|
|
|
|
class CrawlerTestCase(BaseCrawlerTest):
|
|
|
|
def setUp(self):
|
|
self.crawler = Crawler(DefaultSpider, Settings())
|
|
|
|
def test_deprecated_attribute_spiders(self):
|
|
with warnings.catch_warnings(record=True) as w:
|
|
spiders = self.crawler.spiders
|
|
self.assertEqual(len(w), 1)
|
|
self.assertIn("Crawler.spiders", str(w[0].message))
|
|
sl_cls = load_object(self.crawler.settings['SPIDER_LOADER_CLASS'])
|
|
self.assertIsInstance(spiders, sl_cls)
|
|
|
|
self.crawler.spiders
|
|
self.assertEqual(len(w), 1, "Warn deprecated access only once")
|
|
|
|
def test_crawler_accepts_dict(self):
|
|
crawler = Crawler(DefaultSpider, {'foo': 'bar'})
|
|
self.assertEqual(crawler.settings['foo'], 'bar')
|
|
self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
|
|
|
|
def test_crawler_accepts_None(self):
|
|
crawler = Crawler(DefaultSpider)
|
|
self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
|
|
|
|
|
|
class SpiderLoaderWithWrongInterface(object):
|
|
|
|
def unneeded_method(self):
|
|
pass
|
|
|
|
|
|
class CustomSpiderLoader(SpiderLoader):
|
|
pass
|
|
|
|
|
|
class CrawlerRunnerTestCase(BaseCrawlerTest):
|
|
|
|
def test_spider_manager_verify_interface(self):
|
|
settings = Settings({
|
|
'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface'
|
|
})
|
|
with warnings.catch_warnings(record=True) as w, \
|
|
self.assertRaises(AttributeError):
|
|
CrawlerRunner(settings)
|
|
self.assertEqual(len(w), 1)
|
|
self.assertIn("SPIDER_LOADER_CLASS", str(w[0].message))
|
|
self.assertIn("scrapy.interfaces.ISpiderLoader", str(w[0].message))
|
|
|
|
def test_crawler_runner_accepts_dict(self):
|
|
runner = CrawlerRunner({'foo': 'bar'})
|
|
self.assertEqual(runner.settings['foo'], 'bar')
|
|
self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED')
|
|
|
|
def test_crawler_runner_accepts_None(self):
|
|
runner = CrawlerRunner()
|
|
self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED')
|
|
|
|
def test_deprecated_attribute_spiders(self):
|
|
with warnings.catch_warnings(record=True) as w:
|
|
runner = CrawlerRunner(Settings())
|
|
spiders = runner.spiders
|
|
self.assertEqual(len(w), 1)
|
|
self.assertIn("CrawlerRunner.spiders", str(w[0].message))
|
|
self.assertIn("CrawlerRunner.spider_loader", str(w[0].message))
|
|
sl_cls = load_object(runner.settings['SPIDER_LOADER_CLASS'])
|
|
self.assertIsInstance(spiders, sl_cls)
|
|
|
|
def test_spidermanager_deprecation(self):
|
|
with warnings.catch_warnings(record=True) as w:
|
|
runner = CrawlerRunner({
|
|
'SPIDER_MANAGER_CLASS': 'tests.test_crawler.CustomSpiderLoader'
|
|
})
|
|
self.assertIsInstance(runner.spider_loader, CustomSpiderLoader)
|
|
self.assertEqual(len(w), 1)
|
|
self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message))
|
|
|
|
|
|
class CrawlerProcessTest(BaseCrawlerTest):
|
|
def test_crawler_process_accepts_dict(self):
|
|
runner = CrawlerProcess({'foo': 'bar'})
|
|
self.assertEqual(runner.settings['foo'], 'bar')
|
|
self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED')
|
|
|
|
def test_crawler_process_accepts_None(self):
|
|
runner = CrawlerProcess()
|
|
self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED')
|