Integrate add-ons into start-up process

scrapy/cmdline.py

@@ -6,6 +6,7 @@ import inspect
 import pkg_resources
 
 import scrapy
+from scrapy.addons import AddonManager
 from scrapy.crawler import CrawlerProcess
 from scrapy.xlib import lsprofcalltree
 from scrapy.commands import ScrapyCommand
@@ -118,6 +119,9 @@ def execute(argv=None, settings=None):
         conf.settings = settings
     # ------------------------------------------------------------------
 
+    addons = AddonManager()
+    addons.load_cfg()
+
     inproject = inside_project()
     cmds = _get_commands_dict(settings, inproject)
     cmdname = _pop_command_name(argv)
@@ -139,7 +143,7 @@ def execute(argv=None, settings=None):
     opts, args = parser.parse_args(args=argv[1:])
     _run_print_help(parser, cmd.process_options, args, opts)
 
-    cmd.crawler_process = CrawlerProcess(settings)
+    cmd.crawler_process = CrawlerProcess(settings, addons)
     _run_print_help(parser, _run_command, cmd, args, opts)
     sys.exit(cmd.exitcode)
 

scrapy/crawler.py

@@ -6,6 +6,7 @@ import warnings
 from twisted.internet import reactor, defer
 from zope.interface.verify import verifyClass, DoesNotImplement
 
+from scrapy.addons import AddonManager
 from scrapy.core.engine import ExecutionEngine
 from scrapy.resolver import CachingThreadedResolver
 from scrapy.interfaces import ISpiderLoader
@@ -23,7 +24,7 @@ logger = logging.getLogger(__name__)
 
 class Crawler(object):
 
-    def __init__(self, spidercls, settings=None):
+    def __init__(self, spidercls, settings=None, addons=None):
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
 
@@ -31,6 +32,12 @@ class Crawler(object):
         self.settings = settings.copy()
         self.spidercls.update_settings(self.settings)
 
+        self.addons = addons if addons is not None else AddonManager()
+        self.addons.load_settings(self.settings)
+        self.addons.update_addons()
+        self.addons.check_dependency_clashes()
+        self.addons.update_settings(self.settings)
+
         self.signals = SignalManager(self)
         self.stats = load_object(self.settings['STATS_CLASS'])(self)
 
@@ -69,6 +76,7 @@ class Crawler(object):
         try:
             self.spider = self._create_spider(*args, **kwargs)
             self.engine = self._create_engine()
+            self.addons.check_configuration(self)
             start_requests = iter(self.spider.start_requests())
             yield self.engine.open_spider(self.spider, start_requests)
             yield defer.maybeDeferred(self.engine.start)
@@ -111,10 +119,11 @@ class CrawlerRunner(object):
             ":meth:`crawl` and managed by this class."
     )
 
-    def __init__(self, settings=None):
+    def __init__(self, settings=None, addons=None):
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
         self.settings = settings
+        self.addons = addons
         self.spider_loader = _get_spider_loader(settings)
         self._crawlers = set()
         self._active = set()
@@ -181,7 +190,7 @@ class CrawlerRunner(object):
     def _create_crawler(self, spidercls):
         if isinstance(spidercls, six.string_types):
             spidercls = self.spider_loader.load(spidercls)
-        return Crawler(spidercls, self.settings)
+        return Crawler(spidercls, self.settings, self.addons)
 
     def stop(self):
         """
@@ -223,8 +232,8 @@ class CrawlerProcess(CrawlerRunner):
     process. See :ref:`run-from-script` for an example.
     """
 
-    def __init__(self, settings=None):
-        super(CrawlerProcess, self).__init__(settings)
+    def __init__(self, settings=None, addons=None):
+        super(CrawlerProcess, self).__init__(settings, addons)
         install_shutdown_handlers(self._signal_shutdown)
         configure_logging(self.settings)
         log_scrapy_info(self.settings)

scrapy/utils/test.py

@@ -20,7 +20,7 @@ def assert_aws_environ():
     if 'AWS_ACCESS_KEY_ID' not in os.environ:
         raise SkipTest("AWS keys not found")
 
-def get_crawler(spidercls=None, settings_dict=None):
+def get_crawler(spidercls=None, settings_dict=None, addons=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
     will be used to populate the crawler settings with a project level
     priority.
@@ -28,7 +28,7 @@ def get_crawler(spidercls=None, settings_dict=None):
     from scrapy.crawler import CrawlerRunner
     from scrapy.spiders import Spider
 
-    runner = CrawlerRunner(settings_dict)
+    runner = CrawlerRunner(settings_dict, addons)
     return runner.create_crawler(spidercls or Spider)
 
 def get_pythonpath():

tests/test_crawl.py

@@ -6,6 +6,7 @@ from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 
+from scrapy.addons import Addon, AddonManager
 from scrapy.http import Request
 from scrapy.crawler import CrawlerRunner
 from tests import mock
@@ -266,3 +267,19 @@ with multiples lines
 
         self._assert_retried(log)
         self.assertIn("Got response 200", str(log))
+
+    @defer.inlineCallbacks
+    def test_abort_on_addon_failed_check(self):
+        class FailedCheckAddon(Addon):
+            name = 'FailedCheckAddon'
+            version = '1.0'
+            def check_configuration(self, config, crawler):
+                raise ValueError
+        addonmgr = AddonManager()
+        addonmgr.add(FailedCheckAddon())
+        crawler = self.runner.create_crawler(SimpleSpider)
+        crawler.addons = addonmgr
+        # Doesn't work in 'precise' test environment:
+        #with self.assertRaises(ValueError):
+        #    yield crawler.crawl()
+        yield self.assertFailure(crawler.crawl(), ValueError)

tests/test_crawler.py

@@ -2,6 +2,7 @@ import warnings
 import unittest
 
 import scrapy
+from scrapy.addons import Addon, AddonManager
 from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
 from scrapy.settings import Settings, default_settings
 from scrapy.spiderloader import SpiderLoader
@@ -51,6 +52,29 @@ class CrawlerTestCase(BaseCrawlerTest):
         self.assertFalse(settings.frozen)
         self.assertTrue(crawler.settings.frozen)
 
+    def test_populate_addons_settings(self):
+        class TestAddon(Addon):
+            name = 'TestAddon'
+            version = '1.0'
+        addonconfig = {'TEST1': 'addon', 'TEST2': 'addon', 'TEST3': 'addon'}
+        class TestAddon2(Addon):
+            name = 'testAddon2'
+            version = '1.0'
+        addonconfig2 = {'TEST': 'addon2'}
+
+        settings = Settings()
+        settings.set('TESTADDON_TEST1', 'project', priority='project')
+        settings.set('TESTADDON_TEST2', 'default', priority='default')
+        addonmgr = AddonManager()
+        addonmgr.add(TestAddon(), addonconfig)
+        addonmgr.add(TestAddon2(), addonconfig2)
+        crawler = Crawler(DefaultSpider, settings, addonmgr)
+
+        self.assertEqual(crawler.settings['TESTADDON_TEST1'], 'project')
+        self.assertEqual(crawler.settings['TESTADDON_TEST2'], 'addon')
+        self.assertEqual(crawler.settings['TESTADDON_TEST3'], 'addon')
+        self.assertEqual(crawler.settings['TESTADDON2_TEST'], 'addon2')
+
     def test_crawler_accepts_dict(self):
         crawler = Crawler(DefaultSpider, {'foo': 'bar'})
         self.assertEqual(crawler.settings['foo'], 'bar')