Remove more code.

docs/topics/addons.rst

@@ -17,40 +17,26 @@ Activating and configuring add-ons
 Add-ons and their configuration live in Scrapy's
 :class:`~scrapy.addons.AddonManager`. During a :class:`~scrapy.crawler.Crawler`
 initialization the add-on manager will read a list of enabled add-ons from your
-``ADDONS`` setting and their optional configuration from the respective
+``ADDONS`` setting.
-settings.
 
 The ``ADDONS`` setting is a dict in which every key is an addon class or its
 import path and the value is its priority.
 
-The configuration of an add-on, if necessary at all, is stored as a dictionary
+This is an example where two add-ons are enabled in a project's
-setting whose name is the uppercase add-on name.
+``settings.py``::
-
-This is an example where two add-ons (in this case with one requiring no
-configuration) are enabled/configured in a project's ``settings.py``::
 
     ADDONS = {
         'path.to.someaddon': 0,
         path.to.someaddon2: 1,
     }
 
-    SOMEADDON = {
-        'some_config': True,
-    }
-
 
 Writing your own add-ons
 ========================
 
-Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*:
+Add-ons are (any) Python *objects* that include the following method:
 
-.. attribute:: name
+.. method:: update_settings(settings)
-
-    string with add-on name
-
-    :type: ``str``
-
-.. method:: update_settings(config, settings)
 
     This method is called during the initialization of the
     :class:`~scrapy.crawler.Crawler`. Here, you should perform dependency checks
@@ -58,88 +44,26 @@ Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*:
     :class:`~scrapy.settings.Settings` object as wished, e.g. enable components
     for this add-on or set required configuration of other extensions.
 
-    :param config: Configuration of this add-on
-    :type config: ``dict``
-
     :param settings: The settings object storing Scrapy/component configuration
     :type settings: :class:`~scrapy.settings.Settings`
 
-.. method:: check_configuration(config, crawler)
-
-    This method is called when the :class:`~scrapy.crawler.Crawler` has been
-    fully initialized, immediately before it starts crawling. You can perform
-    additional dependency and configuration checks here.
-
-    :param config: Configuration of this add-on
-    :type config: ``dict``
-
-    :param crawler: Fully initialized Scrapy crawler
-    :type crawler: :class:`~scrapy.crawler.Crawler`
-
-
-Add-on base class
-=================
-
-Scrapy comes with a built-in base class for add-ons which provides some
-convenience functionality: the add-on configuration can be exposed into
-Scrapy's settings via :meth:`~scrapy.addons.Addon.export_config`, configurable
-via :attr:`~scrapy.addons.Addon.default_config` and
-:attr:`~scrapy.addons.Addon.config_mapping`.
-
-By default, the base add-on class will expose the add-on configuration into
-Scrapy's settings namespace, in upper case. It is
-easy to write your own functionality while still being able to use the
-convenience functions by overwriting
-:meth:`~scrapy.addons.Addon.update_settings`.
-
-.. module:: scrapy.addons
-   :noindex:
-
-.. autoclass:: Addon
-   :members:
-
 
 Add-on examples
 ===============
 
-Set some basic configuration using the :class:`Addon` base class::
+Set some basic configuration::
 
-    from scrapy.addons import Addon
+    class MyAddon:
-
+        def update_settings(self, settings):
-    class MyAddon(Addon):
-        name = 'myaddon'
-
-        def update_settings(self, config, settings):
-            super().update_settings(settings)
             settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200
             settings["DNSCACHE_ENABLED"] = True
 
 Check dependencies::
 
-    from scrapy.addons import Addon
+    class MyAddon:
-
+        def update_settings(self, settings):
-    class MyAddon(Addon):
-        name = 'myaddon'
-
-        def update_settings(self, config, settings):
             try:
                 import boto
             except ImportError:
-                raise RuntimeError("myaddon requires the boto library")
+                raise RuntimeError("MyAddon requires the boto library")
-            super().update_settings(settings)
+            ...
-
-Check configuration of fully initialized crawler (see
-:ref:`topics-api-crawler`)::
-
-    class MyAddon(object):
-        name = 'myaddon'
-
-        def update_settings(self, config, settings):
-            super().update_settings(settings)
-            settings.set('DNSCACHE_ENABLED', False, priority='addon')
-
-        def check_configuration(self, config, crawler):
-            if crawler.settings.getbool('DNSCACHE_ENABLED'):
-                # The spider, some other add-on, or the user messed with the
-                # DNS cache setting
-                raise ValueError("myaddon is incompatible with DNS cache")

scrapy/addons.py

@@ -1,100 +1,16 @@
-from typing import Any, Dict, Iterator, Mapping, Optional, OrderedDict
+from typing import Any, List
 
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.misc import load_object
 
 
-class Addon(object):
+class AddonManager:
-    name: str
+    """This class facilitates loading and storing :ref:`topics-addons`."""
-
-    default_config = None
-    """``dict`` with default configuration."""
-
-    config_mapping = None
-    """``dict`` with mappings from config names to setting names. The given
-    setting names will be taken as given, not uppercased.
-    """
-
-    def export_config(self, config, settings):
-        """Export the add-on configuration, all keys in caps, into the settings
-        object.
-
-        For example, the add-on configuration ``{'key': 'value'}`` will export
-        the setting ``KEY`` with a value of ``value``. All settings
-        will be exported with ``addon`` priority (see
-        :ref:`topics-api-settings`).
-
-        :param config: Add-on configuration to be exposed
-        :type config: ``dict``
-
-        :param settings: Settings object into which to export the configuration
-        :type settings: :class:`~scrapy.settings.Settings`
-        """
-        conf = self.default_config or {}
-        conf.update(config)
-        # Since default exported config is case-insensitive (everything will be
-        # uppercased), make mapped config case-insensitive as well
-        conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()}
-        for key, val in conf.items():
-            if key.lower() in conf_mapping:
-                key = conf_mapping[key.lower()]
-            else:
-                key = key.upper()
-            settings.set(key, val, "addon")
-
-    def update_settings(self, config, settings):
-        """Modifiy `settings` to enable and configure required components.
-
-        :param config: Add-on configuration
-        :type config: ``dict``
-
-        :param settings: Crawler settings object
-        :type settings: :class:`~scrapy.settings.Settings`
-        """
-        self.export_config(config, settings)
-
-    def check_configuration(self, config, crawler):
-        """Perform post-initialization checks on fully configured `crawler`.
-
-        :param config: Add-on configuration
-        :type config: ``dict``
-
-        :param crawler: the fully-initialized crawler
-        :type crawler: :class:`~scrapy.crawler.Crawler`
-        """
-        pass
-
-
-class AddonManager(Mapping[str, Addon]):
-    """This class facilitates loading and storing :ref:`topics-addons`.
-
-    You can treat it like a read-only dictionary in which keys correspond to
-    add-on names and values correspond to the add-on objects. Add-on
-    configurations are saved in the :attr:`config` dictionary attribute::
-
-        addons = AddonManager()
-        # ... load some add-ons here
-        print(addons.enabled)  # prints names of all enabled add-ons
-        print(addons['TestAddon'].version)  # prints version of add-on with name
-                                           # 'TestAddon'
-        print(addons.configs['TestAddon'])  # prints configuration of 'TestAddon'
-
-    """
 
     def __init__(self) -> None:
-        self._addons: OrderedDict[str, Addon] = OrderedDict[str, Addon]()
+        self.addons: List[Any] = []
-        self.configs: Dict[str, Dict[str, Any]] = {}
 
-    def __getitem__(self, name: str) -> Addon:
+    def add(self, addon: Any) -> None:
-        return self._addons[name]
-
-    def __iter__(self) -> Iterator[str]:
-        return iter(self._addons)
-
-    def __len__(self) -> int:
-        return len(self._addons)
-
-    def add(self, addon: Any, config: Optional[Dict[str, Any]] = None):
         """Store an add-on.
 
         :param addon: The add-on object (or path) to be stored
@@ -107,19 +23,13 @@ class AddonManager(Mapping[str, Addon]):
             addon = load_object(addon)
         if isinstance(addon, type):
             addon = addon()
-        name = addon.name
+        self.addons.append(addon)
-        if name in self:
-            raise ValueError(f"Addon '{name}' already loaded")
-        self._addons[name] = addon
-        self.configs[name] = config or {}
 
-    def load_settings(self, settings):
+    def load_settings(self, settings) -> None:
         """Load add-ons and configurations from settings object.
 
         This will load the addon for every add-on path in the
-        ``ADDONS`` setting. For each of these add-ons, the configuration will be
+        ``ADDONS`` setting.
-        read from the dictionary setting whose name matches the uppercase add-on
-        name.
 
         :param settings: The :class:`~scrapy.settings.Settings` object from \
             which to read the add-on configuration
@@ -127,9 +37,8 @@ class AddonManager(Mapping[str, Addon]):
         """
         paths = build_component_list(settings["ADDONS"])
         addons = [load_object(path) for path in paths]
-        configs = [settings.getdict(addon.name.upper()) for addon in addons]
+        for a in addons:
-        for a, c in zip(addons, configs):
+            self.add(a)
-            self.add(a, c)
 
     def update_settings(self, settings) -> None:
         """Call ``update_settings()`` of all held add-ons.
@@ -138,14 +47,5 @@ class AddonManager(Mapping[str, Addon]):
             updated
         :type settings: :class:`~scrapy.settings.Settings`
         """
-        for name in self:
+        for addon in self.addons:
-            self[name].update_settings(self.configs[name], settings)
+            addon.update_settings(settings)
-
-    def check_configuration(self, crawler) -> None:
-        """Call ``check_configuration()`` of all held add-ons.
-
-        :param crawler: the fully-initialized crawler
-        :type crawler: :class:`~scrapy.crawler.Crawler`
-        """
-        for name in self:
-            self[name].check_configuration(self.configs[name], crawler)

tests/test_addons.py

@@ -1,47 +1,29 @@
 import unittest
+from typing import Any, Dict, Optional
 
-from scrapy.addons import Addon, AddonManager
+from scrapy.addons import AddonManager
 from scrapy.settings import BaseSettings
 
 
-class GoodAddon(object):
+class GoodAddon:
     name = "GoodAddon"
 
-    def update_settings(self, config, settings):
+    def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
-        pass
+        super().__init__()
+        self.config = config or {}
 
-    def check_configuration(self, config, crawler):
+    def update_settings(self, settings):
-        pass
+        settings.update(self.config, "addon")
 
 
 class AddonTest(unittest.TestCase):
-    def setUp(self):
-        class AddonWithAttributes(Addon):
-            name = "Test"
-
-        self.testaddon = AddonWithAttributes()
-
-    def test_export_config(self):
-        settings = BaseSettings()
-        self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"}
-        self.testaddon.default_config = {"key": 55, "defaultkey": 100}
-        self.testaddon.export_config(
-            {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings
-        )
-        self.assertEqual(settings["KEY"], 313)
-        self.assertEqual(settings["DEFAULTKEY"], 100)
-        self.assertEqual(settings["OTHERKEY"], True)
-        self.assertNotIn("MAPPED_key", settings)
-        self.assertNotIn("MAPPED_KEY", settings)
-        self.assertEqual(settings["MAPPING_WORKED"], 99)
-        self.assertEqual(settings.getpriority("KEY"), 15)
-
     def test_update_settings(self):
         settings = BaseSettings()
         settings.set("KEY1", "default", priority="default")
         settings.set("KEY2", "project", priority="project")
-        addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"}
+        addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"}
-        self.testaddon.update_settings(addon_config, settings)
+        testaddon = GoodAddon(addon_config)
+        testaddon.update_settings(settings)
         self.assertEqual(settings["KEY1"], "addon")
         self.assertEqual(settings["KEY2"], "project")
         self.assertEqual(settings["KEY3"], "addon")
@@ -54,8 +36,7 @@ class AddonManagerTest(unittest.TestCase):
     def test_add(self):
         manager = AddonManager()
         manager.add("tests.test_addons.GoodAddon")
-        self.assertCountEqual(manager, ["GoodAddon"])
+        self.assertIsInstance(manager.addons[0], GoodAddon)
-        self.assertIsInstance(manager["GoodAddon"], GoodAddon)
 
     def test_load_settings(self):
         settings = BaseSettings()
@@ -66,7 +47,4 @@ class AddonManagerTest(unittest.TestCase):
         settings.set("GOODADDON", {"key": "val2"})
         manager = AddonManager()
         manager.load_settings(settings)
-        self.assertCountEqual(manager, ["GoodAddon"])
+        self.assertIsInstance(manager.addons[0], GoodAddon)
-        self.assertIsInstance(manager["GoodAddon"], GoodAddon)
-        self.assertCountEqual(manager.configs["GoodAddon"], ["key"])
-        self.assertEqual(manager.configs["GoodAddon"]["key"], "val2")

tests/test_crawl.py

@@ -13,7 +13,6 @@ from twisted.python.failure import Failure
 from twisted.trial.unittest import TestCase
 
 from scrapy import signals
-from scrapy.addons import Addon, AddonManager
 from scrapy.crawler import CrawlerRunner
 from scrapy.exceptions import StopDownload
 from scrapy.http import Request
@@ -408,21 +407,6 @@ with multiples lines
         self._assert_retried(log)
         self.assertIn("Got response 200", str(log))
 
-    @defer.inlineCallbacks
-    def test_abort_on_addon_failed_check(self):
-        class FailedCheckAddon(Addon):
-            name = "FailedCheckAddon"
-
-            def check_configuration(self, config, crawler):
-                raise ValueError
-
-        addonmgr = AddonManager()
-        addonmgr.add(FailedCheckAddon())
-        crawler = get_crawler(SimpleSpider)
-        crawler.addons = addonmgr
-        with self.assertRaises(ValueError):
-            yield crawler.crawl()
-
 
 class CrawlSpiderTestCase(TestCase):
     def setUp(self):