Add create_instance support to addons.

docs/topics/addons.rst

@@ -34,7 +34,7 @@ This is an example where two add-ons are enabled in a project's
 Writing your own add-ons
 ========================
 
-Add-ons are (any) Python *objects* that include the following method:
+Add-ons are (any) Python objects that include the following method:
 
 .. method:: update_settings(settings)
 
@@ -47,6 +47,20 @@ Add-ons are (any) Python *objects* that include the following method:
     :param settings: The settings object storing Scrapy/component configuration
     :type settings: :class:`~scrapy.settings.Settings`
 
+They can also have the following method:
+
+.. classmethod:: from_crawler(cls, crawler)
+   :noindex:
+
+   If present, this class method is called to create an addon instance
+   from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
+   of the addon. Crawler object provides access to all Scrapy core
+   components like settings and signals; it is a way for pipeline to
+   access them and hook its functionality into Scrapy.
+
+   :param crawler: The crawler that uses this addon
+   :type crawler: :class:`~scrapy.crawler.Crawler`
+
 
 Add-on examples
 ===============
@@ -67,3 +81,17 @@ Check dependencies::
             except ImportError:
                 raise RuntimeError("MyAddon requires the boto library")
             ...
+
+Access the crawler instance::
+
+    class MyAddon:
+        def __init__(self, crawler) -> None:
+            super().__init__()
+            self.crawler = crawler
+
+        @classmethod
+        def from_crawler(cls, crawler: Crawler):
+            return cls(crawler)
+
+        def update_settings(self, settings):
+            ...

scrapy/addons.py

@@ -1,13 +1,17 @@
-from typing import Any, List
+from typing import TYPE_CHECKING, Any, List
 
 from scrapy.utils.conf import build_component_list
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import create_instance, load_object
+
+if TYPE_CHECKING:
+    from scrapy.crawler import Crawler
 
 
 class AddonManager:
     """This class facilitates loading and storing :ref:`topics-addons`."""
 
-    def __init__(self) -> None:
+    def __init__(self, crawler: "Crawler") -> None:
+        self.crawler: "Crawler" = crawler
         self.addons: List[Any] = []
 
     def add(self, addon: Any) -> None:
@@ -22,7 +26,7 @@ class AddonManager:
         if isinstance(addon, (type, str)):
             addon = load_object(addon)
         if isinstance(addon, type):
-            addon = addon()
+            addon = create_instance(addon, settings=None, crawler=self.crawler)
         self.addons.append(addon)
 
     def load_settings(self, settings) -> None:

scrapy/crawler.py

@@ -69,7 +69,7 @@ class Crawler:
         self.settings: Settings = settings.copy()
         self.spidercls.update_settings(self.settings)
 
-        self.addons: AddonManager = AddonManager()
+        self.addons: AddonManager = AddonManager(self)
         self.addons.load_settings(self.settings)
         self.addons.update_settings(self.settings)
 

tests/test_addons.py

@@ -1,13 +1,12 @@
 import unittest
 from typing import Any, Dict, Optional
 
-from scrapy.addons import AddonManager
+from scrapy.crawler import Crawler
 from scrapy.settings import BaseSettings
+from scrapy.utils.test import get_crawler
 
 
 class GoodAddon:
-    name = "GoodAddon"
-
     def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
         super().__init__()
         self.config = config or {}
@@ -16,6 +15,20 @@ class GoodAddon:
         settings.update(self.config, "addon")
 
 
+class CreateInstanceAddon:
+    def __init__(self, crawler: Crawler) -> None:
+        super().__init__()
+        self.crawler = crawler
+        self.config = crawler.settings.getdict("MYADDON")
+
+    @classmethod
+    def from_crawler(cls, crawler: Crawler):
+        return cls(crawler)
+
+    def update_settings(self, settings):
+        settings.update(self.config, "addon")
+
+
 class AddonTest(unittest.TestCase):
     def test_update_settings(self):
         settings = BaseSettings()
@@ -30,21 +43,26 @@ class AddonTest(unittest.TestCase):
 
 
 class AddonManagerTest(unittest.TestCase):
-    def setUp(self):
-        self.manager = AddonManager()
-
     def test_add(self):
-        manager = AddonManager()
+        crawler = get_crawler()
+        manager = crawler.addons
         manager.add("tests.test_addons.GoodAddon")
         self.assertIsInstance(manager.addons[0], GoodAddon)
 
     def test_load_settings(self):
-        settings = BaseSettings()
-        settings.set(
-            "ADDONS",
-            {"tests.test_addons.GoodAddon": 0},
-        )
-        settings.set("GOODADDON", {"key": "val2"})
-        manager = AddonManager()
-        manager.load_settings(settings)
+        settings_dict = {
+            "ADDONS": {"tests.test_addons.GoodAddon": 0},
+        }
+        crawler = get_crawler(settings_dict=settings_dict)
+        manager = crawler.addons
         self.assertIsInstance(manager.addons[0], GoodAddon)
+
+    def test_create_instance(self):
+        settings_dict = {
+            "ADDONS": {"tests.test_addons.CreateInstanceAddon": 0},
+            "MYADDON": {"MYADDON_KEY": "val"},
+        }
+        crawler = get_crawler(settings_dict=settings_dict)
+        manager = crawler.addons
+        self.assertIsInstance(manager.addons[0], CreateInstanceAddon)
+        self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val")