diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 3421864fa..f1cc070ad 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -34,7 +34,7 @@ This is an example where two add-ons are enabled in a project's Writing your own add-ons ======================== -Add-ons are (any) Python *objects* that include the following method: +Add-ons are (any) Python objects that include the following method: .. method:: update_settings(settings) @@ -47,6 +47,20 @@ Add-ons are (any) Python *objects* that include the following method: :param settings: The settings object storing Scrapy/component configuration :type settings: :class:`~scrapy.settings.Settings` +They can also have the following method: + +.. classmethod:: from_crawler(cls, crawler) + :noindex: + + If present, this class method is called to create an addon instance + from a :class:`~scrapy.crawler.Crawler`. It must return a new instance + of the addon. Crawler object provides access to all Scrapy core + components like settings and signals; it is a way for pipeline to + access them and hook its functionality into Scrapy. + + :param crawler: The crawler that uses this addon + :type crawler: :class:`~scrapy.crawler.Crawler` + Add-on examples =============== @@ -67,3 +81,17 @@ Check dependencies:: except ImportError: raise RuntimeError("MyAddon requires the boto library") ... + +Access the crawler instance:: + + class MyAddon: + def __init__(self, crawler) -> None: + super().__init__() + self.crawler = crawler + + @classmethod + def from_crawler(cls, crawler: Crawler): + return cls(crawler) + + def update_settings(self, settings): + ... diff --git a/scrapy/addons.py b/scrapy/addons.py index bb4664d8e..ba33f1865 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,13 +1,17 @@ -from typing import Any, List +from typing import TYPE_CHECKING, Any, List from scrapy.utils.conf import build_component_list -from scrapy.utils.misc import load_object +from scrapy.utils.misc import create_instance, load_object + +if TYPE_CHECKING: + from scrapy.crawler import Crawler class AddonManager: """This class facilitates loading and storing :ref:`topics-addons`.""" - def __init__(self) -> None: + def __init__(self, crawler: "Crawler") -> None: + self.crawler: "Crawler" = crawler self.addons: List[Any] = [] def add(self, addon: Any) -> None: @@ -22,7 +26,7 @@ class AddonManager: if isinstance(addon, (type, str)): addon = load_object(addon) if isinstance(addon, type): - addon = addon() + addon = create_instance(addon, settings=None, crawler=self.crawler) self.addons.append(addon) def load_settings(self, settings) -> None: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 7a26cd2e4..12256440b 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -69,7 +69,7 @@ class Crawler: self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) - self.addons: AddonManager = AddonManager() + self.addons: AddonManager = AddonManager(self) self.addons.load_settings(self.settings) self.addons.update_settings(self.settings) diff --git a/tests/test_addons.py b/tests/test_addons.py index 8ba27236d..d52665869 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,13 +1,12 @@ import unittest from typing import Any, Dict, Optional -from scrapy.addons import AddonManager +from scrapy.crawler import Crawler from scrapy.settings import BaseSettings +from scrapy.utils.test import get_crawler class GoodAddon: - name = "GoodAddon" - def __init__(self, config: Optional[Dict[str, Any]] = None) -> None: super().__init__() self.config = config or {} @@ -16,6 +15,20 @@ class GoodAddon: settings.update(self.config, "addon") +class CreateInstanceAddon: + def __init__(self, crawler: Crawler) -> None: + super().__init__() + self.crawler = crawler + self.config = crawler.settings.getdict("MYADDON") + + @classmethod + def from_crawler(cls, crawler: Crawler): + return cls(crawler) + + def update_settings(self, settings): + settings.update(self.config, "addon") + + class AddonTest(unittest.TestCase): def test_update_settings(self): settings = BaseSettings() @@ -30,21 +43,26 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): - def setUp(self): - self.manager = AddonManager() - def test_add(self): - manager = AddonManager() + crawler = get_crawler() + manager = crawler.addons manager.add("tests.test_addons.GoodAddon") self.assertIsInstance(manager.addons[0], GoodAddon) def test_load_settings(self): - settings = BaseSettings() - settings.set( - "ADDONS", - {"tests.test_addons.GoodAddon": 0}, - ) - settings.set("GOODADDON", {"key": "val2"}) - manager = AddonManager() - manager.load_settings(settings) + settings_dict = { + "ADDONS": {"tests.test_addons.GoodAddon": 0}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons self.assertIsInstance(manager.addons[0], GoodAddon) + + def test_create_instance(self): + settings_dict = { + "ADDONS": {"tests.test_addons.CreateInstanceAddon": 0}, + "MYADDON": {"MYADDON_KEY": "val"}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + self.assertIsInstance(manager.addons[0], CreateInstanceAddon) + self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val")