diff --git a/docs/index.rst b/docs/index.rst index 5404969e0..8798aebd1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -222,6 +222,7 @@ Extending Scrapy :hidden: topics/architecture + topics/addons topics/downloader-middleware topics/spider-middleware topics/extensions @@ -235,6 +236,9 @@ Extending Scrapy :doc:`topics/architecture` Understand the Scrapy architecture. +:doc:`topics/addons` + Enable and configure third-party extensions. + :doc:`topics/downloader-middleware` Customize how pages get requested and downloaded. diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst new file mode 100644 index 000000000..1bf2172bd --- /dev/null +++ b/docs/topics/addons.rst @@ -0,0 +1,193 @@ +.. _topics-addons: + +======= +Add-ons +======= + +Scrapy's add-on system is a framework which unifies managing and configuring +components that extend Scrapy's core functionality, such as middlewares, +extensions, or pipelines. It provides users with a plug-and-play experience in +Scrapy extension management, and grants extensive configuration control to +developers. + + +Activating and configuring add-ons +================================== + +During :class:`~scrapy.crawler.Crawler` initialization, the list of enabled +add-ons is read from your ``ADDONS`` setting. + +The ``ADDONS`` setting is a dict in which every key is an add-on class or its +import path and the value is its priority. + +This is an example where two add-ons are enabled in a project's +``settings.py``:: + + ADDONS = { + 'path.to.someaddon': 0, + SomeAddonClass: 1, + } + + +Writing your own add-ons +======================== + +Add-ons are Python classes that include the following method: + +.. method:: update_settings(settings) + + This method is called during the initialization of the + :class:`~scrapy.crawler.Crawler`. Here, you should perform dependency checks + (e.g. for external Python libraries) and update the + :class:`~scrapy.settings.Settings` object as wished, e.g. enable components + for this add-on or set required configuration of other extensions. + + :param settings: The settings object storing Scrapy/component configuration + :type settings: :class:`~scrapy.settings.Settings` + +They can also have the following method: + +.. classmethod:: from_crawler(cls, crawler) + :noindex: + + If present, this class method is called to create an add-on instance + from a :class:`~scrapy.crawler.Crawler`. It must return a new instance + of the add-on. The crawler object provides access to all Scrapy core + components like settings and signals; it is a way for the add-on to access + them and hook its functionality into Scrapy. + + :param crawler: The crawler that uses this add-on + :type crawler: :class:`~scrapy.crawler.Crawler` + +The settings set by the add-on should use the ``addon`` priority (see +:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: + + class MyAddon: + def update_settings(self, settings): + settings.set("DNSCACHE_ENABLED", True, "addon") + +This allows users to override these settings in the project or spider +configuration. This is not possible with settings that are mutable objects, +such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases +you can provide an add-on-specific setting that governs whether the add-on will +modify :setting:`ITEM_PIPELINES`:: + + class MyAddon: + def update_settings(self, settings): + if settings.getbool("MYADDON_ENABLE_PIPELINE"): + settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 + +If the ``update_settings`` method raises +:exc:`scrapy.exceptions.NotConfigured`, the add-on will be skipped. This makes +it easy to enable an add-on only when some conditions are met. + +Fallbacks +--------- + +Some components provided by add-ons need to fall back to "default" +implementations, e.g. a custom download handler needs to send the request that +it doesn't handle via the default download handler, or a stats collector that +includes some additional processing but otherwise uses the default stats +collector. And it's possible that a project needs to use several custom +components of the same type, e.g. two custom download handlers that support +different kinds of custom requests and still need to use the default download +handler for other requests. To make such use cases easier to configure, we +recommend that such custom components should be written in the following way: + +1. The custom component (e.g. ``MyDownloadHandler``) shouldn't inherit from the + default Scrapy one (e.g. + ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler``), but instead + be able to load the class of the fallback component from a special setting + (e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use + it. +2. The add-ons that include these components should read the current value of + the default setting (e.g. ``DOWNLOAD_HANDLERS``) in their + ``update_settings()`` methods, save that value into the fallback setting + (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default + setting to the component provided by the add-on (e.g. + ``MyDownloadHandler``). If the fallback setting is already set by the user, + they shouldn't change it. +3. This way, if there are several add-ons that want to modify the same setting, + all of them will fallback to the component from the previous one and then to + the Scrapy default. The order of that depends on the priority order in the + ``ADDONS`` setting. + + +Add-on examples +=============== + +Set some basic configuration: + +.. code-block:: python + + class MyAddon: + def update_settings(self, settings): + settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 + settings.set("DNSCACHE_ENABLED", True, "addon") + +Check dependencies: + +.. code-block:: python + + class MyAddon: + def update_settings(self, settings): + try: + import boto + except ImportError: + raise NotConfigured("MyAddon requires the boto library") + ... + +Access the crawler instance: + +.. code-block:: python + + class MyAddon: + def __init__(self, crawler) -> None: + super().__init__() + self.crawler = crawler + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def update_settings(self, settings): + ... + +Use a fallback component: + +.. code-block:: python + + from scrapy.core.downloader.handlers.http import HTTPDownloadHandler + + + FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" + + + class MyHandler: + lazy = False + + def __init__(self, settings, crawler): + dhcls = load_object(settings.get(FALLBACK_SETTING)) + self._fallback_handler = create_instance( + dhcls, + settings=None, + crawler=crawler, + ) + + def download_request(self, request, spider): + if request.meta.get("my_params"): + # handle the request + ... + else: + return self._fallback_handler.download_request(request, spider) + + + class MyAddon: + def update_settings(self, settings): + if not settings.get(FALLBACK_SETTING): + settings.set( + FALLBACK_SETTING, + settings.getwithbase("DOWNLOAD_HANDLERS")["https"], + "addon", + ) + settings["DOWNLOAD_HANDLERS"]["https"] = MyHandler diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 268344879..16c28405c 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -137,6 +137,7 @@ Settings API SETTINGS_PRIORITIES = { "default": 0, "command": 10, + "addon": 15, "project": 20, "spider": 30, "cmdline": 40, diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 3e06d84f9..602ab587d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -40,8 +40,9 @@ precedence: 1. Command line options (most precedence) 2. Settings per-spider 3. Project settings module - 4. Default settings per-command - 5. Default global settings (less precedence) + 4. Settings set by add-ons + 5. Default settings per-command + 6. Default global settings (less precedence) The population of these settings sources is taken care of internally, but a manual handling is possible using API calls. See the @@ -89,7 +90,13 @@ project, it's where most of your custom settings will be populated. For a standard Scrapy project, this means you'll be adding or changing the settings in the ``settings.py`` file created for your project. -4. Default settings per-command +4. Settings set by add-ons +-------------------------- + +:ref:`Add-ons ` can modify settings. They should do this with +this priority, though this is not enforced. + +5. Default settings per-command ------------------------------- Each :doc:`Scrapy tool ` command can have its own default @@ -97,7 +104,7 @@ settings, which override the global default settings. Those custom command settings are specified in the ``default_settings`` attribute of the command class. -5. Default global settings +6. Default global settings -------------------------- The global defaults are located in the ``scrapy.settings.default_settings`` @@ -201,6 +208,16 @@ to any particular component. In that case the module of that component will be shown, typically an extension, middleware or pipeline. It also means that the component must be enabled in order for the setting to have any effect. +.. setting:: ADDONS + +ADDONS +------ + +Default: ``{}`` + +A dict containing paths to the add-ons enabled in your project and their +priorities. For more information, see :ref:`topics-addons`. + .. setting:: AWS_ACCESS_KEY_ID AWS_ACCESS_KEY_ID @@ -964,7 +981,6 @@ some of them need to be enabled through a setting. For more information See the :ref:`extensions user guide ` and the :ref:`list of available extensions `. - .. setting:: FEED_TEMPDIR FEED_TEMPDIR diff --git a/scrapy/addons.py b/scrapy/addons.py new file mode 100644 index 000000000..02dd4fde8 --- /dev/null +++ b/scrapy/addons.py @@ -0,0 +1,54 @@ +import logging +from typing import TYPE_CHECKING, Any, List + +from scrapy.exceptions import NotConfigured +from scrapy.settings import Settings +from scrapy.utils.conf import build_component_list +from scrapy.utils.misc import create_instance, load_object + +if TYPE_CHECKING: + from scrapy.crawler import Crawler + +logger = logging.getLogger(__name__) + + +class AddonManager: + """This class facilitates loading and storing :ref:`topics-addons`.""" + + def __init__(self, crawler: "Crawler") -> None: + self.crawler: "Crawler" = crawler + self.addons: List[Any] = [] + + def load_settings(self, settings: Settings) -> None: + """Load add-ons and configurations from a settings object. + + This will load the add-on for every add-on path in the + ``ADDONS`` setting and execute their ``update_settings`` methods. + + :param settings: The :class:`~scrapy.settings.Settings` object from \ + which to read the add-on configuration + :type settings: :class:`~scrapy.settings.Settings` + """ + enabled: List[Any] = [] + for clspath in build_component_list(settings["ADDONS"]): + try: + addoncls = load_object(clspath) + addon = create_instance( + addoncls, settings=settings, crawler=self.crawler + ) + addon.update_settings(settings) + self.addons.append(addon) + except NotConfigured as e: + if e.args: + logger.warning( + "Disabled %(clspath)s: %(eargs)s", + {"clspath": clspath, "eargs": e.args[0]}, + extra={"crawler": self.crawler}, + ) + logger.info( + "Enabled addons:\n%(addons)s", + { + "addons": enabled, + }, + extra={"crawler": self.crawler}, + ) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 192541dd0..c5b3e1903 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -18,6 +18,7 @@ except ImportError: from zope.interface.verify import verifyClass from scrapy import Spider, signals +from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extension import ExtensionManager @@ -68,6 +69,9 @@ class Crawler: self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) + self.addons: AddonManager = AddonManager(self) + self.addons.load_settings(self.settings) + self.signals: SignalManager = SignalManager(self) self.stats: StatsCollector = load_object(self.settings["STATS_CLASS"])(self) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 03e92b565..04b838d2d 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -46,10 +46,9 @@ class MiddlewareManager: enabled.append(clspath) except NotConfigured as e: if e.args: - clsname = clspath.split(".")[-1] logger.warning( - "Disabled %(clsname)s: %(eargs)s", - {"clsname": clsname, "eargs": e.args[0]}, + "Disabled %(clspath)s: %(eargs)s", + {"clspath": clspath, "eargs": e.args[0]}, extra={"crawler": crawler}, ) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b0adb5ba8..bc82cc098 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -39,6 +39,7 @@ if TYPE_CHECKING: SETTINGS_PRIORITIES: Dict[str, int] = { "default": 0, "command": 10, + "addon": 15, "project": 20, "spider": 30, "cmdline": 40, diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 1bada1c70..feb6e8f6c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -17,6 +17,8 @@ import sys from importlib import import_module from pathlib import Path +ADDONS = {} + AJAXCRAWL_ENABLED = False ASYNCIO_EVENT_LOOP = None diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 70187ba74..b3c28da92 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -67,7 +67,7 @@ def load_object(path: Union[str, Callable]) -> Any: if callable(path): return path raise TypeError( - "Unexpected argument type, expected string " f"or object, got: {type(path)}" + f"Unexpected argument type, expected string or object, got: {type(path)}" ) try: diff --git a/sep/sep-021.rst b/sep/sep-021.rst deleted file mode 100644 index e8affa943..000000000 --- a/sep/sep-021.rst +++ /dev/null @@ -1,113 +0,0 @@ -======= =================== -SEP 21 -Title Add-ons -Author Pablo Hoffman -Created 2014-02-14 -Status Draft -======= =================== - -================ -SEP-021: Add-ons -================ - -This proposal introduces add-ons, a unified way to manage Scrapy extensions, -middlewares and pipelines. - -Scrapy currently supports many hooks and mechanisms for extending its -functionality, but no single entry point for enabling and configuring them. -Instead, the hooks are spread over: - -* Spider middlewares (SPIDER_MIDDLEWARES) -* Downloader middlewares (DOWNLOADER_MIDDLEWARES) -* Downloader handlers (DOWNLOADER_HANDLERS) -* Item pipelines (ITEM_PIPELINES) -* Feed exporters and storages (FEED_EXPORTERS, FEED_STORAGES) -* Overridable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc) -* Generic extensions (EXTENSIONS) -* CLI commands (COMMANDS_MODULE) - -One problem of this approach is that enabling an extension often requires -modifying many settings, often in a coordinated way, which is complex and error -prone. Add-ons are meant to fix this by providing a simple mechanism for -enabling extensions. - -Design goals and non-goals -========================== - -Goals: - -* simple to manage: adding or removing extensions should be just a matter of - adding or removing lines in a ``scrapy.cfg`` file -* backward compatibility with enabling extension the "old way" (i.e. modifying - settings directly) - -Non-goals: - -* a way to publish, distribute or discover extensions (use pypi for that) - - -Managing add-ons -================ - -Add-ons are defined in the ``scrapy.cfg`` file, inside the ``[addons]`` -section. - -To enable the "httpcache" addon, either shipped with Scrapy or in the Python -search path, create an entry for it in your ``scrapy.cfg``, like this:: - - [addons] - httpcache = - -You may also specify the full path to an add-on (which may be either a .py file -or a folder containing __init__.py):: - - [addons] - mongodb_pipeline = /path/to/mongodb_pipeline.py - - -Writing add-ons -=============== - -Add-ons are Python modules that implement the following callbacks. - -addon_configure ---------------- - -Receives the Settings object and modifies it to enable the required components. -If it raises an exception, Scrapy will print it and exit. - -Examples: - -.. code-block:: python - - def addon_configure(settings): - settings.overrides["DOWNLOADER_MIDDLEWARES"].update( - { - "scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware": 900, - } - ) - -.. code-block:: python - - def addon_configure(settings): - try: - import boto - except ImportError: - raise RuntimeError("boto library is required") - - -crawler_ready -------------- - -``crawler_ready`` receives a Crawler object after it has been initialized and -is meant to be used to perform post-initialization checks like making sure the -extension and its dependencies were configured properly. If it raises an -exception, Scrapy will print and exit. - -Examples: - -.. code-block:: python - - def crawler_ready(crawler): - if "some.other.addon" not in crawler.extensions.enabled: - raise RuntimeError("Some other addon is required to use this addon") diff --git a/tests/test_addons.py b/tests/test_addons.py new file mode 100644 index 000000000..5d053ed52 --- /dev/null +++ b/tests/test_addons.py @@ -0,0 +1,158 @@ +import itertools +import unittest +from typing import Any, Dict + +from scrapy import Spider +from scrapy.crawler import Crawler, CrawlerRunner +from scrapy.exceptions import NotConfigured +from scrapy.settings import BaseSettings, Settings +from scrapy.utils.test import get_crawler + + +class SimpleAddon: + def update_settings(self, settings): + pass + + +def get_addon_cls(config: Dict[str, Any]) -> type: + class AddonWithConfig: + def update_settings(self, settings: BaseSettings): + settings.update(config, priority="addon") + + return AddonWithConfig + + +class CreateInstanceAddon: + def __init__(self, crawler: Crawler) -> None: + super().__init__() + self.crawler = crawler + self.config = crawler.settings.getdict("MYADDON") + + @classmethod + def from_crawler(cls, crawler: Crawler): + return cls(crawler) + + def update_settings(self, settings): + settings.update(self.config, "addon") + + +class AddonTest(unittest.TestCase): + def test_update_settings(self): + settings = BaseSettings() + settings.set("KEY1", "default", priority="default") + settings.set("KEY2", "project", priority="project") + addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"} + testaddon = get_addon_cls(addon_config)() + testaddon.update_settings(settings) + self.assertEqual(settings["KEY1"], "addon") + self.assertEqual(settings["KEY2"], "project") + self.assertEqual(settings["KEY3"], "addon") + + +class AddonManagerTest(unittest.TestCase): + def test_load_settings(self): + settings_dict = { + "ADDONS": {"tests.test_addons.SimpleAddon": 0}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + self.assertIsInstance(manager.addons[0], SimpleAddon) + + def test_notconfigured(self): + class NotConfiguredAddon: + def update_settings(self, settings): + raise NotConfigured() + + settings_dict = { + "ADDONS": {NotConfiguredAddon: 0}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + self.assertFalse(manager.addons) + + def test_load_settings_order(self): + # Get three addons with different settings + addonlist = [] + for i in range(3): + addon = get_addon_cls({"KEY1": i}) + addon.number = i + addonlist.append(addon) + # Test for every possible ordering + for ordered_addons in itertools.permutations(addonlist): + expected_order = [a.number for a in ordered_addons] + settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} + crawler = get_crawler(settings_dict=settings) + manager = crawler.addons + self.assertEqual([a.number for a in manager.addons], expected_order) + self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1]) + + def test_create_instance(self): + settings_dict = { + "ADDONS": {"tests.test_addons.CreateInstanceAddon": 0}, + "MYADDON": {"MYADDON_KEY": "val"}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + self.assertIsInstance(manager.addons[0], CreateInstanceAddon) + self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val") + + def test_settings_priority(self): + config = { + "KEY": 15, # priority=addon + } + settings_dict = { + "ADDONS": {get_addon_cls(config): 1}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual(crawler.settings.getint("KEY"), 15) + + settings = Settings(settings_dict) + settings.set("KEY", 0, priority="default") + runner = CrawlerRunner(settings) + crawler = runner.create_crawler(Spider) + self.assertEqual(crawler.settings.getint("KEY"), 15) + + settings_dict = { + "KEY": 20, # priority=project + "ADDONS": {get_addon_cls(config): 1}, + } + settings = Settings(settings_dict) + settings.set("KEY", 0, priority="default") + runner = CrawlerRunner(settings) + crawler = runner.create_crawler(Spider) + self.assertEqual(crawler.settings.getint("KEY"), 20) + + def test_fallback_workflow(self): + FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" + + class AddonWithFallback: + def update_settings(self, settings): + if not settings.get(FALLBACK_SETTING): + settings.set( + FALLBACK_SETTING, + settings.getwithbase("DOWNLOAD_HANDLERS")["https"], + "addon", + ) + settings["DOWNLOAD_HANDLERS"]["https"] = "AddonHandler" + + settings_dict = { + "ADDONS": {AddonWithFallback: 1}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual( + crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" + ) + self.assertEqual( + crawler.settings.get(FALLBACK_SETTING), + "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + ) + + settings_dict = { + "ADDONS": {AddonWithFallback: 1}, + "DOWNLOAD_HANDLERS": {"https": "UserHandler"}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual( + crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" + ) + self.assertEqual(crawler.settings.get(FALLBACK_SETTING), "UserHandler") diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 00ff746ee..a42c7b3d1 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -39,7 +39,7 @@ class MOff: pass def __init__(self): - raise NotConfigured + raise NotConfigured("foo") class TestMiddlewareManager(MiddlewareManager): diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 2d9210410..eedb6f6af 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -296,3 +296,7 @@ class UpdateClassPathTest(unittest.TestCase): output = update_classpath("scrapy.unmatched.Path") self.assertEqual(output, "scrapy.unmatched.Path") self.assertEqual(len(w), 0) + + def test_returns_nonstring(self): + for notastring in [None, True, [1, 2, 3], object()]: + self.assertEqual(update_classpath(notastring), notastring)