From cfed9b6659c90e0799361911b1d72ed127edf471 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 15 Jul 2015 17:27:57 +0200 Subject: [PATCH 01/55] Allow passing Python objects to middleware dict settings --- scrapy/middleware.py | 5 ++++- scrapy/utils/misc.py | 7 ++++++- tests/test_middleware.py | 13 +++++++++++++ tests/test_utils_misc/__init__.py | 4 +++- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 2ef5f30e2..690ec6a55 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,4 +1,5 @@ from collections import defaultdict +from inspect import isclass import logging import pprint @@ -31,7 +32,9 @@ class MiddlewareManager(object): for clspath in mwlist: try: mwcls = load_object(clspath) - if crawler and hasattr(mwcls, 'from_crawler'): + if not isclass(mwcls): + mw = mwcls + elif crawler and hasattr(mwcls, 'from_crawler'): mw = mwcls.from_crawler(crawler) elif hasattr(mwcls, 'from_settings'): mw = mwcls.from_settings(settings) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 303a413d8..75f42cc17 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -31,10 +31,15 @@ def arg_to_iter(arg): def load_object(path): """Load an object given its absolute object path, and return it. - object can be a class, function, variable o instance. + If ``path`` is not a string, it will be returned. + + The object can be a class, function, variable, or instance. path ie: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware' """ + if not isinstance(path, six.string_types): + return path + try: dot = path.rindex('.') except ValueError: diff --git a/tests/test_middleware.py b/tests/test_middleware.py index b6d885330..4e3c67d20 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -91,3 +91,16 @@ class MiddlewareManagerTest(unittest.TestCase): mwman = TestMiddlewareManager.from_settings(settings) classes = [x.__class__ for x in mwman.middlewares] self.assertEqual(classes, [M1, M3]) + + def test_instances_from_settings(self): + settings = Settings() + myM3 = M3() + class InstanceTestMiddlewareManager(MiddlewareManager): + @classmethod + def _get_mwlist_from_settings(cls, settings): + return [ 'tests.test_middleware.M1', M2, myM3 ] + mwman = InstanceTestMiddlewareManager.from_settings(settings) + self.assertIsInstance(mwman.middlewares[0], M1) + self.assertIsInstance(mwman.middlewares[1], M2) + self.assertIs(mwman.middlewares[2], myM3) + diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 01460a10b..06af3c009 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -11,7 +11,9 @@ class UtilsMiscTestCase(unittest.TestCase): def test_load_object(self): obj = load_object('scrapy.utils.misc.load_object') - assert obj is load_object + self.assertIs(obj, load_object) + not_a_string = int(1000) + self.assertIs(load_object(not_a_string), not_a_string) self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') From e5b8def0b86102eacdb9a49942e88047a2250527 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 1 Jun 2015 17:39:41 +0200 Subject: [PATCH 02/55] Redraft SEP-021 --- sep/sep-021.rst | 338 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 289 insertions(+), 49 deletions(-) diff --git a/sep/sep-021.rst b/sep/sep-021.rst index 628a95dd2..ce500fc00 100644 --- a/sep/sep-021.rst +++ b/sep/sep-021.rst @@ -17,19 +17,31 @@ Scrapy currently supports many hooks and mechanisms for extending its functionality, but no single entry point for enabling and configuring them. Instead, the hooks are spread over: -* Spider middlewares (SPIDER_MIDDLEWARES) -* Downloader middlewares (DOWNLOADER_MIDDLEWARES) -* Downloader handlers (DOWNLOADER_HANDLERS) -* Item pipelines (ITEM_PIPELINES) -* Feed exporters and storages (FEED_EXPORTERS, FEED_STORAGES) -* Overrideable components (DUPEFILTER_CLASS, STATS_CLASS, SCHEDULER, SPIDER_MANAGER_CLASS, ITEM_PROCESSOR, etc) -* Generic extensions (EXTENSIONS) -* CLI commands (COMMANDS_MODULE) +* Spider middlewares (``SPIDER_MIDDLEWARES``) +* Downloader middlewares (``DOWNLOADER_MIDDLEWARES``) +* Downloader handlers (``DOWNLOADER_HANDLERS``) +* Item pipelines (``ITEM_PIPELINES``) +* Feed exporters and storages (``FEED_EXPORTERS``, ``FEED_STORAGES``) +* Overrideable components (``DUPEFILTER_CLASS``, ``STATS_CLASS``, + ``SCHEDULER``, ``SPIDER_MANAGER_CLASS``, ``ITEM_PROCESSOR``, etc.) +* Generic extensions (``EXTENSIONS``) +* CLI commands (``COMMANDS_MODULE``) + +This approach has several shortfalls: + +* Enabling an extension often requires modifying many settings, often in a + coordinated way, which is complex and error prone. +* Extension developers have little control over ensuring their library + dependencies and configuration requirements are met, especially since most + extensions never 'see' a fully-configured crawler before it starts running. +* The user is burdened with supervising potential interplay of extensions, + especially non-included ones, ranging from setting name clashes to mutually + excluding dependencies/configuration requirements. + +*Add-ons* search to remedy these shortcomings by enhancing Scrapy's extension +management, making it easy-to-use and transparent for users while giving more +configuration control to developers. -One problem of this approach is that enabling an extension often requires -modifying many settings, often in a coordinated way, which is complex and error -prone. Add-ons are meant to fix this by providing a simple mechanism for -enabling extensions. Design goals and non-goals ========================== @@ -37,8 +49,8 @@ Design goals and non-goals Goals: * simple to manage: adding or removing extensions should be just a matter of - adding or removing lines in a ``scrapy.cfg`` file -* backward compatibility with enabling extension the "old way" (ie. modifying + adding or removing lines in a configuration file +* backward compatibility with enabling extension the "old way" (i.e. modifying settings directly) Non-goals: @@ -46,62 +58,290 @@ Non-goals: * a way to publish, distribute or discover extensions (use pypi for that) -Managing add-ons -================ +User experience: managing add-ons +================================= -Add-ons are defined in the ``scrapy.cfg`` file, inside the ``[addons]`` -section. +Add-ons are enabled and configured either via Scrapy's settings, or (for add-ons +not bound to any project) in ``scrapy.cfg``. -To enable the "httpcache" addon, either shipped with Scrapy or in the Python -search path, create an entry for it in your ``scrapy.cfg``, like this:: +In the settings, add-ons can be enabled by adding either their name (for +built-in add-ons), their Python path, or their file path, to a +``INSTALLED_ADDONS`` setting. If necessary, each add-on can be configured by +providing a dictionary-valued setting with the uppercase add-on name. For +example, to enable and configure the built-in ``httpcache`` add-on and enable +(without configuring) two custom add-ons, one via Python path and one via file +path, add these entries to your settings module:: - [addons] - httpcache = + INSTALLED_ADDONS = ( + 'httpcache', + 'mymodule.filters.myfilter', + 'mymodule/filters/otherfilter.py', + ) -You may also specify the full path to an add-on (which may be either a .py file -or a folder containing __init__.py):: + HTTPCACHE = { + 'ignore_http_codes': [404, 503], + } - [addons] - mongodb_pipeline = /path/to/mongodb_pipeline.py +In ``scrapy.cfg``, add-ons are enabled and configured with one section per +add-on. The section names correspond to the entries of ``INSTALLED_ADDONS``. +The configuration from above could look like this:: + + [addon:httpcache] + ignore_http_codes = 404,503 + + [addon:mymodule.filters.myfilter] + + [addon:mymodule/filters/otherfilter.py] -Writing add-ons -=============== +Developer experience: writing add-ons +===================================== -Add-ons are Python modules that implement the following callbacks. +Add-ons are (any) Python *objects* that implement Scrapy's *add-on interface*. +The interface is enforced through ``zope.interface``. This leaves the choice of +Python object up the developer. Examples: -addon_configure ---------------- +* for a small pipeline, the add-on interface could be implemented in the same + class that also implements the ``open/close_spider`` and ``process_item`` + callbacks +* for larger add-ons, or for clearer structure, the interface could be provided + by a stand-alone module -Receives the Settings object and modifies it to enable the required components. -If it raises an exception, Scrapy will print it and exit. +The absolute minimum interface consists of just two attributes: -Examples:: +* ``NAME``: string with add-on name +* ``VERSION``: PEP-440 style version string - def addon_configure(settings): - settings.overrides['DOWNLADER_MIDDLEWARES'].update({ - 'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900, - }) +To be any useful, an add-on should implement at least one of the following +callback methods: + +* ``update_addons()``: adds and configures other add-ons +* ``update_settings()``: sets configuration (such as default values for this + add-on and required settings for other extensions) and enables needed + components. +* ``check_configuration()``: receives the fully-initialized ``Crawler`` + instance before it starts running, performs additional dependency and + configuration requirement checks + +Additionally, an add-on may (and should, where appropriate) provide one or more +variables that can be used for automated detection of possible dependency +clashes: + +* ``REQUIRES``: list of built-in or custom components required by this add-on, + as PEP-440 strings +* ``MODIFIES``: list of components whose functionality is affected or replaced + by this add-on (a custom HTTP cache should list ``httpcache`` here) +* ``PROVIDES``: list of components provided by this add-on (e.g. ``mongodb`` + for an extension that provides generic read/write access to a MongoDB + database, releasing other components from having to provide their own + database access methods) + +update_addons() +----------------- + +Called: +~~~~~~~ + +Shortly after initialisation of the ``Crawler`` object. + +Arguments: +~~~~~~~~~~ + +* ``config``: configuration of this add-on +* ``addons``: the add-on manager, providing methods to add and configure add-ons + +Purpose: +~~~~~~~~ + +* Configure and enable related add-ons, useful for 'umbrella add-ons' which + chain-load other add-ons based on the configuration + +Examples: +~~~~~~~~~ :: - def addon_configure(settings): + def update_addons(config, addons): + if 'httpcache' not in addons.enabled: + addons.add('httpcache', {'expiration_secs': 60}) + +or:: + + def update_addons(config, addons): + if 'otheraddon' in addons.enabled: + addons.configs['otheraddon']['some_config_name'] = True + +update_settings() +----------------- + +Called: +~~~~~~~ + +Directly after the ``update_addons()`` callback of all add-ons has been called. + +Arguments: +~~~~~~~~~~ + +* ``config``: configuration of this add-on +* ``settings``: the crawler's ``Settings`` instance containing all project + settings + +Purpose: +~~~~~~~~ + +* Modify ``settings`` to enable required components +* Expose some add-on specific configuration (``config``) into the global + settings namespace (``settings``) if necessary +* Raise exception if components can not be properly configured (e.g. on missing + dependencies); Scrapy will print this exception *and exit* (making users + explicitly acknowledge that the add-on does not work by forcing them to + disable it). + +Side note: +~~~~~~~~~~ + +The ``MiddlewareManager.from_settings()`` method will receive a slight +modification to allow directly placing Python objects instead of class paths +in the middleware dict settings. This way, add-ons can place already +instantiated components into the settings. This allows keeping configuration +as local to components as possible and avoids cluttering up the global +settings namespace. Furthermore, it allows reusing components (e.g. using +two instances of the same mongodb pipeline to write to different locations). + +Examples: +~~~~~~~~~ + +:: + + def update_settings(config, settings): + # Don't care where this module is located + settings.set['DOWNLADER_MIDDLEWARES']({ + __name__ + '.downloadermw.coolmw': 900, + }) + + # Instantiate components to not expose settings into + # the global namespace + from .pipelines import MySQLPipeline + mysqlpl = MySQLPipeline(password = config['password']) + settings.set['ITEM_PIPELINES']({ + mysqlpl: 200, + }) + +or:: + + def update_settings(config, settings): + # Assuming this class also has a process_item() method + settings.set['ITEM_PIPELINES']({ + self: 200, + }) + +or:: + + def update_settings(config, settings): try: import boto except ImportError: raise RuntimeError("boto library is required") +check_configuration() +--------------------- -crawler_ready -------------- +Called: +~~~~~~~ -``crawler_ready`` receives a Crawler object after it has been initialized and -is meant to be used to perform post-initialization checks like making sure the -extension and its dependencies were configured properly. If it raises an -exception, Scrapy will print and exit. +Shortly before the crawler starts crawling. -Examples:: +Arguments: +~~~~~~~~~~ + +* ``config``: configuration of this add-on +* ``crawler``: fully-initialized ``Crawler`` object, ready to start crawling + +Purpose: +~~~~~~~~ + +* Perform post-initialization checks like making sure the extension and its + dependencies were configured properly. +* Raise exception if a critical check failed; Scrapy will print this exception + *and exit* (see ``update_settings()`` purpose for rationale on this). + +Examples: +~~~~~~~~~ + +:: + + def check_configuration(config, crawler): + if 'some.other.addon' not in crawler.addons.enabled: + raise RuntimeError("Some other add-on required to use this add-on") + + +Implementation +============== + +A new core component, the *add-on manager*, is introduced to Scrapy. It +facilitates loading add-ons, gathering and providing information on them, +calling their callbacks at appropriate times, and performing basic checks for +dependency and configuration clashes. + +Layout +------ + +A new ``AddonManager`` class is introduced, providing methods to + +* add and remove add-ons, +* search for add-ons by name +* read enabled add-ons and their configurations from the settings module and + from ``settings.py``, +* enable and disable add-ons +* check for possible dependency incompatibilites by inspecting the collected + ``REQUIRES``, ``MODIFIES`` and ``PROVIDES`` add-on variables +* call the add-on callbacks + +Integration into start-up process +--------------------------------- + +The settings used to crawl are not complete until the spider-specific settings +have been loaded in ``Crawler.__init__()``. Add-on management follows this +approach and only starts loading add-ons when the crawler is initialised. + +Instantiation and the calls ``update_addons()`` and ``update_settings()`` happen +in ``Crawler.__init__()``. The final checks (i.e. the callback to +``check_configuration()``) is coded into the ``Crawler.crawl()`` method after +creating the engine. + +Finding add-ons +--------------- + +Add-on localisation is governed by the add-on paths given in +``INSTALLED_ADDONS`` (or by the section names if using ``scrapy.cfg``). If +nothing is found at the given path, it is tried again with ``addons.`` +prepended (i.e. pointing to the project's ``addons`` folder or module), then +with ``scrapy.addons.`` prepended (i.e. pointing to Scrapy's ``addons`` +submodule). If the object found has an ``_addon`` attribute, that attribute +will be treated as the found add-on. This allows, for example, to change the +add-on based on the Python version. + +Updating existing extensions +---------------------------- + +An ``Addon`` class is introduced that add-on developers may or may not subclass +depending on how much of the 'default functionality' they want. Naturally, it +does not provide ``NAME`` and ``VERSION``. Its default ``update_settings()`` +exposes the add-on configuration into the global settings namespace with an +appropriate name, e.g. this section from ``scrapy.cfg``:: + + [httpcache] + dir = /some/dir + +would expose ``HTTPCACHE_DIR``. + +Add-on modules will be written for all built-in extensions and placed in +``scrapy.addons``. For many default Scrapy components, it will be sufficient to +create a subclass of ``Addon`` with minor or no method modifications. The +component code remains where it is (i.e. in ``scrapy.pipelines``, etc.). + +Later, the global settings namespace could be cleaned up in a backwards +-incompatible fashion by deprecating support for the global setting names, e.g. +``HTTPCACHE_DIR``, and instead instantiate the components with the add-on +configuration in ``update_settings()``. - def crawler_ready(crawler): - if 'some.other.addon' not in crawler.extensions.enabled: - raise RuntimeError("Some other addon is required to use this addon") From d8af395d7654ba659aa29060cf3cf0ed3dfe1174 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 19 Aug 2015 16:16:53 +0200 Subject: [PATCH 03/55] Introduce add-ons via AddonManager and Addon base class --- docs/topics/api.rst | 11 + docs/topics/settings.rst | 9 + scrapy/addons/__init__.py | 497 ++++++++++++++++++ scrapy/interfaces.py | 23 +- scrapy/settings/__init__.py | 1 + scrapy/settings/default_settings.py | 2 + scrapy/utils/conf.py | 12 +- scrapy/utils/misc.py | 42 +- scrapy/utils/project.py | 12 + tests/test_addons/__init__.py | 388 ++++++++++++++ tests/test_addons/addonmod.py | 16 + tests/test_addons/addons.py | 40 ++ tests/test_addons/cfg.cfg | 5 + tests/test_addons/project/__init__.py | 0 tests/test_addons/project/addons/__init__.py | 0 tests/test_addons/project/addons/addonmod.py | 7 + tests/test_addons/project/addons/addonmod2.py | 7 + tests/test_addons/scrapy_addons/__init__.py | 0 tests/test_addons/scrapy_addons/addonmod.py | 7 + tests/test_addons/scrapy_addons/addonmod2.py | 7 + tests/test_addons/scrapy_addons/addonmod3.py | 7 + tests/test_utils_misc/__init__.py | 26 +- tests/test_utils_misc/testmod.py | 1 + tests/test_utils_misc/testpkg/__init__.py | 1 + tests/test_utils_misc/testpkg/submod.py | 1 + tests/test_utils_project.py | 27 + 26 files changed, 1142 insertions(+), 7 deletions(-) create mode 100644 scrapy/addons/__init__.py create mode 100644 tests/test_addons/__init__.py create mode 100644 tests/test_addons/addonmod.py create mode 100644 tests/test_addons/addons.py create mode 100644 tests/test_addons/cfg.cfg create mode 100644 tests/test_addons/project/__init__.py create mode 100644 tests/test_addons/project/addons/__init__.py create mode 100644 tests/test_addons/project/addons/addonmod.py create mode 100644 tests/test_addons/project/addons/addonmod2.py create mode 100644 tests/test_addons/scrapy_addons/__init__.py create mode 100644 tests/test_addons/scrapy_addons/addonmod.py create mode 100644 tests/test_addons/scrapy_addons/addonmod2.py create mode 100644 tests/test_addons/scrapy_addons/addonmod3.py create mode 100644 tests/test_utils_misc/testmod.py create mode 100644 tests/test_utils_misc/testpkg/__init__.py create mode 100644 tests/test_utils_misc/testpkg/submod.py create mode 100644 tests/test_utils_project.py diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 42c0133c1..0c22b3ce9 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -149,6 +149,17 @@ Settings API .. autoclass:: BaseSettings :members: +.. _topics-api-addonmanager: + +AddonManager API +================ + +.. module:: scrapy.addons + :synopsis: Add-on manager + +.. autoclass:: AddonManager + :members: + .. _topics-api-spiderloader: SpiderLoader API diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index aa0417e1a..afcb8dd21 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -585,6 +585,15 @@ some of them need to be enabled through a setting. For more information See the :ref:`extensions user guide ` and the :ref:`list of available extensions `. +.. setting:: INSTALLED_ADDONS + +INSTALLED_ADDONS +---------------- + +Default: ``()`` + +A tuple containing paths to the add-ons enabled in your project. For more +information, see :ref:`topics-addons`. .. setting:: ITEM_PIPELINES diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py new file mode 100644 index 000000000..59e59e15f --- /dev/null +++ b/scrapy/addons/__init__.py @@ -0,0 +1,497 @@ +from collections import defaultdict, Mapping +from importlib import import_module +from inspect import isclass +import os +import six +import warnings + +from pkg_resources import WorkingSet, Distribution, Requirement +import zope.interface +from zope.interface.verify import verifyObject + +from scrapy.exceptions import NotConfigured +from scrapy.interfaces import IAddon +from scrapy.settings import BaseSettings +from scrapy.utils.conf import config_from_filepath, get_config +from scrapy.utils.misc import load_module_or_object +from scrapy.utils.project import get_project_path + + +@zope.interface.implementer(IAddon) +class Addon(object): + + basic_settings = None + """``dict`` of settings that will be exported via :meth:`export_basics`.""" + + default_config = None + """``dict`` with default configuration.""" + + config_mapping = None + """``dict`` with mappings from config names to setting names. The given + setting names will be taken as given, i.e. they will be neither prefixed + nor uppercased. + """ + + component_type = None + """Component setting into which to export via :meth:`export_component`. Can + be any of the dictionary-like component setting names (e.g. + ``DOWNLOADER_MIDDLEWARES``) or any of their abbreviations in + :attr:`~scrapy.addons.COMPONENT_TYPE_ABBR`. If ``None``, + :meth:`export_component` will do nothing. + """ + + component_key = None + """Key to be used in the component dictionary setting when exporting via + :meth:`export_component`. This is only useful for the settings that have + no order, e.g. ``DOWNLOAD_HANDLERS`` or ``FEED_EXPORTERS``. + """ + + component_order = 0 + """Component order to use when not given in the add-on configuration. Has + no effect for component types that use :attr:`component_key`. + """ + + component = None + """Component to be inserted via :meth:`export_component`. This can be + anything that can be used in the dictionary-like component settings, i.e. + a class path, a class, or an instance. If ``None``, it is assumed that the + add-on itself is also provides the component interface, and ``self`` will be + used. + """ + + settings_prefix = None + """Prefix with which the add-on configuration will be exported into the + global settings namespace via :meth:`export_config`. If ``None``, + :attr:`name` will be used. If ``False``, no configuration will be exported. + """ + + def export_component(self, config, settings): + """Export the component in :attr:`component` into the dictionary-like + component setting derived from :attr:`component_type`. + + Where applicable, the order parameter of the component (i.e. the + dictionary value) will be retrieved from the ``order`` add-on + configuration value. + + :param config: Add-on configuration from which to read component order + :type config: ``dict`` + + :param settings: Settings object into which to export component + :type settings: :class:`~scrapy.settings.Settings` + """ + if self.component_type: + comp = self.component or self + if self.component_key: + # e.g. for DOWNLOAD_HANDLERS: {'http': 'myclass'} + k = self.component_key + v = comp + else: + # e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100} + k = comp + v = config.get('order', self.component_order) + settings.set(self.component_type, {k: v}, 'addon') + + def export_basics(self, settings): + """Export the :attr:`basic_settings` attribute into the settings object. + + All settings will be exported with ``addon`` priority (see + :ref:`topics-api-settings`). + + :param settings: Settings object into which to expose the basic settings + :type settings: :class:`~scrapy.settings.Settings` + """ + for setting, value in six.iteritems(self.basic_settings or {}): + settings.set(setting, value, 'addon') + + def export_config(self, config, settings): + """Export the add-on configuration, all keys in caps and with + :attr:`settings_prefix` or :attr:`name` prepended, into the settings + object. + + For example, the add-on configuration ``{'key': 'value'}`` will export + the setting ``ADDONNAME_KEY`` with a value of ``value``. All settings + will be exported with ``addon`` priority (see + :ref:`topics-api-settings`). + + :param config: Add-on configuration to be exposed + :type config: ``dict`` + + :param settings: Settings object into which to export the configuration + :type settings: :class:`~scrapy.settings.Settings` + """ + if self.settings_prefix is False: + return + conf = self.default_config or {} + conf.update(config) + prefix = self.settings_prefix or self.name + # Since default exported config is case-insensitive (everything will be + # uppercased), make mapped config case-insensitive as well + conf_mapping = {k.lower(): v + for k, v in six.iteritems(self.config_mapping or {})} + for key, val in six.iteritems(conf): + if key.lower() in conf_mapping: + key = conf_mapping[key.lower()] + else: + key = (prefix + '_' + key).upper() + settings.set(key, val, 'addon') + + def update_settings(self, config, settings): + """Export both the basic settings and the add-on configuration. I.e., + call :meth:`export_basics` and :meth:`export_config`. + + For more advanced add-ons, you may want to override this callback. + + :param config: Add-on configuration + :type config: ``dict`` + + :param settings: Crawler settings object + :type settings: :class:`~scrapy.settings.Settings` + """ + self.export_component(config, settings) + self.export_basics(settings) + self.export_config(config, settings) + + +class AddonManager(Mapping): + """This class facilitates loading and storing :ref:`topics-addons`. + + You can treat it like a read-only dictionary in which keys correspond to + add-on names and values correspond to the add-on objects:: + + addons = AddonManager() + # ... load some add-ons here + print addons.enabled # prints names of all enabled add-ons + print addons['TestAddon'].version # prints version of add-on with name + # 'TestAddon' + + """ + + def __init__(self): + self._addons = {} + self.configs = {} + self._disable_on_add = [] + + def __getitem__(self, name): + return self._addons[name] + + def __delitem__(self, name): + del self._addons[name] + del self.configs[name] + + def __iter__(self): + return iter(self._addons) + + def __len__(self): + return len(self._addons) + + def add(self, addon, config=None): + """Store an add-on. + + If ``addon`` is a string, it will be treated as add-on path and passed + to :meth:`get_addon`. Otherwise, ``addon`` must be a Python object + implementing or providing Scrapy's add-on interface. The interface + will be enforced through ``zope.interface``'s ``verifyObject()``. + + If ``addon`` is a class, it will be instantiated. You can avoid this + (for example if you have implemented the add-on callbacks as class + methods) by declaring -- via ``zope.interface`` -- that your class + directly *provides* ``scrapy.interfaces.IAddon``. + + :param addon: The add-on object (or path) to be stored + :type addon: Any Python object providing the add-on interface or ``str`` + + :param config: The add-on configuration dictionary + :type config: ``dict`` + """ + addon = self.get_addon(addon) + if isclass(addon) and not IAddon.providedBy(addon): + addon = addon() + if not IAddon.providedBy(addon): + zope.interface.alsoProvides(addon, IAddon) + # zope.interface's exceptions are already quite helpful. Still, should + # we catch them and log an error message? + verifyObject(IAddon, addon) + name = addon.name + if name in self: + raise ValueError("Addon '{}' already loaded".format(name)) + self._addons[name] = addon + self.configs[name] = config or {} + if name in self._disable_on_add: + self.configs[name]['_enabled'] = False + self._disable_on_add.remove(name) + + def remove(self, addon): + """Remove an add-on. + + If ``addon`` is the name of a stored add-on, that add-on will be + removed. Otherwise, you can use the argument in the same fashion as + in :meth:`add`. + + :param addon: The add-on name, object, or path to be removed + :type addon: Any Python object providing the add-on interface or ``str`` + """ + if addon in self: + del self[addon] + elif hasattr(addon, 'name') and addon.name in self: + del self[addon.name] + else: + try: + del self[self.get_addon(addon).name] + except NameError: + raise KeyError + + @staticmethod + def get_addon(path): + """Get an add-on object by its Python or file path. + + ``path`` is assumed to be either a Python or a file path of a Scrapy + add-on. If no object is found at ``path``, it is tried again first with + ``projectname.addons`` prepended (pointing to the current project's + ``addons`` folder), then with ``scrapy.addons`` prepended (poiting to + Scrapy's built-in add-ons). These convenience shortcuts will only work + with Python paths, not file paths. + + If the object or module pointed to by ``path`` has an attribute named + ``_addon`` that attribute will be assumed to be the add-on. + :meth:`get_addon` will keep following ``_addon`` attributes until it + finds an object that does not have an attribute named ``_addon``. + + :param path: Python or file path to an add-on + :type path: ``str`` + """ + if isinstance(path, six.string_types): + prefixes = ['', 'scrapy.addons.'] + try: + prefixes.insert(1, get_project_path() + '.addons.') + except NotConfigured: + warnings.warn("Unable to locate project Python path") + for prefix in prefixes: + fullpath = prefix + path + try: + obj = load_module_or_object(fullpath) + except NameError: + pass + else: + break + else: + raise NameError("Could not find add-on '%s'" % path) + else: + obj = path + if hasattr(obj, '_addon'): + obj = AddonManager.get_addon(obj._addon) + return obj + + def load_dict(self, addonsdict): + """Load add-ons and configurations from given dictionary. + + Each add-on should be an entry in the dictionary, where the key + corresponds to the add-on path. The value should be a dictionary + representing the add-on configuration. + + Example add-on dictionary:: + + addonsdict = { + 'path.to.addon1': { + 'setting1': 'value', + 'setting2': 42, + }, + 'path/to/addon2.py': { + 'addon2setting': True, + }, + } + + :param addonsdict: dictionary where keys correspond to add-on paths \ + and values correspond to their configuration + :type addonsdict: ``dict`` + """ + for addonpath, addoncfg in six.iteritems(addonsdict): + self.add(addonpath, addoncfg) + + def load_settings(self, settings): + """Load add-ons and configurations from settings object. + + This will invoke :meth:`get_addon` for every add-on path in the + ``INSTALLED_ADDONS`` setting. For each of these add-ons, the + configuration will be read from the dictionary setting whose name + matches the uppercase add-on name. + + :param settings: The :class:`~scrapy.settings.Settings` object from \ + which to read the add-on configuration + :type settings: :class:`~scrapy.settings.Settings` + """ + paths = settings.getlist('INSTALLED_ADDONS') + addons = [self.get_addon(path) for path in paths] + configs = [settings.getdict(addon.name.upper()) for addon in addons] + for a, c in zip(addons, configs): + self.add(a, c) + + def load_cfg(self, cfg=None): + """Load add-ons and configurations from given ``ConfigParser`` object or + config file path. + + Each add-on should have its own section, where the section has a name in + the form ``addon:my_addon_path``. The add-on object is searched for via + the :meth:`get_addon` method, ``my_addon_path`` can be either a Python + or a file path. + + If ``cfg`` is ``None``, ``scrapy.cfg`` will be used. + + :param cfg: ``ConfigParser`` object or config file path from which to \ + read add-on configuration + :type cfg: ``ConfigParser`` or ``str`` + """ + if cfg is None: + cfg = get_config() + elif isinstance(cfg, six.string_types): + cfg = config_from_filepath(cfg) + for secname in cfg.sections(): + if secname.startswith("addon:"): + addonkey = secname.split("addon:", 1)[1] + addoncfg = dict(cfg.items(secname)) + self.add(addonkey, addoncfg) + + def check_dependency_clashes(self): + """Check for incompatibilities in add-on dependencies. + + Add-ons can provide information about their dependencies in their + ``provides``, ``modifies`` and ``requires`` attributes. This method will + raise an ``ImportError`` if + + * a component required by an add-on is not provided by any other add-on, + or + * a component modified by an add-on is not provided by any other add-on, + or + * the same component is provided by more than one add-on, + + and warn when a component required by an add-on is modified by any other + add-on. + """ + # Collect all active add-ons and the components they provide + ws = WorkingSet('') + def add_dist(project_name, version, **kwargs): + if project_name in ws.entry_keys.get('scrapy', []): + raise ImportError("Component {} provided by multiple add-ons" + "".format(project_name)) + else: + dist = Distribution(project_name=project_name, version=version, + **kwargs) + ws.add(dist, entry='scrapy') + for name in self: + ver = self[name].version + add_dist(name, ver) + for provides_name in getattr(self[name], 'provides', []): + add_dist(provides_name, ver) + + # Collect all required and modified components + def compile_attribute_dict(attribute_name): + attrs = defaultdict(list) + for name in self: + for entry in getattr(self[name], attribute_name, []): + attrs[entry].append(name) + return attrs + modified = compile_attribute_dict('modifies') + required = compile_attribute_dict('requires') + + req_or_mod = set(required.keys()).union(modified.keys()) + for reqstr in req_or_mod: + req = Requirement.parse(reqstr) + # May raise VersionConflict. Do we want to catch it and raise + # our own exception or is it helpful enough? + if ws.find(req) is None: + raise ImportError( + "Add-ons {} require or modify missing component {}" + "".format(required[reqstr]+modified[reqstr], reqstr)) + + mod_and_req = set(required.keys()).intersection(modified.keys()) + for conflict in mod_and_req: + warnings.warn("Component '{}', required by add-ons {}, is modified " + "by add-ons {}".format(conflict, required[conflict], + modified[conflict])) + + def disable(self, addon): + """Disable an add-on, i.e. prevent its callbacks from being called. + + If you disable an add-on before it is loaded, it will be disabled as + soon as it is added to the :class:`AddonManager`. + + :param addon: Name of the add-on to be disabled + :type addon: ``str`` + """ + if addon in self: + self.configs[addon]['_enabled'] = False + else: + self._disable_on_add.append(addon) + + def enable(self, addon): + """Re-enable a disabled add-on. + + Will raise ``ValueError`` if the add-on is neither already loaded nor + marked for being disabled on adding. + + :param addon: Name of the add-on to be enabled + :type addon: ``str`` + """ + if addon in self: + self.configs[addon]['_enabled'] = True + elif addon in self._disable_on_add: + self._disable_on_add.remove(addon) + else: + raise ValueError("Add-ons need to be added before they can be " + "enabled") + + @property + def disabled(self): + """Names of disabled add-ons""" + return ([a for a in self if not self.configs[a].get('_enabled', True)] + + self._disable_on_add) + + @property + def enabled(self): + """Names of enabled add-ons""" + return [a for a in self if self.configs[a].get('_enabled', True)] + + def _call_if_exists(self, obj, cbname, *args, **kwargs): + if obj is None: + return + try: + cb = getattr(obj, cbname) + except AttributeError: + return + else: + cb(*args, **kwargs) + + def _call_addon(self, addonname, cbname, *args, **kwargs): + if self.configs[addonname].get('_enabled', True): + self._call_if_exists(self[addonname], cbname, + self.configs[addonname], *args, **kwargs) + + def update_addons(self): + """Call ``update_addons()`` of all held add-ons. + + This will also call ``update_addons()`` of all add-ons that are added + last minute during the ``update_addons()`` routine of other add-ons. + """ + called_addons = set() + while called_addons != set(self): + for name in set(self).difference(called_addons): + called_addons.add(name) + self._call_addon(name, 'update_addons', self) + + def update_settings(self, settings): + """Call ``update_settings()`` of all held add-ons. + + :param settings: The :class:`~scrapy.settings.Settings` object to be \ + updated + :type settings: :class:`~scrapy.settings.Settings` + """ + for name in self: + self._call_addon(name, 'update_settings', settings) + + def check_configuration(self, crawler): + """Call ``check_configuration()`` of all held add-ons. + + :param crawler: the fully-initialized crawler + :type crawler: :class:`~scrapy.crawler.Crawler` + """ + for name in self: + self._call_addon(name, 'check_configuration', crawler) diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index eb93c6f7e..75b72899e 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,6 +1,6 @@ -from zope.interface import Interface +import zope.interface -class ISpiderLoader(Interface): +class ISpiderLoader(zope.interface.Interface): def from_settings(settings): """Return an instance of the class for the given settings""" @@ -20,3 +20,22 @@ class ISpiderLoader(Interface): # ISpiderManager is deprecated, don't use it! # An alias is kept for backwards compatibility. ISpiderManager = ISpiderLoader + + +class IAddon(zope.interface.Interface): + """Scrapy add-on""" + + name = zope.interface.Attribute("""Add-on name""") + version = zope.interface.Attribute("""Add-on version string (PEP440)""") + + # XXX: Can methods be declared optional? I.e., can I enforce the signature + # but not the existence of a method? + + #def update_addons(config, addons): + # """Enables and configures other add-ons""" + + #def update_settings(config, settings): + # """Modifies `settings` to enable and configure required components""" + + #def check_configuration(config, crawler): + # """Performs post-initialization checks on fully configured `crawler`""" diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 342d2585e..be9f740eb 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -14,6 +14,7 @@ from . import default_settings SETTINGS_PRIORITIES = { 'default': 0, 'command': 10, + 'addon': 15, 'project': 20, 'spider': 30, 'cmdline': 40, diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 8435b0354..a230750fb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -167,6 +167,8 @@ HTTPCACHE_DBM_MODULE = 'anydbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False +INSTALLED_ADDONS = () + ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index e8af90f11..5a5418104 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -82,14 +82,20 @@ def init_env(project='default', set_syspath=True): sys.path.append(projdir) -def get_config(use_closest=True): - """Get Scrapy config file as a SafeConfigParser""" - sources = get_sources(use_closest) +def config_from_filepath(sources): + """Create a SafeConfigParser and read in the given `sources`, which can be + either a filename or a list of filenames.""" cfg = SafeConfigParser() cfg.read(sources) return cfg +def get_config(use_closest=True): + """Get Scrapy config file as a SafeConfigParser""" + sources = get_sources(use_closest) + return config_from_filepath(sources) + + def get_sources(use_closest=True): xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \ os.path.expanduser('~/.config') diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 75f42cc17..9461d93e9 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,5 +1,8 @@ """Helper functions which doesn't fit anywhere else""" +import itertools +import os.path import re +import sys import hashlib from importlib import import_module from pkgutil import iter_modules @@ -56,6 +59,26 @@ def load_object(path): return obj +def load_module_or_object(path): + """Load python module or (non-module) object from given path. + + Path can be both a Python or a file path. + """ + try: + return import_module(path) + except ImportError: + pass + try: + return load_object(path) + except (ValueError, NameError, ImportError): + pass + try: + return get_module_from_filepath(path) + except ImportError: + pass + raise NameError("Could not load '%s'" % path) + + def walk_modules(path): """Loads a module and all its submodules from a the given module path and returns them. If *any* module throws an exception while importing, that @@ -78,6 +101,23 @@ def walk_modules(path): return mods +def get_module_from_filepath(path): + """Load and return a python module/package from a file path""" + path = path.rstrip("/") + if path.endswith('.py'): + path = path.rsplit('.py', 1)[0] + basefolder, modname = os.path.split(path) + # XXX: There are other ways to import modules from a full path which don't + # need to modify PYTHONPATH, see + # https://stackoverflow.com/questions/67631/ + # These methods differ between py2 and py3, and apparently the + # py3 method was deprecated in Python 3.4 + sys.path.insert(0, basefolder) + mod = import_module(modname) + sys.path.pop(0) + return mod + + def extract_regex(regex, text, encoding='utf-8'): """Extract a list of unicode strings from the given text/encoding using the following policies: @@ -118,7 +158,7 @@ def md5sum(file): m.update(d) return m.hexdigest() + def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" return True if rel is not None and 'nofollow' in rel.split() else False - diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index a15a0d90f..a1266c879 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -71,3 +71,15 @@ def get_project_settings(): settings.setdict(env_overrides, priority='project') return settings + +def get_project_path(): + """Return the Python path of the current project. + + This fails when the settings module does not live in the project's root. + """ + if not inside_project(): + raise NotConfigured("Not inside a project") + settings_module_path = os.environ.get(ENVVAR) + if not settings_module_path: + raise NotConfigured("Unable to locate project's python path") + return settings_module_path.rsplit('.', 1)[0] diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py new file mode 100644 index 000000000..c98f0ab65 --- /dev/null +++ b/tests/test_addons/__init__.py @@ -0,0 +1,388 @@ +import os.path +import six +from six.moves.configparser import SafeConfigParser +import sys +from tests import mock +import unittest +import warnings + +from pkg_resources import VersionConflict +import zope.interface +from zope.interface.verify import verifyObject +from zope.interface.exceptions import BrokenImplementation + +import scrapy.addons +from scrapy.addons import Addon, AddonManager +from scrapy.crawler import Crawler +from scrapy.interfaces import IAddon +from scrapy.settings import BaseSettings, Settings + +from . import addons +from . import addonmod + + +class AddonTest(unittest.TestCase): + + def setUp(self): + self.rawaddon = Addon() + class AddonWithAttributes(Addon): + name = 'Test' + version = '1.0' + self.testaddon = AddonWithAttributes() + + def test_interface(self): + # Raw Addon should fail exactly b/c name and version are not given + self.assertFalse(hasattr(self.rawaddon, 'name')) + self.assertFalse(hasattr(self.rawaddon, 'version')) + self.assertRaises(BrokenImplementation, verifyObject, IAddon, + self.rawaddon) + verifyObject(IAddon, self.testaddon) + + def test_export_component(self): + settings = BaseSettings({'ITEM_PIPELINES': {}}, 'default') + self.testaddon.component_type = None + self.testaddon.export_component({}, settings) + self.assertEqual(len(settings['ITEM_PIPELINES']), 0) + self.testaddon.component_type = 'ITEM_PIPELINES' + self.testaddon.component = 'test.component' + self.testaddon.export_component({}, settings) + six.assertCountEqual(self, settings['ITEM_PIPELINES'], + ['test.component']) + self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 0) + self.testaddon.component_order = 313 + self.testaddon.export_component({}, settings) + self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 313) + self.testaddon.component_type = 'DOWNLOAD_HANDLERS' + self.testaddon.component_key = 'http' + self.testaddon.export_component({}, settings) + self.assertEqual(settings['DOWNLOAD_HANDLERS']['http'], + 'test.component') + + def test_export_basics(self): + settings = BaseSettings() + self.testaddon.basic_settings = {'TESTKEY': 313, 'OTHERKEY': True} + self.testaddon.export_basics(settings) + self.assertEqual(settings['TESTKEY'], 313) + self.assertEqual(settings['OTHERKEY'], True) + self.assertEqual(settings.getpriority('TESTKEY'), 15) + + def test_export_config(self): + settings = BaseSettings() + self.testaddon.settings_prefix = None + self.testaddon.config_mapping = {'MAPPED_key': 'MAPPING_WORKED'} + self.testaddon.default_config = {'key': 55, 'defaultkey': 100} + self.testaddon.export_config({'key': 313, 'OTHERKEY': True, + 'mapped_KEY': 99}, settings) + self.assertEqual(settings['TEST_KEY'], 313) + self.assertEqual(settings['TEST_DEFAULTKEY'], 100) + self.assertEqual(settings['TEST_OTHERKEY'], True) + self.assertNotIn('MAPPED_key', settings) + self.assertNotIn('MAPPED_KEY', settings) + self.assertEqual(settings['MAPPING_WORKED'], 99) + self.assertEqual(settings.getpriority('TEST_KEY'), 15) + + self.testaddon.settings_prefix = 'PREF' + self.testaddon.export_config({'newkey': 99}, settings) + self.assertEqual(settings['PREF_NEWKEY'], 99) + + with mock.patch.object(settings, 'set') as mock_set: + self.testaddon.settings_prefix = False + self.testaddon.export_config({'thirdnewkey': 99}, settings) + self.assertEqual(mock_set.call_count, 0) + + def test_update_settings(self): + settings = BaseSettings() + settings.set('TEST_KEY1', 'default', priority='default') + settings.set('TEST_KEY2', 'project', priority='project') + self.testaddon.settings_prefix = None + self.testaddon.basic_settings = {'OTHERTEST_KEY': 'addon'} + addon_config = {'key1': 'addon', 'key2': 'addon', 'key3': 'addon'} + self.testaddon.update_settings(addon_config, settings) + self.assertEqual(settings['OTHERTEST_KEY'], 'addon') + self.assertEqual(settings['TEST_KEY1'], 'addon') + self.assertEqual(settings['TEST_KEY2'], 'project') + self.assertEqual(settings['TEST_KEY3'], 'addon') + + +class AddonManagerTest(unittest.TestCase): + + TESTCFGPATH = os.path.join(os.path.dirname(__file__), 'cfg.cfg') + ADDONMODPATH = os.path.join(os.path.dirname(__file__), 'addonmod.py') + + def setUp(self): + self.manager = AddonManager() + + def test_add(self): + manager = AddonManager() + manager.add(addonmod, {'key': 'val1'}) + manager.add('tests.test_addons.addons.GoodAddon') + six.assertCountEqual(self, manager, ['AddonModule', 'GoodAddon']) + self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) + six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) + self.assertEqual(manager.configs['AddonModule']['key'], 'val1') + self.assertRaises(ValueError, manager.add, addonmod) + + def test_add_dont_instantiate_providing_classes(self): + class ProviderGoodAddon(addons.GoodAddon): + pass + zope.interface.directlyProvides(ProviderGoodAddon, IAddon) + manager = AddonManager() + manager.add(ProviderGoodAddon) + self.assertIs(manager['GoodAddon'], ProviderGoodAddon) + + def test_add_verifies(self): + brokenaddon = self.manager.get_addon( + 'tests.test_addons.addons.BrokenAddon') + self.assertRaises(zope.interface.exceptions.BrokenImplementation, + self.manager.add, + brokenaddon) + + def test_add_adds_missing_interface_declaration(self): + class GoodAddonWithoutDeclaration(object): + name = 'GoodAddonWithoutDeclaration' + version = '1.0' + self.manager.add(GoodAddonWithoutDeclaration) + + def test_remove(self): + manager = AddonManager() + def test_gets_removed(removearg): + manager.add(addonmod) + self.assertIn('AddonModule', manager) + manager.remove(removearg) + self.assertNotIn('AddonModule', manager) + test_gets_removed('AddonModule') + test_gets_removed(addonmod) + test_gets_removed('tests.test_addons.addonmod') + test_gets_removed(self.ADDONMODPATH) + self.assertRaises(KeyError, manager.remove, 'nonexistent') + self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) + + def test_get_addon(self): + goodaddon = self.manager.get_addon( + 'tests.test_addons.addons.GoodAddon') + self.assertIs(goodaddon, addons.GoodAddon) + + loaded_addonmod = self.manager.get_addon(self.ADDONMODPATH) + # XXX: The module is in fact imported twice under different names into + # sys.modules, is there a good assertion for module equality? + self.assertEqual(loaded_addonmod.name, addonmod.name) + + # Does not provide interface, but has _addon attribute pointing to + # GoodAddon instance + addonspath = os.path.join(os.path.dirname(__file__), 'addons.py') + goodaddon = self.manager.get_addon(addonspath) + # XXX: Again, the imported class and addons.GoodAddon are different + # since they are imported twice. How to use isInstance? + self.assertEqual(goodaddon.name, addons.GoodAddon.name) + + self.assertRaises(NameError, self.manager.get_addon, 'xy.n_onexistent') + + def test_get_addon_forward(self): + class SomeCls(object): + _addon = 'tests.test_addons.addons.GoodAddon' + self.assertIs(self.manager.get_addon(SomeCls()), addons.GoodAddon) + + def test_get_addon_nested(self): + x = addons.GoodAddon('outer') + x._addon = addons.GoodAddon('middle') + x._addon._addon = addons.GoodAddon('inner') + self.assertIs(self.manager.get_addon(x), x._addon._addon) + + @mock.patch.object(scrapy.addons, 'get_project_path', + return_value='tests.test_addons.project') + def test_get_addon_prefixes(self, get_project_path_mock): + # From python path + self.assertEqual(self.manager.get_addon('addonmod').FROM, + 'test_addons.addonmod') + + # From project 'addons' folder + self.assertEqual(self.manager.get_addon('addonmod2').FROM, + 'test_addons.project.addons.addonmod2') + # Assert prefix priority '' > 'project.addons' + self.assertEqual(self.manager.get_addon('addonmod').FROM, + 'test_addons.addonmod') + + # From scrapy's 'addons' + from . import scrapy_addons + with mock.patch.dict('sys.modules', {'scrapy.addons': scrapy_addons}): + self.assertEqual(self.manager.get_addon('addonmod3').FROM, + 'test_addons.scrapy_addons.addonmod3') + # Assert prefix priority 'project.addons' > 'scrapy.addons' + self.assertEqual(self.manager.get_addon('addonmod2').FROM, + 'test_addons.project.addons.addonmod2') + # Assert prefix priority '' > 'scrapy.addons.' + self.assertEqual(self.manager.get_addon('addonmod').FROM, + 'test_addons.addonmod') + + def test_load_dict_load_settings(self): + def _test_load_method(func, *args, **kwargs): + manager = AddonManager() + getattr(manager, func)(*args, **kwargs) + six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) + self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) + six.assertCountEqual(self, manager.configs['GoodAddon'], + ['key']) + self.assertEqual(manager.configs['GoodAddon']['key'], 'val2') + # XXX: Check module equality, see above + self.assertEqual(manager['AddonModule'].name, addonmod.name) + self.assertIn('key', manager.configs['AddonModule']) + self.assertEqual(manager.configs['AddonModule']['key'], 'val1') + + addonsdict = { + self.ADDONMODPATH: { + 'key': 'val1', + }, + 'tests.test_addons.addons.GoodAddon': {'key': 'val2'}, + } + _test_load_method('load_dict', addonsdict) + + settings = BaseSettings() + settings.set('INSTALLED_ADDONS', [ + self.ADDONMODPATH, + 'tests.test_addons.addons.GoodAddon', + ]) + settings.set('ADDONMODULE', {'key': 'val1'}) + settings.set('GOODADDON', {'key': 'val2'}) + _test_load_method('load_settings', settings) + + def test_load_cfg(self): + manager = AddonManager() + manager.load_cfg(self.TESTCFGPATH) + six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) + self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) + six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) + self.assertEqual(manager.configs['GoodAddon']['key'], 'val1') + # XXX: Check module equality, see above + self.assertEqual(manager['AddonModule'].name, addonmod.name) + six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) + self.assertEqual(manager.configs['AddonModule']['key'], 'val2') + + def test_enabled_disabled(self): + manager = AddonManager() + manager.add(addons.GoodAddon('FirstAddon')) + manager.add(addons.GoodAddon('SecondAddon')) + self.assertEqual(set(manager.enabled), + set(('FirstAddon', 'SecondAddon'))) + self.assertEqual(manager.disabled, []) + manager.disable('FirstAddon') + self.assertEqual(manager.enabled, ['SecondAddon']) + self.assertEqual(manager.disabled, ['FirstAddon']) + manager.enable('FirstAddon') + self.assertEqual(set(manager.enabled), + set(('FirstAddon', 'SecondAddon'))) + self.assertEqual(manager.disabled, []) + + def test_enable_before_add(self): + manager = AddonManager() + self.assertRaises(ValueError, manager.enable, 'FirstAddon') + manager.disable('FirstAddon') + manager.enable('FirstAddon') + manager.add(addons.GoodAddon('FirstAddon')) + self.assertIn('FirstAddon', manager.enabled) + + def test_disable_before_add(self): + manager = AddonManager() + manager.disable('FirstAddon') + manager.add(addons.GoodAddon('FirstAddon')) + self.assertEqual(manager.disabled, ['FirstAddon']) + + def test_callbacks(self): + first_addon = addons.GoodAddon('FirstAddon') + second_addon = addons.GoodAddon('SecondAddon') + + manager = AddonManager() + manager.add(first_addon, {'test': 'first'}) + manager.add(second_addon, {'test': 'second'}) + crawler = mock.create_autospec(Crawler) + settings = BaseSettings() + + with mock.patch.object(first_addon, 'update_addons') as ua_first, \ + mock.patch.object(second_addon, 'update_addons') as ua_second, \ + mock.patch.object(first_addon, 'update_settings') as us_first, \ + mock.patch.object(second_addon, 'update_settings') as us_second, \ + mock.patch.object(first_addon, 'check_configuration') as cc_first, \ + mock.patch.object(second_addon, 'check_configuration') as cc_second: + manager.update_addons() + ua_first.assert_called_once_with(manager.configs['FirstAddon'], + manager) + ua_second.assert_called_once_with(manager.configs['SecondAddon'], + manager) + manager.update_settings(settings) + us_first.assert_called_once_with(manager.configs['FirstAddon'], + settings) + us_second.assert_called_once_with(manager.configs['SecondAddon'], + settings) + manager.check_configuration(crawler) + cc_first.assert_called_once_with(manager.configs['FirstAddon'], + crawler) + cc_second.assert_called_once_with(manager.configs['SecondAddon'], + crawler) + self.assertEqual(ua_first.call_count, 1) + self.assertEqual(ua_second.call_count, 1) + self.assertEqual(us_first.call_count, 1) + self.assertEqual(us_second.call_count, 1) + + us_first.reset_mock() + us_second.reset_mock() + manager.disable('FirstAddon') + manager.update_settings(settings) + self.assertEqual(us_first.call_count, 0) + manager.enable('FirstAddon') + manager.update_settings(settings) + self.assertEqual(us_first.call_count, 1) + self.assertEqual(us_second.call_count, 2) + + def test_update_addons_last_minute_add(self): + class AddedAddon(addons.GoodAddon): + name = 'AddedAddon' + + class FirstAddon(addons.GoodAddon): + name = 'FirstAddon' + def update_addons(self, config, addons): + addons.add(AddedAddon()) + + manager = AddonManager() + first_addon = FirstAddon() + with mock.patch.object(first_addon, 'update_addons', + wraps=first_addon.update_addons) as ua_first, \ + mock.patch.object(AddedAddon, 'update_addons') as ua_added: + manager.add(first_addon, {'non-empty': 'dict'}) + manager.update_addons() + six.assertCountEqual(self, manager, ['FirstAddon', 'AddedAddon']) + ua_first.assert_called_once_with(manager.configs['FirstAddon'], + manager) + ua_added.assert_called_once_with(manager.configs['AddedAddon'], + manager) + + def test_check_dependency_clashes_attributes(self): + provides = addons.GoodAddon("ProvidesAddon") + provides.provides = ('test', ) + provides2 = addons.GoodAddon("ProvidesAddon2") + provides2.provides = ('test', ) + requires = addons.GoodAddon("RequiresAddon") + requires.requires = ('test', ) + requires_name = addons.GoodAddon("RequiresNameAddon") + requires_name.requires = ('ProvidesAddon', ) + requires_newer = addons.GoodAddon("RequiresNewerAddon") + requires_newer.requires = ('test>=2.0', ) + modifies = addons.GoodAddon("ModifiesAddon") + modifies.modifies = ('test', ) + + def check_with(*addons): + manager = AddonManager() + for a in addons: + manager.add(a) + return manager.check_dependency_clashes() + + self.assertRaises(ImportError, check_with, requires) + self.assertRaises(ImportError, check_with, modifies) + self.assertRaises(ImportError, check_with, provides, provides2) + self.assertRaises(VersionConflict, check_with, provides, requires_newer) + with warnings.catch_warnings(record=True) as w: + check_with(provides, modifies) + check_with(provides) + check_with(provides, requires) + check_with(provides, requires_name) + self.assertEqual(len(w), 0) + check_with(requires, provides, modifies) + self.assertEqual(len(w), 1) diff --git a/tests/test_addons/addonmod.py b/tests/test_addons/addonmod.py new file mode 100644 index 000000000..8ecf4b81d --- /dev/null +++ b/tests/test_addons/addonmod.py @@ -0,0 +1,16 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = "test_addons.addonmod" + +name = "AddonModule" +version = "1.0" + +def update_settings(config, settings): + pass + +def check_configuration(config, crawler): + pass diff --git a/tests/test_addons/addons.py b/tests/test_addons/addons.py new file mode 100644 index 000000000..f3442b192 --- /dev/null +++ b/tests/test_addons/addons.py @@ -0,0 +1,40 @@ +import zope.interface + +from scrapy.addons import Addon +from scrapy.interfaces import IAddon + + +class Addon(object): + FROM = 'test_addons.addons' + + +@zope.interface.declarations.implementer(IAddon) +class GoodAddon(object): + + name = 'GoodAddon' + version = '1.0' + + def __init__(self, name=None, version=None): + if name is not None: + self.name = name + if version is not None: + self.version = version + + def update_addons(self, config, addons): + pass + + def update_settings(self, config, settings): + pass + + def check_configuration(self, config, crawler): + pass + + +@zope.interface.declarations.implementer(IAddon) +class BrokenAddon(object): + + name = 'BrokenAddon' + # No version + + +_addon = GoodAddon() diff --git a/tests/test_addons/cfg.cfg b/tests/test_addons/cfg.cfg new file mode 100644 index 000000000..98c4f0f25 --- /dev/null +++ b/tests/test_addons/cfg.cfg @@ -0,0 +1,5 @@ +[addon:tests.test_addons.addons.GoodAddon] +key = val1 + +[addon:tests/test_addons/addonmod.py] +key = val2 diff --git a/tests/test_addons/project/__init__.py b/tests/test_addons/project/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_addons/project/addons/__init__.py b/tests/test_addons/project/addons/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_addons/project/addons/addonmod.py b/tests/test_addons/project/addons/addonmod.py new file mode 100644 index 000000000..66ca644f8 --- /dev/null +++ b/tests/test_addons/project/addons/addonmod.py @@ -0,0 +1,7 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = 'test_addons.project.addons.addonmod' diff --git a/tests/test_addons/project/addons/addonmod2.py b/tests/test_addons/project/addons/addonmod2.py new file mode 100644 index 000000000..0dbdd70ff --- /dev/null +++ b/tests/test_addons/project/addons/addonmod2.py @@ -0,0 +1,7 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = 'test_addons.project.addons.addonmod2' diff --git a/tests/test_addons/scrapy_addons/__init__.py b/tests/test_addons/scrapy_addons/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_addons/scrapy_addons/addonmod.py b/tests/test_addons/scrapy_addons/addonmod.py new file mode 100644 index 000000000..fa479aa68 --- /dev/null +++ b/tests/test_addons/scrapy_addons/addonmod.py @@ -0,0 +1,7 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = 'test_addons.scrapy_addons.addonmod' diff --git a/tests/test_addons/scrapy_addons/addonmod2.py b/tests/test_addons/scrapy_addons/addonmod2.py new file mode 100644 index 000000000..da053af4a --- /dev/null +++ b/tests/test_addons/scrapy_addons/addonmod2.py @@ -0,0 +1,7 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = 'test_addons.scrapy_addons.addonmod2' diff --git a/tests/test_addons/scrapy_addons/addonmod3.py b/tests/test_addons/scrapy_addons/addonmod3.py new file mode 100644 index 000000000..c64521478 --- /dev/null +++ b/tests/test_addons/scrapy_addons/addonmod3.py @@ -0,0 +1,7 @@ +import zope.interface + +from scrapy.interfaces import IAddon + +zope.interface.moduleProvides(IAddon) + +FROM = 'test_addons.scrapy_addons.addonmod3' diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 06af3c009..f33562b7d 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -3,7 +3,8 @@ import os import unittest from scrapy.item import Item, Field -from scrapy.utils.misc import load_object, arg_to_iter, walk_modules +from scrapy.utils.misc import (load_object, load_module_or_object, arg_to_iter, + walk_modules, get_module_from_filepath) __doctests__ = ['scrapy.utils.misc'] @@ -17,6 +18,15 @@ class UtilsMiscTestCase(unittest.TestCase): self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') + def test_load_module_or_object(self): + testmod = load_module_or_object(__name__ + '.testmod') + self.assertTrue(hasattr(testmod, 'TESTVAR')) + testmod = load_module_or_object( + os.path.join(os.path.dirname(__file__), 'testmod.py')) + self.assertTrue(hasattr(testmod, 'TESTVAR')) + obj = load_object('scrapy.utils.misc.load_object') + self.assertIs(obj, load_object) + def test_walk_modules(self): mods = walk_modules('tests.test_utils_misc.test_walk_modules') expected = [ @@ -57,6 +67,20 @@ class UtilsMiscTestCase(unittest.TestCase): finally: sys.path.remove(egg) + def test_get_module_from_filepath(self): + testmodpath = os.path.join(os.path.dirname(__file__), 'testmod.py') + testmod = get_module_from_filepath(testmodpath) + self.assertTrue(hasattr(testmod, 'TESTVAR')) + + testpkgpath = os.path.join(os.path.dirname(__file__), 'testpkg') + testpkg = get_module_from_filepath(testpkgpath) + self.assertTrue(hasattr(testpkg, 'TESTVAR2')) + # Check submodule access + import testpkg.submod + self.assertTrue(hasattr(testpkg.submod, 'TESTVAR3')) + self.assertIs(testpkg.submod.TESTVAR3, + load_object(testpkg.__name__ + ".submod.TESTVAR3")) + def test_arg_to_iter(self): class TestItem(Item): diff --git a/tests/test_utils_misc/testmod.py b/tests/test_utils_misc/testmod.py new file mode 100644 index 000000000..eb540335f --- /dev/null +++ b/tests/test_utils_misc/testmod.py @@ -0,0 +1 @@ +TESTVAR = True diff --git a/tests/test_utils_misc/testpkg/__init__.py b/tests/test_utils_misc/testpkg/__init__.py new file mode 100644 index 000000000..12cc2f6d9 --- /dev/null +++ b/tests/test_utils_misc/testpkg/__init__.py @@ -0,0 +1 @@ +TESTVAR2 = True diff --git a/tests/test_utils_misc/testpkg/submod.py b/tests/test_utils_misc/testpkg/submod.py new file mode 100644 index 000000000..8a07e3592 --- /dev/null +++ b/tests/test_utils_misc/testpkg/submod.py @@ -0,0 +1 @@ +TESTVAR3 = True diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py new file mode 100644 index 000000000..cea4d9950 --- /dev/null +++ b/tests/test_utils_project.py @@ -0,0 +1,27 @@ +import os +from tests import mock +import unittest + +from scrapy.exceptions import NotConfigured +from scrapy.utils.project import get_project_path, inside_project + + +class UtilsProjectTestCase(unittest.TestCase): + + @mock.patch('scrapy.utils.project.inside_project', return_value=True) + def test_get_project_path(self, mock_ip): + def _test(settingsmod, expected): + with mock.patch.dict('os.environ', + {'SCRAPY_SETTINGS_MODULE': settingsmod}): + self.assertEqual(get_project_path(), expected) + _test('project.settings', 'project') + _test('project.othername', 'project') + _test('nested.project.settings', 'nested.project') + + with mock.patch.dict('os.environ', {}, clear=True): + self.assertRaises(NotConfigured, get_project_path) + + mock_ip.return_value = False + with mock.patch.dict('os.environ', + {'SCRAPY_SETTINGS_MODULE': 'some.settings'}): + self.assertRaises(NotConfigured, get_project_path) From 07455b1883cacab141572df4310315bb53e65ec9 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 19 Aug 2015 16:17:09 +0200 Subject: [PATCH 04/55] Integrate add-ons into start-up process --- scrapy/cmdline.py | 6 +++++- scrapy/crawler.py | 19 ++++++++++++++----- scrapy/utils/test.py | 4 ++-- tests/test_crawl.py | 17 +++++++++++++++++ tests/test_crawler.py | 24 ++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 8 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 35050c13d..b403df570 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -6,6 +6,7 @@ import inspect import pkg_resources import scrapy +from scrapy.addons import AddonManager from scrapy.crawler import CrawlerProcess from scrapy.xlib import lsprofcalltree from scrapy.commands import ScrapyCommand @@ -118,6 +119,9 @@ def execute(argv=None, settings=None): conf.settings = settings # ------------------------------------------------------------------ + addons = AddonManager() + addons.load_cfg() + inproject = inside_project() cmds = _get_commands_dict(settings, inproject) cmdname = _pop_command_name(argv) @@ -139,7 +143,7 @@ def execute(argv=None, settings=None): opts, args = parser.parse_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) - cmd.crawler_process = CrawlerProcess(settings) + cmd.crawler_process = CrawlerProcess(settings, addons) _run_print_help(parser, _run_command, cmd, args, opts) sys.exit(cmd.exitcode) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index bdcfa9d0c..8107a50aa 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -6,6 +6,7 @@ import warnings from twisted.internet import reactor, defer from zope.interface.verify import verifyClass, DoesNotImplement +from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine from scrapy.resolver import CachingThreadedResolver from scrapy.interfaces import ISpiderLoader @@ -23,7 +24,7 @@ logger = logging.getLogger(__name__) class Crawler(object): - def __init__(self, spidercls, settings=None): + def __init__(self, spidercls, settings=None, addons=None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -31,6 +32,12 @@ class Crawler(object): self.settings = settings.copy() self.spidercls.update_settings(self.settings) + self.addons = addons if addons is not None else AddonManager() + self.addons.load_settings(self.settings) + self.addons.update_addons() + self.addons.check_dependency_clashes() + self.addons.update_settings(self.settings) + self.signals = SignalManager(self) self.stats = load_object(self.settings['STATS_CLASS'])(self) @@ -69,6 +76,7 @@ class Crawler(object): try: self.spider = self._create_spider(*args, **kwargs) self.engine = self._create_engine() + self.addons.check_configuration(self) start_requests = iter(self.spider.start_requests()) yield self.engine.open_spider(self.spider, start_requests) yield defer.maybeDeferred(self.engine.start) @@ -111,10 +119,11 @@ class CrawlerRunner(object): ":meth:`crawl` and managed by this class." ) - def __init__(self, settings=None): + def __init__(self, settings=None, addons=None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings + self.addons = addons self.spider_loader = _get_spider_loader(settings) self._crawlers = set() self._active = set() @@ -181,7 +190,7 @@ class CrawlerRunner(object): def _create_crawler(self, spidercls): if isinstance(spidercls, six.string_types): spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings) + return Crawler(spidercls, self.settings, self.addons) def stop(self): """ @@ -223,8 +232,8 @@ class CrawlerProcess(CrawlerRunner): process. See :ref:`run-from-script` for an example. """ - def __init__(self, settings=None): - super(CrawlerProcess, self).__init__(settings) + def __init__(self, settings=None, addons=None): + super(CrawlerProcess, self).__init__(settings, addons) install_shutdown_handlers(self._signal_shutdown) configure_logging(self.settings) log_scrapy_info(self.settings) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 51edfd353..0e11ec7d1 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -20,7 +20,7 @@ def assert_aws_environ(): if 'AWS_ACCESS_KEY_ID' not in os.environ: raise SkipTest("AWS keys not found") -def get_crawler(spidercls=None, settings_dict=None): +def get_crawler(spidercls=None, settings_dict=None, addons=None): """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level priority. @@ -28,7 +28,7 @@ def get_crawler(spidercls=None, settings_dict=None): from scrapy.crawler import CrawlerRunner from scrapy.spiders import Spider - runner = CrawlerRunner(settings_dict) + runner = CrawlerRunner(settings_dict, addons) return runner.create_crawler(spidercls or Spider) def get_pythonpath(): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 814eb30d2..7358009e9 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -6,6 +6,7 @@ from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase +from scrapy.addons import Addon, AddonManager from scrapy.http import Request from scrapy.crawler import CrawlerRunner from tests import mock @@ -266,3 +267,19 @@ with multiples lines self._assert_retried(log) self.assertIn("Got response 200", str(log)) + + @defer.inlineCallbacks + def test_abort_on_addon_failed_check(self): + class FailedCheckAddon(Addon): + name = 'FailedCheckAddon' + version = '1.0' + def check_configuration(self, config, crawler): + raise ValueError + addonmgr = AddonManager() + addonmgr.add(FailedCheckAddon()) + crawler = self.runner.create_crawler(SimpleSpider) + crawler.addons = addonmgr + # Doesn't work in 'precise' test environment: + #with self.assertRaises(ValueError): + # yield crawler.crawl() + yield self.assertFailure(crawler.crawl(), ValueError) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 53a1202e3..dfad11405 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -2,6 +2,7 @@ import warnings import unittest import scrapy +from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader @@ -51,6 +52,29 @@ class CrawlerTestCase(BaseCrawlerTest): self.assertFalse(settings.frozen) self.assertTrue(crawler.settings.frozen) + def test_populate_addons_settings(self): + class TestAddon(Addon): + name = 'TestAddon' + version = '1.0' + addonconfig = {'TEST1': 'addon', 'TEST2': 'addon', 'TEST3': 'addon'} + class TestAddon2(Addon): + name = 'testAddon2' + version = '1.0' + addonconfig2 = {'TEST': 'addon2'} + + settings = Settings() + settings.set('TESTADDON_TEST1', 'project', priority='project') + settings.set('TESTADDON_TEST2', 'default', priority='default') + addonmgr = AddonManager() + addonmgr.add(TestAddon(), addonconfig) + addonmgr.add(TestAddon2(), addonconfig2) + crawler = Crawler(DefaultSpider, settings, addonmgr) + + self.assertEqual(crawler.settings['TESTADDON_TEST1'], 'project') + self.assertEqual(crawler.settings['TESTADDON_TEST2'], 'addon') + self.assertEqual(crawler.settings['TESTADDON_TEST3'], 'addon') + self.assertEqual(crawler.settings['TESTADDON2_TEST'], 'addon2') + def test_crawler_accepts_dict(self): crawler = Crawler(DefaultSpider, {'foo': 'bar'}) self.assertEqual(crawler.settings['foo'], 'bar') From d91647c38b9504ca514a7cfc4d6c4fdb54c3d853 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 17 Aug 2015 02:48:12 +0200 Subject: [PATCH 05/55] Add built-in add-ons --- scrapy/addons/__init__.py | 3 + scrapy/addons/builtins.py | 293 +++++++++++++++++++++++++++++ tests/test_addons/test_builtins.py | 42 +++++ 3 files changed, 338 insertions(+) create mode 100644 scrapy/addons/builtins.py create mode 100644 tests/test_addons/test_builtins.py diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 59e59e15f..420d46b68 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -495,3 +495,6 @@ class AddonManager(Mapping): """ for name in self: self._call_addon(name, 'check_configuration', crawler) + + +from scrapy.addons.builtins import * diff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.py new file mode 100644 index 000000000..ff7902afb --- /dev/null +++ b/scrapy/addons/builtins.py @@ -0,0 +1,293 @@ +import scrapy +from scrapy.addons import Addon + +__all__ = ['make_builtin_addon', + + 'depth', 'httperror', 'offsite', 'referer', 'urllength', + + 'ajaxcrawl', 'chunked', 'cookies', 'defaultheaders', + 'downloadtimeout', 'httpauth', 'httpcache', 'httpcompression', + 'httpproxy', 'metarefresh', 'redirect', 'retry', 'robotstxt', + 'stats', 'useragent', + + 'autothrottle', 'corestats', 'closespider', 'debugger', 'feedexport', + 'logstats', 'memdebug', 'memusage', 'spiderstate', 'stacktracedump', + 'statsmailer', 'telnetconsole', + ] + + +def make_builtin_addon(addon_name, comp_type, comp, order=0, + addon_default_config=None, addon_version=None): + class ThisAddon(Addon): + name = addon_name + version = addon_version or scrapy.__version__ + component_type = comp_type + component = comp + component_order = order + default_config = addon_default_config or {} + + return ThisAddon + + +# XXX: Below are CLASSES that have lowercase names. This is in line with the +# original SEP-021 but violates PEP8. +# We might consider prepending all built-in addon names with scrapy_ or similar +# to reduce the chance of name clashes. + +# SPIDER MIDDLEWARES + +depth = make_builtin_addon( + 'depth', + 'SPIDER_MIDDLEWARES', + 'scrapy.spidermiddlewares.depth.DepthMiddleware', + 900, +) + +httperror = make_builtin_addon( + 'httperror', + 'SPIDER_MIDDLEWARES', + 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', + 50, +) + +offsite = make_builtin_addon( + 'offsite', + 'SPIDER_MIDDLEWARES', + 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', + 500, +) + +referer = make_builtin_addon( + 'referer', + 'SPIDER_MIDDLEWARES', + 'scrapy.spidermiddlewares.referer.RefererMiddleware', + 700, + {'enabled': True}, +) + +urllength = make_builtin_addon( + 'urllength', + 'SPIDER_MIDDLEWARES', + 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', + 800, +) + + +# DOWNLOADER MIDDLEWARES + +ajaxcrawl = make_builtin_addon( + 'ajaxcrawl', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware', + 560, +) + +chunked = make_builtin_addon( + 'chunked', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware', + 830, +) + +cookies = make_builtin_addon( + 'cookies', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware', + 700, + {'enabled': True}, +) + +defaultheaders = make_builtin_addon( + 'defaultheaders', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', + 550, +) +# Assume every config entry is a header +def defaultheaders_export_config(self, config, settings): + conf = self.default_config or {} + conf.update(config) + settings.set('DEFAULT_REQUEST_HEADERS', conf, 'addon') +defaultheaders.export_config = defaultheaders_export_config + +downloadtimeout = make_builtin_addon( + 'downloadtimeout', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', + 350, +) +downloadtimeout.config_mapping = {'timeout': 'DOWNLOAD_TIMEOUT', + 'download_timeout': 'DOWNLOAD_TIMEOUT'} + +httpauth = make_builtin_addon( + 'httpauth', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', + 300, +) + +httpcache = make_builtin_addon( + 'httpcache', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', + 900, + {'enabled': True}, +) + +httpcompression = make_builtin_addon( + 'httpcompression', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', + 590, + {'enabled': True}, +) +httpcompression.config_mapping = {'enabled': 'COMPRESSION_ENABLED'} + +httpproxy = make_builtin_addon( + 'httpproxy', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', + 750, +) + +metarefresh = make_builtin_addon( + 'metarefresh', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', + 580, + {'enabled': True}, +) +metarefresh.config_mapping = {'max_times': 'REDIRECT_MAX_TIMES'} + +redirect = make_builtin_addon( + 'redirect', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', + 600, + {'enabled': True}, +) + +retry = make_builtin_addon( + 'retry', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.retry.RetryMiddleware', + 500, + {'enabled': True}, +) + +robotstxt = make_builtin_addon( + 'robotstxt', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', + 100, + {'obey': True}, +) + +stats = make_builtin_addon( + 'stats', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.stats.DownloaderStats', + 850, +) + +useragent = make_builtin_addon( + 'useragent', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', + 400, +) +useragent.config_mapping = {'user_agent': 'USER_AGENT'} + + +# ITEM PIPELINES + + +# EXTENSIONS + +autothrottle = make_builtin_addon( + 'throttle', + 'EXTENSIONS', + 'scrapy.extensions.throttle.AutoThrottle', + 0, + {'enabled': True}, +) + +corestats = make_builtin_addon( + 'corestats', + 'EXTENSIONS' + 'scrapy.extensions.corestats.CoreStats', + 0, +) + +closespider = make_builtin_addon( + 'closespider', + 'EXTENSIONS' + 'scrapy.extensions.closespider.CloseSpider', + 0, +) + +debugger = make_builtin_addon( + 'debugger', + 'EXTENSIONS' + 'scrapy.extensions.debug.Debugger', + 0, +) + +feedexport = make_builtin_addon( + 'feedexport', + 'EXTENSIONS' + 'scrapy.extensions.feedexport.FeedExporter', + 0, +) +feedexport.settings_prefix = 'FEED' + +logstats = make_builtin_addon( + 'logstats', + 'EXTENSIONS' + 'scrapy.extensions.logstats.LogStats', + 0, +) + +memdebug = make_builtin_addon( + 'memdebug', + 'EXTENSIONS' + 'scrapy.extensions.memdebug.MemoryDebugger', + 0, + {'enabled': True}, +) + +memusage = make_builtin_addon( + 'memusage', + 'EXTENSIONS' + 'scrapy.extensions.memusage.MemoryUsage', + 0, + {'enabled': True}, +) + +spiderstate = make_builtin_addon( + 'spiderstate', + 'EXTENSIONS' + 'scrapy.extensions.spiderstate.SpiderState', + 0, +) + +stacktracedump = make_builtin_addon( + 'stacktracedump', + 'EXTENSIONS' + 'scrapy.extensions.debug.StackTraceDump', + 0, +) + +statsmailer = make_builtin_addon( + 'statsmailer', + 'EXTENSIONS' + 'scrapy.extensions.statsmailer.StatsMailer', + 0, +) + +telnetconsole = make_builtin_addon( + 'telnetconsole', + 'EXTENSIONS' + 'scrapy.telnet.TelnetConsole', + 0, +) diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.py new file mode 100644 index 000000000..607c911fb --- /dev/null +++ b/tests/test_addons/test_builtins.py @@ -0,0 +1,42 @@ +import unittest + +import scrapy +import scrapy.addons +from scrapy.addons.builtins import make_builtin_addon +from scrapy.settings import Settings + + +class BuiltinAddonsTest(unittest.TestCase): + + def test_make_builtin_addon(self): + httpcache = make_builtin_addon( + 'httpcache', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', + 900, + {'enabled': True}, + ) + self.assertEqual(httpcache.name, 'httpcache') + self.assertEqual(httpcache.component_type, 'DOWNLOADER_MIDDLEWARES') + self.assertEqual(httpcache.component, 'scrapy.downloadermiddlewares.' + 'httpcache.HttpCacheMiddleware') + self.assertEqual(httpcache.component_order, 900) + self.assertEqual(httpcache.default_config, {'enabled': True}) + self.assertEqual(httpcache.version, scrapy.__version__) + httpcache = make_builtin_addon( + 'httpcache', + 'DOWNLOADER_MIDDLEWARES', + 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', + 900, + {'enabled': True}, + '99.9', + ) + self.assertEqual(httpcache.version, '99.9') + + def test_defaultheaders_export_config(self): + settings = Settings() + dh = scrapy.addons.defaultheaders() + dh.export_config({'X-Test-Header': 'val'}, settings) + self.assertIn('X-Test-Header', settings['DEFAULT_REQUEST_HEADERS']) + self.assertEqual(settings['DEFAULT_REQUEST_HEADERS']['X-Test-Header'], + 'val') From 2946b674144833254f3ec6dd6aece0156f71a21b Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Tue, 28 Jul 2015 12:58:05 +0200 Subject: [PATCH 06/55] Document add-ons --- docs/index.rst | 4 + docs/topics/addons.rst | 387 ++++++++++++++++++++++++++++++++++++++ scrapy/addons/__init__.py | 4 +- 3 files changed, 394 insertions(+), 1 deletion(-) create mode 100644 docs/topics/addons.rst diff --git a/docs/index.rst b/docs/index.rst index 0d21f5d40..3e8a220e9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -206,6 +206,7 @@ Extending Scrapy :hidden: topics/architecture + topics/addons topics/downloader-middleware topics/spider-middleware topics/extensions @@ -217,6 +218,9 @@ Extending Scrapy :doc:`topics/architecture` Understand the Scrapy architecture. +:doc:`topics/addons` + Enable and configure built-in and third-party extensions. + :doc:`topics/downloader-middleware` Customize how pages get requested and downloaded. diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst new file mode 100644 index 000000000..39ef286eb --- /dev/null +++ b/docs/topics/addons.rst @@ -0,0 +1,387 @@ +.. _topics-addons: + +======= +Add-ons +======= + +Scrapy's add-on system is a framework which unifies managing and configuring +components that extend Scrapy's core functionality, such as middlewares, +extensions, or pipelines. It provides users with a plug-and-play experience in +Scrapy extension management, and grants extensive configuration control to +developers. + + +Activating and configuring add-ons +================================== + +Add-ons and their configuration live in Scrapy's +:class:`~scrapy.addons.AddonManager`. During Scrapy's start-up process, and +only then, the add-on manager will read a list of enabled add-ons and their +configurations from your settings. There are two places where you can provide +the paths to add-ons you want to enable: + +* the ``INSTALLED_ADDONS`` setting, and +* the ``scrapy.cfg`` file. + +As Scrapy settings can be modified from many places, e.g. in a project's +``settings.py``, in a Spider's ``custom_settings`` attribute, or from the +command line, using the ``INSTALLED_ADDONS`` setting is the preferred way to +manage add-ons. + +The ``INSTALLED_ADDONS`` setting a tuple in which every item is a path to an +add-on. The path can be both a Python or a file path. While more precise, it is +not necessary to specify the full add-on Python path if it is either built into +Scrapy or lives in your project's ``addons`` submodule. + +The configuration of an add-on, if necessary at all, is stored as a dictionary +setting whose name is the uppercase add-on name. + +This is an example where an internal add-on and two third-party add-ons (in this +case with one requiring no configuration) are enabled/configured in a project's +``settings.py``:: + + INSTALLED_ADDONS = ( + 'httpcache', + 'path.to.some.addon', + 'path/to/other/addon.py', + ) + + HTTPCACHE = { + 'expiration_secs': 60, + 'ignore_http_codes': [404, 405], + } + + SOMEADDON = { + 'some_config': True, + } + +It is also possible to manage add-ons from ``scrapy.cfg``. While the syntax is +a little friendlier, be aware that this file, and therefore the configuration in +it, is not bound to a particular Scrapy project. While this should not pose a +problem when you use the project on your development machine only, a common +stumbling block is that ``scrapy.cfg`` is not deployed via ``scrapyd-deploy``. + +In ``scrapy.cfg``, section names, prepended with ``addon:``, replace the +dictionary keys. I.e., the configuration from above would look like this: + +.. code-block:: cfg + + [addon:httpcache] + expiration_secs = 60 + ignore_http_codes = 404,405 + + [addon:path.to.some.addon] + some_config = true + + [addon:path/to/other/addon.py] + + +Enabling and configuring add-ons within Python code +--------------------------------------------------- + +The :class:`~scrapy.addons.AddonManager` will only read from Scrapy's settings +and from ``scrapy.cfg`` *at the beginning* of Scrapy's start-up process. +Afterwards, i.e. as soon as the :class:`~scrapy.addons.AddonManager` is +populated, changing the ``INSTALLED_ADDONS`` setting or any of the add-on +configuration dictionary settings will have no effect. + +If you want to enable, disable, or configure add-ons in Python code, for example +when writing your own add-on, you will have to use the +:class:`~scrapy.addons.AddonManager`. You can access the add-on manager through +either ``crawler.addons`` or, if you are writing an add-on, through the +``addons`` argument of the :meth:`update_addons` callback. The add-on manager +provides many useful methods and attributes to facilitate interacting with the +add-ons framework, e.g.: + +* an :meth:`~scrapy.addons.AddonManager.add` method to load add-ons, +* the :attr:`~scrapy.addons.AddonManager.enabled` list of enabled add-ons, +* :meth:`~scrapy.addons.AddonManager.enable` and + :meth:`~scrapy.addons.AddonManager.disable` methods, +* the :attr:`~scrapy.addons.AddonManager.configs` dictionary which holds the + configuration of all add-ons + +In this example, we ensure that the ``httpcache`` add-on is loaded, and that +its ``expiration_secs`` configuration is set to ``60``:: + + # addons is an instance of AddonManager + if 'httpcache' not in addons: + addons.add('httpcache', {'expiration_secs': 60}) + else: + addons.configs['httpcache']['expiration_secs'] = 60 + + +Writing your own add-ons +======================== + +Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*. +The interface is enforced through ``zope.interface``. This leaves the choice of +Python object up the developer. Examples: + +* for a small pipeline, the add-on interface could be implemented in the same + class that also implements the ``open/close_spider`` and ``process_item`` + callbacks +* for larger add-ons, or for clearer structure, the interface could be provided + by a stand-alone module + +The absolute minimum interface consists of two attributes: + +.. attribute:: name + + string with add-on name + +.. attribute:: version + + version string (PEP-404, e.g. ``'1.0.1'``) + +Of course, stating just these two attributes will not get you very far. Add-ons +can provide three callback methods that are called at various stages before the +crawling process: + +.. method:: update_settings(config, settings) + + This method is called during the initialization of the + :class:`~scrapy.crawler.Crawler`. Here, you should perform dependency checks + (e.g. for external Python libraries) and update the + :class:`~scrapy.settings.Settings` object as wished, e.g. enable components + for this add-on or set required configuration of other extensions. + + :param config: Configuration of this add-on + :type config: ``dict`` + + :param settings: The settings object storing Scrapy/component configuration + :type settings: :class:`~scrapy.settings.Settings` + +.. method:: check_configuration(config, crawler) + + This method is called when the :class:`~scrapy.crawler.Crawler` has been + fully initialized, immediately before it starts crawling. You can perform + additional dependency and configuration checks here. + + :param config: Configuration of this add-on + :type config: ``dict`` + + :param crawler: Fully initialized Scrapy crawler + :type crawler: :class:`~scrapy.crawler.Crawler` + +.. method:: update_addons(config, addons) + + This method is called immediately before :meth:`update_settings`, and should + be used to enable and configure other *add-ons* only. + + When using this callback, be aware that there is no guarantee in which order + the :meth:`update_addons` callbacks of enabled add-ons will be called. + Add-ons that are added to the :class:`~scrapy.addons.AddonManager` during + this callback will also have their :meth:`update_addons` method called. + + :param config: Configuration of this add-on + :type config: ``dict`` + + :param addons: Add-on manager holding all loaded add-ons + :type addons: :class:`~scrapy.addons.AddonManager` + +Additionally, add-ons may (and should, where appropriate) provide one or more +attributes that can be used for limited automated detection of possible +dependency clashes: + +.. attribute:: requires + + list of built-in or custom components needed by this add-on, as strings. + +.. attribute:: modifies + + list of built-in or custom components whose functionality is affected or + replaced by this add-on (a custom HTTP cache should list ``httpcache`` here) + +.. attribute:: provides + + list of components provided by this add-on (e.g. ``mongodb`` for an + extension that provides generic read/write access to a MongoDB database) + +The entries in the :attr:`requires` and :attr:`modifies` attributes can be add-on +names or components from other add-ons' :attr:`provides` attribute. You can +specify :pep:`440`-style information about required versions. Examples:: + + requires = ['httpcache'] + requires = ['otheraddon >= 2.0', 'yetanotheraddon'] + +The Python object or module that is pointed to by an add-on path (e.g. given in +the ``INSTALLED_ADDONS`` setting, or given to +:meth:`~scrapy.addons.AddonManager.add`) does not necessarily have to be an +add-on. Instead, it can provide an ``_addon`` attribute. This attribute can be +either an add-on or another add-on path. + + +Add-on base class +================= + +Scrapy comes with a built-in base class for add-ons which provides some +convenience functionality: + +* basic settings can be exported via :meth:`~scrapy.addons.Addon.export_basics`, + configurable via :attr:`~scrapy.addons.Addon.basic_settings`. +* a single component (e.g. an item pipeline or a downloader middleware) can be + inserted into Scrapy's settings via + :meth:`~scrapy.addons.Addon.export_component`, configurable via + :attr:`~scrapy.addons.Addon.component_type`, + :attr:`~scrapy.addons.Addon.component_key`, + :attr:`~scrapy.addons.Addon.component`, and the ``order`` key in + :attr:`~scrapy.addons.Addon.default_config`. +* the add-on configuration can be exposed into Scrapy's settings via + :meth:`~scrapy.addons.Addon.export_config`, configurable via + :attr:`~scrapy.addons.Addon.default_config`, + :attr:`~scrapy.addons.Addon.config_mapping`, and + :attr:`~scrapy.addons.Addon.settings_prefix`. + +By default, the base add-on class will expose the add-on configuration into +Scrapy's settings namespace, in caps and with the add-on name prepended. It is +easy to write your own functionality while still being able to use the +convenience functions by overwriting +:meth:`~scrapy.addons.Addon.update_settings`. + +.. module:: scrapy.addons + +.. autoclass:: Addon + :members: + + +Add-on examples +=============== + +Set some basic configuration using the :class:`Addon` base class:: + + from scrapy.addons import Addon + + class MyAddon(Addon): + name = 'myaddon' + version = '1.0' + component = 'path.to.mypipeline' + component_type = 'ITEM_PIPELINES' + component_order = 200 + basic_settings = { + 'DNSCACHE_ENABLED': False, + } + +Check dependencies:: + + from scrapy.addons import Addon + + class MyAddon(Addon): + name = 'myaddon' + version = '1.0' + + def update_settings(self, config, settings): + try: + import boto + except ImportError: + raise RuntimeError("myaddon requires the boto library") + else: + self.export_config(config, settings) + +Enable a component that lives relative to the add-on (see +:ref:`topics-api-settings`):: + + from scrapy.addons import Addon + + class MyAddon(Addon): + name = 'myaddon' + version = '1.0' + component = __name__ + '.downloadermw.coolmw' + component_type = 'DOWNLOADER_MIDDLEWARES' + component_order = 900 + +Instantiate components ad hoc:: + + from path.to.my.pipelines import MySQLPipeline + + class MyAddon(object): + name = 'myaddon' + version = '1.0' + + def update_settings(self, config, settings): + mysqlpl = MySQLPipeline(password=config['password']) + settings.set( + 'ITEM_PIPELINES', + {mysqlpl: 200}, + priority='addon', + ) + +Provide add-on interface along component interface:: + + class MyPipeline(object): + name = 'mypipeline' + version = '1.0' + + def process_item(self, item, spider): + # Do some processing here + return item + + def update_settings(self, config, settings): + settings.set( + 'ITEM_PIPELINES', + {self: 200}, + priority='addon', + ) + +Enable another addon (see :ref:`topics-api-addonmanager`):: + + class MyAddon(object): + name = 'myaddon' + version = '1.0' + + def update_addons(self, config, addons): + if 'httpcache' not in addons.enabled: + addons.add('httpcache', {'expiration_secs': 60}) + +Check configuration of fully initialized crawler (see +:ref:`topics-api-crawler`):: + + class MyAddon(object): + name = 'myaddon' + version = '1.0' + + def update_settings(self, config, settings): + settings.set('DNSCACHE_ENABLED', False, priority='addon') + + def check_configuration(self, config, crawler): + if crawler.settings.getbool('DNSCACHE_ENABLED'): + # The spider, some other add-on, or the user messed with the + # DNS cache setting + raise ValueError("myaddon is incompatible with DNS cache") + +Provide add-on interface through a module: + +.. No idea why just using '::' doesn't work for this one +.. code-block:: python + + name = 'AddonModule' + version = '1.0' + + class MyPipeline(object): + # ... + + class MyDownloaderMiddleware(object): + # ... + + def update_settings(config, settings): + settings.set( + 'ITEM_PIPELINES', + {MyPipeline(): 200}, + priority='addon', + } + settings.set( + 'DOWNLOADER_MIDDLEWARES', + {MyDownloaderMiddleware(): 800}, + priority='addon', + } + +Forward to other add-ons depending on Python version:: + + # This could be a Python module, say project/pipelines/mypipeline.py, but + # could also be done inside a class, etc. + import six + + if six.PY3: + # We're running Python 3 + _addon = 'path.to.addon' + else: + _addon = 'path.to.other.addon' diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 420d46b68..15460143c 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -156,13 +156,15 @@ class AddonManager(Mapping): """This class facilitates loading and storing :ref:`topics-addons`. You can treat it like a read-only dictionary in which keys correspond to - add-on names and values correspond to the add-on objects:: + add-on names and values correspond to the add-on objects. Add-on + configurations are saved in the :attr:`config` dictionary attribute:: addons = AddonManager() # ... load some add-ons here print addons.enabled # prints names of all enabled add-ons print addons['TestAddon'].version # prints version of add-on with name # 'TestAddon' + print addons.configs['TestAddon'] # prints configuration of 'TestAddon' """ From 25498c3c210ea5aa1a45aed02618033d832cfc1c Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 24 Aug 2015 18:04:58 +0200 Subject: [PATCH 07/55] Remove unused imports in add-ons --- scrapy/addons/__init__.py | 3 --- tests/test_addons/__init__.py | 4 +--- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 15460143c..b1d6e14cb 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -1,7 +1,5 @@ from collections import defaultdict, Mapping -from importlib import import_module from inspect import isclass -import os import six import warnings @@ -11,7 +9,6 @@ from zope.interface.verify import verifyObject from scrapy.exceptions import NotConfigured from scrapy.interfaces import IAddon -from scrapy.settings import BaseSettings from scrapy.utils.conf import config_from_filepath, get_config from scrapy.utils.misc import load_module_or_object from scrapy.utils.project import get_project_path diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index c98f0ab65..84870ec52 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -1,7 +1,5 @@ import os.path import six -from six.moves.configparser import SafeConfigParser -import sys from tests import mock import unittest import warnings @@ -15,7 +13,7 @@ import scrapy.addons from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler from scrapy.interfaces import IAddon -from scrapy.settings import BaseSettings, Settings +from scrapy.settings import BaseSettings from . import addons from . import addonmod From 4ac6a83072f6e507f7f952b9c466419bec43364c Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Wed, 26 Aug 2015 01:43:59 +0200 Subject: [PATCH 08/55] Fix class signatures in Extensions docs --- docs/topics/extensions.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 847353868..b29e1802f 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -183,7 +183,7 @@ Telnet console extension .. module:: scrapy.extensions.telnet :synopsis: Telnet console -.. class:: scrapy.extensions.telnet.TelnetConsole +.. class:: TelnetConsole Provides a telnet console for getting into a Python interpreter inside the currently running Scrapy process, which can be very useful for debugging. @@ -200,7 +200,7 @@ Memory usage extension .. module:: scrapy.extensions.memusage :synopsis: Memory usage extension -.. class:: scrapy.extensions.memusage.MemoryUsage +.. class:: MemoryUsage .. note:: This extension does not work in Windows. @@ -229,7 +229,7 @@ Memory debugger extension .. module:: scrapy.extensions.memdebug :synopsis: Memory debugger extension -.. class:: scrapy.extensions.memdebug.MemoryDebugger +.. class:: MemoryDebugger An extension for debugging memory usage. It collects information about: @@ -245,7 +245,7 @@ Close spider extension .. module:: scrapy.extensions.closespider :synopsis: Close spider extension -.. class:: scrapy.extensions.closespider.CloseSpider +.. class:: CloseSpider Closes a spider automatically when some conditions are met, using a specific closing reason for each condition. @@ -316,7 +316,7 @@ StatsMailer extension .. module:: scrapy.extensions.statsmailer :synopsis: StatsMailer extension -.. class:: scrapy.extensions.statsmailer.StatsMailer +.. class:: StatsMailer This simple extension can be used to send a notification e-mail every time a domain has finished scraping, including the Scrapy stats collected. The email @@ -332,7 +332,7 @@ Debugging extensions Stack trace dump extension ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. class:: scrapy.extensions.debug.StackTraceDump +.. class:: StackTraceDump Dumps information about the running process when a `SIGQUIT`_ or `SIGUSR2`_ signal is received. The information dumped is the following: @@ -361,7 +361,7 @@ There are at least two ways to send Scrapy the `SIGQUIT`_ signal: Debugger extension ~~~~~~~~~~~~~~~~~~ -.. class:: scrapy.extensions.debug.Debugger +.. class:: Debugger Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_ signal is received. After the debugger is exited, the Scrapy process continues From 18c7f3dbe2d225166b5dbd54a3872abfc91e77b0 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 21 Aug 2015 16:29:27 +0200 Subject: [PATCH 09/55] Document built-in add-ons --- docs/topics/addons.rst | 123 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 39ef286eb..4dab15a2a 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -385,3 +385,126 @@ Forward to other add-ons depending on Python version:: _addon = 'path.to.addon' else: _addon = 'path.to.other.addon' + + +Built-in add-on reference +========================= + +Scrapy comes with gateway add-ons that you can use to configure the built-in +middlewares and extensions. For example, to activate and configure the +:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`, instead +of placing this in your ``settings.py``:: + + HTTPCACHE_ENABLED = True + HTTPCACHE_EXPIRATION_SECS = 60 + HTTPCACHE_IGNORE_HTTP_CODES = [404] + +you can also use the add-on framework:: + + INSTALLED_ADDONS = ( + # ..., + 'httpcache', + ) + + HTTPCACHE = { + 'expiration_secs': 60, + 'ignore_http_codes': [404], + } + +Note that you *must* enable built-in addons by placing them in your +``INSTALLED_ADDONS`` setting before you can use them for configuring built-in +components. I.e., configuring the ``HTTPCACHE`` setting will have no effect +when ``httpcache`` is not listed in ``INSTALLED_ADDONS``. + +In general, the add-on names match the lowercase name of the component, with its +type suffix removed (i.e. the add-on configuring the +:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` is called +``httpcache``), and the configuration option names match the names of the +settings they map to, with the component prefix removed (i.e. +``expiration_secs`` maps to :setting:`HTTPCACHE_EXPIRATION_SECS`, as above). +The available add-ons are: + + ++--------------------------------------+--------------------------------------+ +| Add-on | Notes | ++======================================+======================================+ +| **Spider middlewares** | ++--------------------------------------+--------------------------------------+ +| depth (:class:`~scrapy.spidermi\ | | +| ddlewares.depth.DepthMiddleware`) | | ++--------------------------------------+--------------------------------------+ +| httperror (:class:`~scrapy.spid\ | | +| ermiddlewares.httperror.HttpErrorMi\ | | +| ddleware`) | | ++--------------------------------------+--------------------------------------+ +| offsite (:class:`~scrapy.spid\ | | +| ermiddlewares.offsite.OffsiteMiddle\ | | +| ware`) | | ++--------------------------------------+--------------------------------------+ +| referer (:class:`~scrapy.spid\ | | +| ermiddlewares.referer.RefererMiddle\ | | +| ware`) | | ++--------------------------------------+--------------------------------------+ +| urllength (:class:`~scrapy.spid\ | | +| ermiddlewares.urllength.UrlLengthMi\ | | +| ddleware`) | | ++--------------------------------------+--------------------------------------+ +| **Downloader middlewares** | ++--------------------------------------+--------------------------------------+ +| ajaxcrawl (:class:`~scrapy.download\ | | +| ermiddlewares.ajaxcrawl.AjaxCrawlMi\ | | +| ddleware`) | | ++--------------------------------------+--------------------------------------+ +| chunked (:class:`~scrapy.download\ | | +| ermiddlewares.chunked.ChunkedTrans\ | | +| ferMiddleware`) | | ++--------------------------------------+--------------------------------------+ +| cookies (:class:`~scrapy.download\ | | +| ermiddlewares.cookies.CookiesMiddle\ | | +| ware`) | | ++--------------------------------------+--------------------------------------+ +| defaultheaders (:class:`~scrapy.down\| Every configuration entry is treated | +| loadermiddlewares.defaultheaders.Def\| as a default header. | +| aultHeadersMiddleware`) | | ++--------------------------------------+--------------------------------------+ +| **Extensions** | ++--------------------------------------+--------------------------------------+ +| autothrottle | Installing sets | +| (:ref:`topics-autothrottle`) | :setting:`AUTOTHROTTLE_ENABLED` to | +| | ``True``. | ++--------------------------------------+--------------------------------------+ +| corestats (:class:`~scrapy.exten\ | | +| sions.corestats.CoreStats`) | | ++--------------------------------------+--------------------------------------+ +| closespider (:class:`~scrapy.exten\ | | +| sions.closespider.CloseSpider`) | | ++--------------------------------------+--------------------------------------+ +| debugger (:class:`~scrapy.exten\ | | +| sions.debug.Debugger`) | | ++--------------------------------------+--------------------------------------+ +| feedexport (:ref:`topics-feed-expor\ | | +| ts`) | | ++--------------------------------------+--------------------------------------+ +| logstats (:class:`~scrapy.exten\ | | +| sions.logstats.LogStats`) | | ++--------------------------------------+--------------------------------------+ +| memdebug (:class:`~scrapy.exten\ | Installing sets | +| sions.memdebug.MemoryDebugger`) | :setting:`MEMDEBUG_ENABLED` to | +| | ``True``. | ++--------------------------------------+--------------------------------------+ +| memusage (:class:`~scrapy.exten\ | Installing sets | +| sions.memusage.MemoryUsage`) | :setting:`MEMUSAGE_ENABLED` to | +| | ``True``. | ++--------------------------------------+--------------------------------------+ +| spiderstate (:class:`~scrapy.exten\ | | +| sions.spiderstate.SpiderState`) | | ++--------------------------------------+--------------------------------------+ +| stacktracedump (:class:`~scrapy.ext\ | | +| ensions.debug.StackTraceDump`) | | ++--------------------------------------+--------------------------------------+ +| statsmailer (:class:`~scrapy.exten\ | | +| sions.statsmailer.StatsMailer`) | | ++--------------------------------------+--------------------------------------+ +| telnetconsole (:ref:`topics-telnet\ | | +| console`) | | ++--------------------------------------+--------------------------------------+ From d18b6a61d7db295c5f4b1800402a064ed1f40059 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Thu, 29 Oct 2015 16:34:52 +0100 Subject: [PATCH 10/55] Add missing AddonManager tests --- tests/test_addons/__init__.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index 84870ec52..ab06023e3 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -14,6 +14,7 @@ from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler from scrapy.interfaces import IAddon from scrapy.settings import BaseSettings +from scrapy.utils.conf import config_from_filepath from . import addons from . import addonmod @@ -244,16 +245,22 @@ class AddonManagerTest(unittest.TestCase): _test_load_method('load_settings', settings) def test_load_cfg(self): + def _check_loaded_addons(manager): + six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) + self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) + six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) + self.assertEqual(manager.configs['GoodAddon']['key'], 'val1') + # XXX: Check module equality, see above + self.assertEqual(manager['AddonModule'].name, addonmod.name) + six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) + self.assertEqual(manager.configs['AddonModule']['key'], 'val2') manager = AddonManager() manager.load_cfg(self.TESTCFGPATH) - six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) - self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) - self.assertEqual(manager.configs['GoodAddon']['key'], 'val1') - # XXX: Check module equality, see above - self.assertEqual(manager['AddonModule'].name, addonmod.name) - six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) - self.assertEqual(manager.configs['AddonModule']['key'], 'val2') + _check_loaded_addons(manager) + manager = AddonManager() + preloaded_cfg = config_from_filepath(self.TESTCFGPATH) + manager.load_cfg(preloaded_cfg) + _check_loaded_addons(manager) def test_enabled_disabled(self): manager = AddonManager() @@ -330,6 +337,11 @@ class AddonManagerTest(unittest.TestCase): self.assertEqual(us_first.call_count, 1) self.assertEqual(us_second.call_count, 2) + # This will become relevant when we let spiders implement the add-on + # interface and should be replaced with a test where + # AddonManager.spidercls = None then. + manager._call_if_exists(None, 'irrelevant') + def test_update_addons_last_minute_add(self): class AddedAddon(addons.GoodAddon): name = 'AddedAddon' From 9f7fcf5582ed61787180df20a62a301cc09fce4a Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 6 Nov 2015 22:18:10 +0100 Subject: [PATCH 11/55] Make update_classpath() util function return non-string objects --- scrapy/utils/deprecate.py | 3 +++ tests/test_utils_deprecate.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 0fc33e0c4..9293b1480 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -1,5 +1,6 @@ """Some helpers for deprecation messages""" +import six import warnings import inspect from scrapy.exceptions import ScrapyDeprecationWarning @@ -149,6 +150,8 @@ DEPRECATION_RULES = [ def update_classpath(path): """Update a deprecated path from an object with its new location""" + if not isinstance(path, six.string_types): + return path for prefix, replacement in DEPRECATION_RULES: if path.startswith(prefix): new_path = path.replace(prefix, replacement, 1) diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 3e7236fb1..7a35c424b 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -279,3 +279,7 @@ class UpdateClassPathTest(unittest.TestCase): output = update_classpath('scrapy.unmatched.Path') self.assertEqual(output, 'scrapy.unmatched.Path') self.assertEqual(len(w), 0) + + def test_returns_nonstring(self): + for notastring in [None, True, [1, 2, 3], object()]: + self.assertEqual(update_classpath(notastring), notastring) From f7ed239fcb47663d881e72ffab4386244b215ce4 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 6 Nov 2015 22:27:01 +0100 Subject: [PATCH 12/55] Replace INSTALLED_ADDONS tuple setting with ADDONS dictionary setting --- scrapy/addons/__init__.py | 15 ++++++++------- scrapy/settings/default_settings.py | 4 ++-- tests/test_addons/__init__.py | 26 ++++++++++++++++++++++---- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index b1d6e14cb..a1a9a388a 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -1,4 +1,4 @@ -from collections import defaultdict, Mapping +from collections import defaultdict, Mapping, OrderedDict from inspect import isclass import six import warnings @@ -9,7 +9,8 @@ from zope.interface.verify import verifyObject from scrapy.exceptions import NotConfigured from scrapy.interfaces import IAddon -from scrapy.utils.conf import config_from_filepath, get_config +from scrapy.utils.conf import (build_component_list, config_from_filepath, + get_config) from scrapy.utils.misc import load_module_or_object from scrapy.utils.project import get_project_path @@ -166,7 +167,7 @@ class AddonManager(Mapping): """ def __init__(self): - self._addons = {} + self._addons = OrderedDict() self.configs = {} self._disable_on_add = [] @@ -310,15 +311,15 @@ class AddonManager(Mapping): """Load add-ons and configurations from settings object. This will invoke :meth:`get_addon` for every add-on path in the - ``INSTALLED_ADDONS`` setting. For each of these add-ons, the - configuration will be read from the dictionary setting whose name - matches the uppercase add-on name. + ``ADDONS`` setting. For each of these add-ons, the configuration will be + read from the dictionary setting whose name matches the uppercase add-on + name. :param settings: The :class:`~scrapy.settings.Settings` object from \ which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - paths = settings.getlist('INSTALLED_ADDONS') + paths = build_component_list(settings['ADDONS']) addons = [self.get_addon(path) for path in paths] configs = [settings.getdict(addon.name.upper()) for addon in addons] for a, c in zip(addons, configs): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a230750fb..6068a1393 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -18,6 +18,8 @@ import sys from importlib import import_module from os.path import join, abspath, dirname +ADDONS = {} + AJAXCRAWL_ENABLED = False AUTOTHROTTLE_ENABLED = False @@ -167,8 +169,6 @@ HTTPCACHE_DBM_MODULE = 'anydbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False -INSTALLED_ADDONS = () - ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index ab06023e3..aa8dfbb63 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -1,3 +1,5 @@ +from collections import OrderedDict +import itertools import os.path import six from tests import mock @@ -236,14 +238,30 @@ class AddonManagerTest(unittest.TestCase): _test_load_method('load_dict', addonsdict) settings = BaseSettings() - settings.set('INSTALLED_ADDONS', [ - self.ADDONMODPATH, - 'tests.test_addons.addons.GoodAddon', - ]) + settings.set('ADDONS', {self.ADDONMODPATH: 0, + 'tests.test_addons.addons.GoodAddon': 0}) settings.set('ADDONMODULE', {'key': 'val1'}) settings.set('GOODADDON', {'key': 'val2'}) _test_load_method('load_settings', settings) + def test_load_dict_load_settings_order(self): + def _test_load_method(expected_order, func, *args, **kwargs): + manager = AddonManager() + getattr(manager, func)(*args, **kwargs) + self.assertEqual(list(manager.keys()), expected_order) + + # Get three addons named 0, 1, 2 + addonlist = [addons.GoodAddon(str(x)) for x in range(3)] + # Test both methods for every possible mutation + for ordered_addons in itertools.permutations(addonlist): + expected_order = [a.name for a in ordered_addons] + addonsdict = OrderedDict((a, {}) for a in ordered_addons) + _test_load_method(expected_order, 'load_dict', addonsdict) + settings = BaseSettings({ + 'ADDONS': {a: i for i, a in enumerate(ordered_addons)} + }) + _test_load_method(expected_order, 'load_settings', settings) + def test_load_cfg(self): def _check_loaded_addons(manager): six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) From b10caf91a14dcf0d0d4fe89865a53e2542588772 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Fri, 6 Nov 2015 23:14:42 +0100 Subject: [PATCH 13/55] Drop support for add-on configuration in scrapy.cfg --- scrapy/addons/__init__.py | 28 +--------------------------- scrapy/cmdline.py | 1 - tests/test_addons/__init__.py | 20 -------------------- tests/test_addons/cfg.cfg | 5 ----- 4 files changed, 1 insertion(+), 53 deletions(-) delete mode 100644 tests/test_addons/cfg.cfg diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index a1a9a388a..40a98676d 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -9,8 +9,7 @@ from zope.interface.verify import verifyObject from scrapy.exceptions import NotConfigured from scrapy.interfaces import IAddon -from scrapy.utils.conf import (build_component_list, config_from_filepath, - get_config) +from scrapy.utils.conf import build_component_list from scrapy.utils.misc import load_module_or_object from scrapy.utils.project import get_project_path @@ -325,31 +324,6 @@ class AddonManager(Mapping): for a, c in zip(addons, configs): self.add(a, c) - def load_cfg(self, cfg=None): - """Load add-ons and configurations from given ``ConfigParser`` object or - config file path. - - Each add-on should have its own section, where the section has a name in - the form ``addon:my_addon_path``. The add-on object is searched for via - the :meth:`get_addon` method, ``my_addon_path`` can be either a Python - or a file path. - - If ``cfg`` is ``None``, ``scrapy.cfg`` will be used. - - :param cfg: ``ConfigParser`` object or config file path from which to \ - read add-on configuration - :type cfg: ``ConfigParser`` or ``str`` - """ - if cfg is None: - cfg = get_config() - elif isinstance(cfg, six.string_types): - cfg = config_from_filepath(cfg) - for secname in cfg.sections(): - if secname.startswith("addon:"): - addonkey = secname.split("addon:", 1)[1] - addoncfg = dict(cfg.items(secname)) - self.add(addonkey, addoncfg) - def check_dependency_clashes(self): """Check for incompatibilities in add-on dependencies. diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index b403df570..b7c349793 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -120,7 +120,6 @@ def execute(argv=None, settings=None): # ------------------------------------------------------------------ addons = AddonManager() - addons.load_cfg() inproject = inside_project() cmds = _get_commands_dict(settings, inproject) diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index aa8dfbb63..f10819685 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -16,7 +16,6 @@ from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler from scrapy.interfaces import IAddon from scrapy.settings import BaseSettings -from scrapy.utils.conf import config_from_filepath from . import addons from . import addonmod @@ -107,7 +106,6 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): - TESTCFGPATH = os.path.join(os.path.dirname(__file__), 'cfg.cfg') ADDONMODPATH = os.path.join(os.path.dirname(__file__), 'addonmod.py') def setUp(self): @@ -262,24 +260,6 @@ class AddonManagerTest(unittest.TestCase): }) _test_load_method(expected_order, 'load_settings', settings) - def test_load_cfg(self): - def _check_loaded_addons(manager): - six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) - self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) - self.assertEqual(manager.configs['GoodAddon']['key'], 'val1') - # XXX: Check module equality, see above - self.assertEqual(manager['AddonModule'].name, addonmod.name) - six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) - self.assertEqual(manager.configs['AddonModule']['key'], 'val2') - manager = AddonManager() - manager.load_cfg(self.TESTCFGPATH) - _check_loaded_addons(manager) - manager = AddonManager() - preloaded_cfg = config_from_filepath(self.TESTCFGPATH) - manager.load_cfg(preloaded_cfg) - _check_loaded_addons(manager) - def test_enabled_disabled(self): manager = AddonManager() manager.add(addons.GoodAddon('FirstAddon')) diff --git a/tests/test_addons/cfg.cfg b/tests/test_addons/cfg.cfg deleted file mode 100644 index 98c4f0f25..000000000 --- a/tests/test_addons/cfg.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[addon:tests.test_addons.addons.GoodAddon] -key = val1 - -[addon:tests/test_addons/addonmod.py] -key = val2 From 8e5d067af1e50a32f8930294199798535357edf2 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Sat, 7 Nov 2015 19:56:47 +0100 Subject: [PATCH 14/55] Drop BaseSettings.get_addon() prefix magic --- scrapy/addons/__init__.py | 27 +++---------------- tests/test_addons/__init__.py | 27 ------------------- tests/test_addons/project/__init__.py | 0 tests/test_addons/project/addons/__init__.py | 0 tests/test_addons/project/addons/addonmod.py | 7 ----- tests/test_addons/project/addons/addonmod2.py | 7 ----- tests/test_addons/scrapy_addons/__init__.py | 0 tests/test_addons/scrapy_addons/addonmod.py | 7 ----- tests/test_addons/scrapy_addons/addonmod2.py | 7 ----- tests/test_addons/scrapy_addons/addonmod3.py | 7 ----- 10 files changed, 4 insertions(+), 85 deletions(-) delete mode 100644 tests/test_addons/project/__init__.py delete mode 100644 tests/test_addons/project/addons/__init__.py delete mode 100644 tests/test_addons/project/addons/addonmod.py delete mode 100644 tests/test_addons/project/addons/addonmod2.py delete mode 100644 tests/test_addons/scrapy_addons/__init__.py delete mode 100644 tests/test_addons/scrapy_addons/addonmod.py delete mode 100644 tests/test_addons/scrapy_addons/addonmod2.py delete mode 100644 tests/test_addons/scrapy_addons/addonmod3.py diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 40a98676d..ddd18c7fb 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -7,11 +7,9 @@ from pkg_resources import WorkingSet, Distribution, Requirement import zope.interface from zope.interface.verify import verifyObject -from scrapy.exceptions import NotConfigured from scrapy.interfaces import IAddon from scrapy.utils.conf import build_component_list from scrapy.utils.misc import load_module_or_object -from scrapy.utils.project import get_project_path @zope.interface.implementer(IAddon) @@ -244,14 +242,8 @@ class AddonManager(Mapping): """Get an add-on object by its Python or file path. ``path`` is assumed to be either a Python or a file path of a Scrapy - add-on. If no object is found at ``path``, it is tried again first with - ``projectname.addons`` prepended (pointing to the current project's - ``addons`` folder), then with ``scrapy.addons`` prepended (poiting to - Scrapy's built-in add-ons). These convenience shortcuts will only work - with Python paths, not file paths. - - If the object or module pointed to by ``path`` has an attribute named - ``_addon`` that attribute will be assumed to be the add-on. + add-on. If the object or module pointed to by ``path`` has an attribute + named ``_addon`` that attribute will be assumed to be the add-on. :meth:`get_addon` will keep following ``_addon`` attributes until it finds an object that does not have an attribute named ``_addon``. @@ -259,20 +251,9 @@ class AddonManager(Mapping): :type path: ``str`` """ if isinstance(path, six.string_types): - prefixes = ['', 'scrapy.addons.'] try: - prefixes.insert(1, get_project_path() + '.addons.') - except NotConfigured: - warnings.warn("Unable to locate project Python path") - for prefix in prefixes: - fullpath = prefix + path - try: - obj = load_module_or_object(fullpath) - except NameError: - pass - else: - break - else: + obj = load_module_or_object(path) + except NameError: raise NameError("Could not find add-on '%s'" % path) else: obj = path diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index f10819685..4f1074221 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -11,7 +11,6 @@ import zope.interface from zope.interface.verify import verifyObject from zope.interface.exceptions import BrokenImplementation -import scrapy.addons from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler from scrapy.interfaces import IAddon @@ -187,32 +186,6 @@ class AddonManagerTest(unittest.TestCase): x._addon._addon = addons.GoodAddon('inner') self.assertIs(self.manager.get_addon(x), x._addon._addon) - @mock.patch.object(scrapy.addons, 'get_project_path', - return_value='tests.test_addons.project') - def test_get_addon_prefixes(self, get_project_path_mock): - # From python path - self.assertEqual(self.manager.get_addon('addonmod').FROM, - 'test_addons.addonmod') - - # From project 'addons' folder - self.assertEqual(self.manager.get_addon('addonmod2').FROM, - 'test_addons.project.addons.addonmod2') - # Assert prefix priority '' > 'project.addons' - self.assertEqual(self.manager.get_addon('addonmod').FROM, - 'test_addons.addonmod') - - # From scrapy's 'addons' - from . import scrapy_addons - with mock.patch.dict('sys.modules', {'scrapy.addons': scrapy_addons}): - self.assertEqual(self.manager.get_addon('addonmod3').FROM, - 'test_addons.scrapy_addons.addonmod3') - # Assert prefix priority 'project.addons' > 'scrapy.addons' - self.assertEqual(self.manager.get_addon('addonmod2').FROM, - 'test_addons.project.addons.addonmod2') - # Assert prefix priority '' > 'scrapy.addons.' - self.assertEqual(self.manager.get_addon('addonmod').FROM, - 'test_addons.addonmod') - def test_load_dict_load_settings(self): def _test_load_method(func, *args, **kwargs): manager = AddonManager() diff --git a/tests/test_addons/project/__init__.py b/tests/test_addons/project/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_addons/project/addons/__init__.py b/tests/test_addons/project/addons/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_addons/project/addons/addonmod.py b/tests/test_addons/project/addons/addonmod.py deleted file mode 100644 index 66ca644f8..000000000 --- a/tests/test_addons/project/addons/addonmod.py +++ /dev/null @@ -1,7 +0,0 @@ -import zope.interface - -from scrapy.interfaces import IAddon - -zope.interface.moduleProvides(IAddon) - -FROM = 'test_addons.project.addons.addonmod' diff --git a/tests/test_addons/project/addons/addonmod2.py b/tests/test_addons/project/addons/addonmod2.py deleted file mode 100644 index 0dbdd70ff..000000000 --- a/tests/test_addons/project/addons/addonmod2.py +++ /dev/null @@ -1,7 +0,0 @@ -import zope.interface - -from scrapy.interfaces import IAddon - -zope.interface.moduleProvides(IAddon) - -FROM = 'test_addons.project.addons.addonmod2' diff --git a/tests/test_addons/scrapy_addons/__init__.py b/tests/test_addons/scrapy_addons/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/test_addons/scrapy_addons/addonmod.py b/tests/test_addons/scrapy_addons/addonmod.py deleted file mode 100644 index fa479aa68..000000000 --- a/tests/test_addons/scrapy_addons/addonmod.py +++ /dev/null @@ -1,7 +0,0 @@ -import zope.interface - -from scrapy.interfaces import IAddon - -zope.interface.moduleProvides(IAddon) - -FROM = 'test_addons.scrapy_addons.addonmod' diff --git a/tests/test_addons/scrapy_addons/addonmod2.py b/tests/test_addons/scrapy_addons/addonmod2.py deleted file mode 100644 index da053af4a..000000000 --- a/tests/test_addons/scrapy_addons/addonmod2.py +++ /dev/null @@ -1,7 +0,0 @@ -import zope.interface - -from scrapy.interfaces import IAddon - -zope.interface.moduleProvides(IAddon) - -FROM = 'test_addons.scrapy_addons.addonmod2' diff --git a/tests/test_addons/scrapy_addons/addonmod3.py b/tests/test_addons/scrapy_addons/addonmod3.py deleted file mode 100644 index c64521478..000000000 --- a/tests/test_addons/scrapy_addons/addonmod3.py +++ /dev/null @@ -1,7 +0,0 @@ -import zope.interface - -from scrapy.interfaces import IAddon - -zope.interface.moduleProvides(IAddon) - -FROM = 'test_addons.scrapy_addons.addonmod3' From 388c5c4b78509adaf801fb8f66d2d1fdbe1ce9d4 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 9 Nov 2015 00:36:16 +0100 Subject: [PATCH 15/55] Fix component exporting for Addon base class --- scrapy/addons/__init__.py | 2 +- tests/test_addons/__init__.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index ddd18c7fb..48de99c1a 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -84,7 +84,7 @@ class Addon(object): # e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100} k = comp v = config.get('order', self.component_order) - settings.set(self.component_type, {k: v}, 'addon') + settings[self.component_type].update({k: v}, 'addon') def export_basics(self, settings): """Export the :attr:`basic_settings` attribute into the settings object. diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index 4f1074221..b135dd04d 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -38,7 +38,9 @@ class AddonTest(unittest.TestCase): verifyObject(IAddon, self.testaddon) def test_export_component(self): - settings = BaseSettings({'ITEM_PIPELINES': {}}, 'default') + settings = BaseSettings({'ITEM_PIPELINES': BaseSettings(), + 'DOWNLOAD_HANDLERS': BaseSettings()}, + 'default') self.testaddon.component_type = None self.testaddon.export_component({}, settings) self.assertEqual(len(settings['ITEM_PIPELINES']), 0) From e924d382380a6bc7285b8a9713ac14588681af3b Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 9 Nov 2015 01:09:23 +0100 Subject: [PATCH 16/55] Drop component configuration (copied from default_settings) from built-in add-ons --- scrapy/addons/builtins.py | 244 +++++------------------------ tests/test_addons/test_builtins.py | 21 +-- 2 files changed, 37 insertions(+), 228 deletions(-) diff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.py index ff7902afb..9babdeb6f 100644 --- a/scrapy/addons/builtins.py +++ b/scrapy/addons/builtins.py @@ -13,17 +13,14 @@ __all__ = ['make_builtin_addon', 'autothrottle', 'corestats', 'closespider', 'debugger', 'feedexport', 'logstats', 'memdebug', 'memusage', 'spiderstate', 'stacktracedump', 'statsmailer', 'telnetconsole', - ] + ] -def make_builtin_addon(addon_name, comp_type, comp, order=0, - addon_default_config=None, addon_version=None): +def make_builtin_addon(addon_name, addon_default_config=None, + addon_version=None): class ThisAddon(Addon): name = addon_name version = addon_version or scrapy.__version__ - component_type = comp_type - component = comp - component_order = order default_config = addon_default_config or {} return ThisAddon @@ -36,73 +33,26 @@ def make_builtin_addon(addon_name, comp_type, comp, order=0, # SPIDER MIDDLEWARES -depth = make_builtin_addon( - 'depth', - 'SPIDER_MIDDLEWARES', - 'scrapy.spidermiddlewares.depth.DepthMiddleware', - 900, -) +depth = make_builtin_addon('depth') -httperror = make_builtin_addon( - 'httperror', - 'SPIDER_MIDDLEWARES', - 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware', - 50, -) +httperror = make_builtin_addon('httperror') -offsite = make_builtin_addon( - 'offsite', - 'SPIDER_MIDDLEWARES', - 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware', - 500, -) +offsite = make_builtin_addon('offsite') -referer = make_builtin_addon( - 'referer', - 'SPIDER_MIDDLEWARES', - 'scrapy.spidermiddlewares.referer.RefererMiddleware', - 700, - {'enabled': True}, -) +referer = make_builtin_addon('referer') -urllength = make_builtin_addon( - 'urllength', - 'SPIDER_MIDDLEWARES', - 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware', - 800, -) +urllength = make_builtin_addon('urllength') # DOWNLOADER MIDDLEWARES -ajaxcrawl = make_builtin_addon( - 'ajaxcrawl', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware', - 560, -) +ajaxcrawl = make_builtin_addon('ajaxcrawl', {'enabled': True}) -chunked = make_builtin_addon( - 'chunked', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware', - 830, -) +chunked = make_builtin_addon('chunked') -cookies = make_builtin_addon( - 'cookies', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware', - 700, - {'enabled': True}, -) +cookies = make_builtin_addon('cookies') -defaultheaders = make_builtin_addon( - 'defaultheaders', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware', - 550, -) +defaultheaders = make_builtin_addon('defaultheaders') # Assume every config entry is a header def defaultheaders_export_config(self, config, settings): conf = self.default_config or {} @@ -110,92 +60,31 @@ def defaultheaders_export_config(self, config, settings): settings.set('DEFAULT_REQUEST_HEADERS', conf, 'addon') defaultheaders.export_config = defaultheaders_export_config -downloadtimeout = make_builtin_addon( - 'downloadtimeout', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware', - 350, -) +downloadtimeout = make_builtin_addon('downloadtimeout') downloadtimeout.config_mapping = {'timeout': 'DOWNLOAD_TIMEOUT', 'download_timeout': 'DOWNLOAD_TIMEOUT'} -httpauth = make_builtin_addon( - 'httpauth', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware', - 300, -) +httpauth = make_builtin_addon('httpauth') -httpcache = make_builtin_addon( - 'httpcache', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', - 900, - {'enabled': True}, -) +httpcache = make_builtin_addon('httpcache', {'enabled': True}) -httpcompression = make_builtin_addon( - 'httpcompression', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware', - 590, - {'enabled': True}, -) +httpcompression = make_builtin_addon('httpcompression') httpcompression.config_mapping = {'enabled': 'COMPRESSION_ENABLED'} -httpproxy = make_builtin_addon( - 'httpproxy', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware', - 750, -) +httpproxy = make_builtin_addon('httpproxy') -metarefresh = make_builtin_addon( - 'metarefresh', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware', - 580, - {'enabled': True}, -) +metarefresh = make_builtin_addon('metarefresh') metarefresh.config_mapping = {'max_times': 'REDIRECT_MAX_TIMES'} -redirect = make_builtin_addon( - 'redirect', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware', - 600, - {'enabled': True}, -) +redirect = make_builtin_addon('redirect') -retry = make_builtin_addon( - 'retry', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.retry.RetryMiddleware', - 500, - {'enabled': True}, -) +retry = make_builtin_addon('retry') -robotstxt = make_builtin_addon( - 'robotstxt', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware', - 100, - {'obey': True}, -) +robotstxt = make_builtin_addon('robotstxt', {'obey': True}) -stats = make_builtin_addon( - 'stats', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.stats.DownloaderStats', - 850, -) +stats = make_builtin_addon('stats') -useragent = make_builtin_addon( - 'useragent', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware', - 400, -) +useragent = make_builtin_addon('useragent') useragent.config_mapping = {'user_agent': 'USER_AGENT'} @@ -204,90 +93,27 @@ useragent.config_mapping = {'user_agent': 'USER_AGENT'} # EXTENSIONS -autothrottle = make_builtin_addon( - 'throttle', - 'EXTENSIONS', - 'scrapy.extensions.throttle.AutoThrottle', - 0, - {'enabled': True}, -) +autothrottle = make_builtin_addon('autothrottle', {'enabled': True}) -corestats = make_builtin_addon( - 'corestats', - 'EXTENSIONS' - 'scrapy.extensions.corestats.CoreStats', - 0, -) +corestats = make_builtin_addon('corestats') -closespider = make_builtin_addon( - 'closespider', - 'EXTENSIONS' - 'scrapy.extensions.closespider.CloseSpider', - 0, -) +closespider = make_builtin_addon('closespider') -debugger = make_builtin_addon( - 'debugger', - 'EXTENSIONS' - 'scrapy.extensions.debug.Debugger', - 0, -) +debugger = make_builtin_addon('debugger') -feedexport = make_builtin_addon( - 'feedexport', - 'EXTENSIONS' - 'scrapy.extensions.feedexport.FeedExporter', - 0, -) +feedexport = make_builtin_addon('feedexport') feedexport.settings_prefix = 'FEED' -logstats = make_builtin_addon( - 'logstats', - 'EXTENSIONS' - 'scrapy.extensions.logstats.LogStats', - 0, -) +logstats = make_builtin_addon('logstats') -memdebug = make_builtin_addon( - 'memdebug', - 'EXTENSIONS' - 'scrapy.extensions.memdebug.MemoryDebugger', - 0, - {'enabled': True}, -) +memdebug = make_builtin_addon('memdebug', {'enabled': True}) -memusage = make_builtin_addon( - 'memusage', - 'EXTENSIONS' - 'scrapy.extensions.memusage.MemoryUsage', - 0, - {'enabled': True}, -) +memusage = make_builtin_addon('memusage', {'enabled': True}) -spiderstate = make_builtin_addon( - 'spiderstate', - 'EXTENSIONS' - 'scrapy.extensions.spiderstate.SpiderState', - 0, -) +spiderstate = make_builtin_addon('spiderstate') -stacktracedump = make_builtin_addon( - 'stacktracedump', - 'EXTENSIONS' - 'scrapy.extensions.debug.StackTraceDump', - 0, -) +stacktracedump = make_builtin_addon('stacktracedump') -statsmailer = make_builtin_addon( - 'statsmailer', - 'EXTENSIONS' - 'scrapy.extensions.statsmailer.StatsMailer', - 0, -) +statsmailer = make_builtin_addon('statsmailer') -telnetconsole = make_builtin_addon( - 'telnetconsole', - 'EXTENSIONS' - 'scrapy.telnet.TelnetConsole', - 0, -) +telnetconsole = make_builtin_addon('telnetconsole') diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.py index 607c911fb..c89876950 100644 --- a/tests/test_addons/test_builtins.py +++ b/tests/test_addons/test_builtins.py @@ -9,28 +9,11 @@ from scrapy.settings import Settings class BuiltinAddonsTest(unittest.TestCase): def test_make_builtin_addon(self): - httpcache = make_builtin_addon( - 'httpcache', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', - 900, - {'enabled': True}, - ) + httpcache = make_builtin_addon('httpcache', {'enabled': True}) self.assertEqual(httpcache.name, 'httpcache') - self.assertEqual(httpcache.component_type, 'DOWNLOADER_MIDDLEWARES') - self.assertEqual(httpcache.component, 'scrapy.downloadermiddlewares.' - 'httpcache.HttpCacheMiddleware') - self.assertEqual(httpcache.component_order, 900) self.assertEqual(httpcache.default_config, {'enabled': True}) self.assertEqual(httpcache.version, scrapy.__version__) - httpcache = make_builtin_addon( - 'httpcache', - 'DOWNLOADER_MIDDLEWARES', - 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware', - 900, - {'enabled': True}, - '99.9', - ) + httpcache = make_builtin_addon('httpcache', {'enabled': True}, '99.9') self.assertEqual(httpcache.version, '99.9') def test_defaultheaders_export_config(self): From b7b00fb95669196167a5e5f76b1f3b8e2feb7b84 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 9 Nov 2015 01:18:41 +0100 Subject: [PATCH 17/55] PEP8ify add-ons and tests --- scrapy/addons/__init__.py | 6 ++++-- tests/test_addons/__init__.py | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 48de99c1a..169c79eac 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -323,6 +323,7 @@ class AddonManager(Mapping): """ # Collect all active add-ons and the components they provide ws = WorkingSet('') + def add_dist(project_name, version, **kwargs): if project_name in ws.entry_keys.get('scrapy', []): raise ImportError("Component {} provided by multiple add-ons" @@ -354,8 +355,9 @@ class AddonManager(Mapping): # our own exception or is it helpful enough? if ws.find(req) is None: raise ImportError( - "Add-ons {} require or modify missing component {}" - "".format(required[reqstr]+modified[reqstr], reqstr)) + "Add-ons {} require or modify missing component {}" + "".format(required[reqstr]+modified[reqstr], reqstr) + ) mod_and_req = set(required.keys()).intersection(modified.keys()) for conflict in mod_and_req: diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index b135dd04d..32ee25ad8 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -24,6 +24,7 @@ class AddonTest(unittest.TestCase): def setUp(self): self.rawaddon = Addon() + class AddonWithAttributes(Addon): name = 'Test' version = '1.0' @@ -132,7 +133,7 @@ class AddonManagerTest(unittest.TestCase): def test_add_verifies(self): brokenaddon = self.manager.get_addon( - 'tests.test_addons.addons.BrokenAddon') + 'tests.test_addons.addons.BrokenAddon') self.assertRaises(zope.interface.exceptions.BrokenImplementation, self.manager.add, brokenaddon) @@ -145,11 +146,13 @@ class AddonManagerTest(unittest.TestCase): def test_remove(self): manager = AddonManager() + def test_gets_removed(removearg): manager.add(addonmod) self.assertIn('AddonModule', manager) manager.remove(removearg) self.assertNotIn('AddonModule', manager) + test_gets_removed('AddonModule') test_gets_removed(addonmod) test_gets_removed('tests.test_addons.addonmod') @@ -158,8 +161,7 @@ class AddonManagerTest(unittest.TestCase): self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) def test_get_addon(self): - goodaddon = self.manager.get_addon( - 'tests.test_addons.addons.GoodAddon') + goodaddon = self.manager.get_addon('tests.test_addons.addons.GoodAddon') self.assertIs(goodaddon, addons.GoodAddon) loaded_addonmod = self.manager.get_addon(self.ADDONMODPATH) @@ -194,8 +196,7 @@ class AddonManagerTest(unittest.TestCase): getattr(manager, func)(*args, **kwargs) six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['GoodAddon'], - ['key']) + six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) self.assertEqual(manager.configs['GoodAddon']['key'], 'val2') # XXX: Check module equality, see above self.assertEqual(manager['AddonModule'].name, addonmod.name) @@ -284,17 +285,17 @@ class AddonManagerTest(unittest.TestCase): ua_first.assert_called_once_with(manager.configs['FirstAddon'], manager) ua_second.assert_called_once_with(manager.configs['SecondAddon'], - manager) + manager) manager.update_settings(settings) us_first.assert_called_once_with(manager.configs['FirstAddon'], settings) us_second.assert_called_once_with(manager.configs['SecondAddon'], - settings) + settings) manager.check_configuration(crawler) cc_first.assert_called_once_with(manager.configs['FirstAddon'], crawler) cc_second.assert_called_once_with(manager.configs['SecondAddon'], - crawler) + crawler) self.assertEqual(ua_first.call_count, 1) self.assertEqual(ua_second.call_count, 1) self.assertEqual(us_first.call_count, 1) @@ -321,6 +322,7 @@ class AddonManagerTest(unittest.TestCase): class FirstAddon(addons.GoodAddon): name = 'FirstAddon' + def update_addons(self, config, addons): addons.add(AddedAddon()) From 33dfb3e167f7d31e2573ab39c2b80bf7728c2b00 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Mon, 9 Nov 2015 11:22:24 +0100 Subject: [PATCH 18/55] Remove unused project path util function --- scrapy/utils/project.py | 12 ------------ tests/test_utils_project.py | 27 --------------------------- 2 files changed, 39 deletions(-) delete mode 100644 tests/test_utils_project.py diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index a1266c879..a15a0d90f 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -71,15 +71,3 @@ def get_project_settings(): settings.setdict(env_overrides, priority='project') return settings - -def get_project_path(): - """Return the Python path of the current project. - - This fails when the settings module does not live in the project's root. - """ - if not inside_project(): - raise NotConfigured("Not inside a project") - settings_module_path = os.environ.get(ENVVAR) - if not settings_module_path: - raise NotConfigured("Unable to locate project's python path") - return settings_module_path.rsplit('.', 1)[0] diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py deleted file mode 100644 index cea4d9950..000000000 --- a/tests/test_utils_project.py +++ /dev/null @@ -1,27 +0,0 @@ -import os -from tests import mock -import unittest - -from scrapy.exceptions import NotConfigured -from scrapy.utils.project import get_project_path, inside_project - - -class UtilsProjectTestCase(unittest.TestCase): - - @mock.patch('scrapy.utils.project.inside_project', return_value=True) - def test_get_project_path(self, mock_ip): - def _test(settingsmod, expected): - with mock.patch.dict('os.environ', - {'SCRAPY_SETTINGS_MODULE': settingsmod}): - self.assertEqual(get_project_path(), expected) - _test('project.settings', 'project') - _test('project.othername', 'project') - _test('nested.project.settings', 'nested.project') - - with mock.patch.dict('os.environ', {}, clear=True): - self.assertRaises(NotConfigured, get_project_path) - - mock_ip.return_value = False - with mock.patch.dict('os.environ', - {'SCRAPY_SETTINGS_MODULE': 'some.settings'}): - self.assertRaises(NotConfigured, get_project_path) From a65fc0db7d7d728bc8abbb37eff358870ce7a9b5 Mon Sep 17 00:00:00 2001 From: Jakob de Maeyer Date: Thu, 12 Nov 2015 18:37:26 +0100 Subject: [PATCH 19/55] Drop support for providing file paths as add-ons --- scrapy/utils/misc.py | 24 --------------------- tests/test_addons/__init__.py | 26 ++++++----------------- tests/test_utils_misc/__init__.py | 19 +---------------- tests/test_utils_misc/testpkg/__init__.py | 1 - tests/test_utils_misc/testpkg/submod.py | 1 - 5 files changed, 8 insertions(+), 63 deletions(-) delete mode 100644 tests/test_utils_misc/testpkg/__init__.py delete mode 100644 tests/test_utils_misc/testpkg/submod.py diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 9461d93e9..e55edd63e 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,8 +1,5 @@ """Helper functions which doesn't fit anywhere else""" -import itertools -import os.path import re -import sys import hashlib from importlib import import_module from pkgutil import iter_modules @@ -72,10 +69,6 @@ def load_module_or_object(path): return load_object(path) except (ValueError, NameError, ImportError): pass - try: - return get_module_from_filepath(path) - except ImportError: - pass raise NameError("Could not load '%s'" % path) @@ -101,23 +94,6 @@ def walk_modules(path): return mods -def get_module_from_filepath(path): - """Load and return a python module/package from a file path""" - path = path.rstrip("/") - if path.endswith('.py'): - path = path.rsplit('.py', 1)[0] - basefolder, modname = os.path.split(path) - # XXX: There are other ways to import modules from a full path which don't - # need to modify PYTHONPATH, see - # https://stackoverflow.com/questions/67631/ - # These methods differ between py2 and py3, and apparently the - # py3 method was deprecated in Python 3.4 - sys.path.insert(0, basefolder) - mod = import_module(modname) - sys.path.pop(0) - return mod - - def extract_regex(regex, text, encoding='utf-8'): """Extract a list of unicode strings from the given text/encoding using the following policies: diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index 32ee25ad8..a4e278fa5 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -108,8 +108,6 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): - ADDONMODPATH = os.path.join(os.path.dirname(__file__), 'addonmod.py') - def setUp(self): self.manager = AddonManager() @@ -156,7 +154,6 @@ class AddonManagerTest(unittest.TestCase): test_gets_removed('AddonModule') test_gets_removed(addonmod) test_gets_removed('tests.test_addons.addonmod') - test_gets_removed(self.ADDONMODPATH) self.assertRaises(KeyError, manager.remove, 'nonexistent') self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) @@ -164,18 +161,12 @@ class AddonManagerTest(unittest.TestCase): goodaddon = self.manager.get_addon('tests.test_addons.addons.GoodAddon') self.assertIs(goodaddon, addons.GoodAddon) - loaded_addonmod = self.manager.get_addon(self.ADDONMODPATH) - # XXX: The module is in fact imported twice under different names into - # sys.modules, is there a good assertion for module equality? - self.assertEqual(loaded_addonmod.name, addonmod.name) + loaded_addonmod = self.manager.get_addon("tests.test_addons.addonmod") + self.assertIs(loaded_addonmod, addonmod) - # Does not provide interface, but has _addon attribute pointing to - # GoodAddon instance addonspath = os.path.join(os.path.dirname(__file__), 'addons.py') - goodaddon = self.manager.get_addon(addonspath) - # XXX: Again, the imported class and addons.GoodAddon are different - # since they are imported twice. How to use isInstance? - self.assertEqual(goodaddon.name, addons.GoodAddon.name) + goodaddon = self.manager.get_addon("tests.test_addons.addons") + self.assertIsInstance(goodaddon, addons.GoodAddon) self.assertRaises(NameError, self.manager.get_addon, 'xy.n_onexistent') @@ -198,21 +189,18 @@ class AddonManagerTest(unittest.TestCase): self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) self.assertEqual(manager.configs['GoodAddon']['key'], 'val2') - # XXX: Check module equality, see above - self.assertEqual(manager['AddonModule'].name, addonmod.name) + self.assertEqual(manager['AddonModule'], addonmod) self.assertIn('key', manager.configs['AddonModule']) self.assertEqual(manager.configs['AddonModule']['key'], 'val1') addonsdict = { - self.ADDONMODPATH: { - 'key': 'val1', - }, + "tests.test_addons.addonmod": {'key': 'val1'}, 'tests.test_addons.addons.GoodAddon': {'key': 'val2'}, } _test_load_method('load_dict', addonsdict) settings = BaseSettings() - settings.set('ADDONS', {self.ADDONMODPATH: 0, + settings.set('ADDONS', {"tests.test_addons.addonmod": 0, 'tests.test_addons.addons.GoodAddon': 0}) settings.set('ADDONMODULE', {'key': 'val1'}) settings.set('GOODADDON', {'key': 'val2'}) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index f33562b7d..8ea8786d7 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -4,7 +4,7 @@ import unittest from scrapy.item import Item, Field from scrapy.utils.misc import (load_object, load_module_or_object, arg_to_iter, - walk_modules, get_module_from_filepath) + walk_modules) __doctests__ = ['scrapy.utils.misc'] @@ -21,9 +21,6 @@ class UtilsMiscTestCase(unittest.TestCase): def test_load_module_or_object(self): testmod = load_module_or_object(__name__ + '.testmod') self.assertTrue(hasattr(testmod, 'TESTVAR')) - testmod = load_module_or_object( - os.path.join(os.path.dirname(__file__), 'testmod.py')) - self.assertTrue(hasattr(testmod, 'TESTVAR')) obj = load_object('scrapy.utils.misc.load_object') self.assertIs(obj, load_object) @@ -67,20 +64,6 @@ class UtilsMiscTestCase(unittest.TestCase): finally: sys.path.remove(egg) - def test_get_module_from_filepath(self): - testmodpath = os.path.join(os.path.dirname(__file__), 'testmod.py') - testmod = get_module_from_filepath(testmodpath) - self.assertTrue(hasattr(testmod, 'TESTVAR')) - - testpkgpath = os.path.join(os.path.dirname(__file__), 'testpkg') - testpkg = get_module_from_filepath(testpkgpath) - self.assertTrue(hasattr(testpkg, 'TESTVAR2')) - # Check submodule access - import testpkg.submod - self.assertTrue(hasattr(testpkg.submod, 'TESTVAR3')) - self.assertIs(testpkg.submod.TESTVAR3, - load_object(testpkg.__name__ + ".submod.TESTVAR3")) - def test_arg_to_iter(self): class TestItem(Item): diff --git a/tests/test_utils_misc/testpkg/__init__.py b/tests/test_utils_misc/testpkg/__init__.py deleted file mode 100644 index 12cc2f6d9..000000000 --- a/tests/test_utils_misc/testpkg/__init__.py +++ /dev/null @@ -1 +0,0 @@ -TESTVAR2 = True diff --git a/tests/test_utils_misc/testpkg/submod.py b/tests/test_utils_misc/testpkg/submod.py deleted file mode 100644 index 8a07e3592..000000000 --- a/tests/test_utils_misc/testpkg/submod.py +++ /dev/null @@ -1 +0,0 @@ -TESTVAR3 = True From 075ad6f196e7936edfd75b57d10b7313da7fe4f4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 16:34:18 +0400 Subject: [PATCH 20/55] Test, linter etc. fixes. --- docs/topics/addons.rst | 24 ++- scrapy/addons/__init__.py | 106 +++++----- scrapy/addons/builtins.py | 135 +++++++----- scrapy/utils/misc.py | 2 +- tests/test_addons/__init__.py | 329 ++++++++++++++--------------- tests/test_addons/addonmod.py | 2 + tests/test_addons/addons.py | 11 +- tests/test_addons/test_builtins.py | 18 +- tests/test_crawl.py | 2 +- tests/test_middleware.py | 4 +- 10 files changed, 329 insertions(+), 304 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 4dab15a2a..ba6e839a5 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -350,29 +350,31 @@ Check configuration of fully initialized crawler (see Provide add-on interface through a module: -.. No idea why just using '::' doesn't work for this one .. code-block:: python - name = 'AddonModule' - version = '1.0' + name = "AddonModule" + version = "1.0" + class MyPipeline(object): - # ... + ... + class MyDownloaderMiddleware(object): - # ... + ... + def update_settings(config, settings): settings.set( - 'ITEM_PIPELINES', + "ITEM_PIPELINES", {MyPipeline(): 200}, - priority='addon', - } + priority="addon", + ) settings.set( - 'DOWNLOADER_MIDDLEWARES', + "DOWNLOADER_MIDDLEWARES", {MyDownloaderMiddleware(): 800}, - priority='addon', - } + priority="addon", + ) Forward to other add-ons depending on Python version:: diff --git a/scrapy/addons/__init__.py b/scrapy/addons/__init__.py index 169c79eac..aad9ef789 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons/__init__.py @@ -1,10 +1,10 @@ -from collections import defaultdict, Mapping, OrderedDict -from inspect import isclass -import six import warnings +from collections import OrderedDict, defaultdict +from collections.abc import Mapping +from inspect import isclass -from pkg_resources import WorkingSet, Distribution, Requirement import zope.interface +from pkg_resources import Distribution, Requirement, WorkingSet from zope.interface.verify import verifyObject from scrapy.interfaces import IAddon @@ -14,7 +14,6 @@ from scrapy.utils.misc import load_module_or_object @zope.interface.implementer(IAddon) class Addon(object): - basic_settings = None """``dict`` of settings that will be exported via :meth:`export_basics`.""" @@ -83,8 +82,8 @@ class Addon(object): else: # e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100} k = comp - v = config.get('order', self.component_order) - settings[self.component_type].update({k: v}, 'addon') + v = config.get("order", self.component_order) + settings[self.component_type].update({k: v}, "addon") def export_basics(self, settings): """Export the :attr:`basic_settings` attribute into the settings object. @@ -95,8 +94,8 @@ class Addon(object): :param settings: Settings object into which to expose the basic settings :type settings: :class:`~scrapy.settings.Settings` """ - for setting, value in six.iteritems(self.basic_settings or {}): - settings.set(setting, value, 'addon') + for setting, value in (self.basic_settings or {}).items(): + settings.set(setting, value, "addon") def export_config(self, config, settings): """Export the add-on configuration, all keys in caps and with @@ -121,14 +120,13 @@ class Addon(object): prefix = self.settings_prefix or self.name # Since default exported config is case-insensitive (everything will be # uppercased), make mapped config case-insensitive as well - conf_mapping = {k.lower(): v - for k, v in six.iteritems(self.config_mapping or {})} - for key, val in six.iteritems(conf): + conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()} + for key, val in conf.items(): if key.lower() in conf_mapping: key = conf_mapping[key.lower()] else: - key = (prefix + '_' + key).upper() - settings.set(key, val, 'addon') + key = (prefix + "_" + key).upper() + settings.set(key, val, "addon") def update_settings(self, config, settings): """Export both the basic settings and the add-on configuration. I.e., @@ -210,11 +208,11 @@ class AddonManager(Mapping): verifyObject(IAddon, addon) name = addon.name if name in self: - raise ValueError("Addon '{}' already loaded".format(name)) + raise ValueError(f"Addon '{name}' already loaded") self._addons[name] = addon self.configs[name] = config or {} if name in self._disable_on_add: - self.configs[name]['_enabled'] = False + self.configs[name]["_enabled"] = False self._disable_on_add.remove(name) def remove(self, addon): @@ -229,7 +227,7 @@ class AddonManager(Mapping): """ if addon in self: del self[addon] - elif hasattr(addon, 'name') and addon.name in self: + elif hasattr(addon, "name") and addon.name in self: del self[addon.name] else: try: @@ -250,14 +248,14 @@ class AddonManager(Mapping): :param path: Python or file path to an add-on :type path: ``str`` """ - if isinstance(path, six.string_types): + if isinstance(path, str): try: obj = load_module_or_object(path) except NameError: - raise NameError("Could not find add-on '%s'" % path) + raise NameError(f"Could not find add-on '{path}'") else: obj = path - if hasattr(obj, '_addon'): + if hasattr(obj, "_addon"): obj = AddonManager.get_addon(obj._addon) return obj @@ -284,7 +282,7 @@ class AddonManager(Mapping): and values correspond to their configuration :type addonsdict: ``dict`` """ - for addonpath, addoncfg in six.iteritems(addonsdict): + for addonpath, addoncfg in addonsdict.items(): self.add(addonpath, addoncfg) def load_settings(self, settings): @@ -299,7 +297,7 @@ class AddonManager(Mapping): which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - paths = build_component_list(settings['ADDONS']) + paths = build_component_list(settings["ADDONS"]) addons = [self.get_addon(path) for path in paths] configs = [settings.getdict(addon.name.upper()) for addon in addons] for a, c in zip(addons, configs): @@ -322,20 +320,23 @@ class AddonManager(Mapping): add-on. """ # Collect all active add-ons and the components they provide - ws = WorkingSet('') + ws = WorkingSet("") def add_dist(project_name, version, **kwargs): - if project_name in ws.entry_keys.get('scrapy', []): - raise ImportError("Component {} provided by multiple add-ons" - "".format(project_name)) + if project_name in ws.entry_keys.get("scrapy", []): + raise ImportError( + f"Component {project_name} provided by multiple add-ons" + ) else: - dist = Distribution(project_name=project_name, version=version, - **kwargs) - ws.add(dist, entry='scrapy') + dist = Distribution( + project_name=project_name, version=version, **kwargs + ) + ws.add(dist, entry="scrapy") + for name in self: ver = self[name].version add_dist(name, ver) - for provides_name in getattr(self[name], 'provides', []): + for provides_name in getattr(self[name], "provides", []): add_dist(provides_name, ver) # Collect all required and modified components @@ -345,8 +346,9 @@ class AddonManager(Mapping): for entry in getattr(self[name], attribute_name, []): attrs[entry].append(name) return attrs - modified = compile_attribute_dict('modifies') - required = compile_attribute_dict('requires') + + modified = compile_attribute_dict("modifies") + required = compile_attribute_dict("requires") req_or_mod = set(required.keys()).union(modified.keys()) for reqstr in req_or_mod: @@ -355,15 +357,16 @@ class AddonManager(Mapping): # our own exception or is it helpful enough? if ws.find(req) is None: raise ImportError( - "Add-ons {} require or modify missing component {}" - "".format(required[reqstr]+modified[reqstr], reqstr) + f"Add-ons {required[reqstr] + modified[reqstr]} require" + f" or modify missing component {reqstr}" ) mod_and_req = set(required.keys()).intersection(modified.keys()) for conflict in mod_and_req: - warnings.warn("Component '{}', required by add-ons {}, is modified " - "by add-ons {}".format(conflict, required[conflict], - modified[conflict])) + warnings.warn( + f"Component '{conflict}', required by add-ons {required[conflict]}," + f" is modified by add-ons {modified[conflict]}" + ) def disable(self, addon): """Disable an add-on, i.e. prevent its callbacks from being called. @@ -375,7 +378,7 @@ class AddonManager(Mapping): :type addon: ``str`` """ if addon in self: - self.configs[addon]['_enabled'] = False + self.configs[addon]["_enabled"] = False else: self._disable_on_add.append(addon) @@ -389,23 +392,23 @@ class AddonManager(Mapping): :type addon: ``str`` """ if addon in self: - self.configs[addon]['_enabled'] = True + self.configs[addon]["_enabled"] = True elif addon in self._disable_on_add: self._disable_on_add.remove(addon) else: - raise ValueError("Add-ons need to be added before they can be " - "enabled") + raise ValueError("Add-ons need to be added before they can be " "enabled") @property def disabled(self): """Names of disabled add-ons""" - return ([a for a in self if not self.configs[a].get('_enabled', True)] + - self._disable_on_add) + return [ + a for a in self if not self.configs[a].get("_enabled", True) + ] + self._disable_on_add @property def enabled(self): """Names of enabled add-ons""" - return [a for a in self if self.configs[a].get('_enabled', True)] + return [a for a in self if self.configs[a].get("_enabled", True)] def _call_if_exists(self, obj, cbname, *args, **kwargs): if obj is None: @@ -418,9 +421,10 @@ class AddonManager(Mapping): cb(*args, **kwargs) def _call_addon(self, addonname, cbname, *args, **kwargs): - if self.configs[addonname].get('_enabled', True): - self._call_if_exists(self[addonname], cbname, - self.configs[addonname], *args, **kwargs) + if self.configs[addonname].get("_enabled", True): + self._call_if_exists( + self[addonname], cbname, self.configs[addonname], *args, **kwargs + ) def update_addons(self): """Call ``update_addons()`` of all held add-ons. @@ -432,7 +436,7 @@ class AddonManager(Mapping): while called_addons != set(self): for name in set(self).difference(called_addons): called_addons.add(name) - self._call_addon(name, 'update_addons', self) + self._call_addon(name, "update_addons", self) def update_settings(self, settings): """Call ``update_settings()`` of all held add-ons. @@ -442,7 +446,7 @@ class AddonManager(Mapping): :type settings: :class:`~scrapy.settings.Settings` """ for name in self: - self._call_addon(name, 'update_settings', settings) + self._call_addon(name, "update_settings", settings) def check_configuration(self, crawler): """Call ``check_configuration()`` of all held add-ons. @@ -451,7 +455,7 @@ class AddonManager(Mapping): :type crawler: :class:`~scrapy.crawler.Crawler` """ for name in self: - self._call_addon(name, 'check_configuration', crawler) + self._call_addon(name, "check_configuration", crawler) -from scrapy.addons.builtins import * +from scrapy.addons.builtins import * # noqa diff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.py index 9babdeb6f..ea3afbf99 100644 --- a/scrapy/addons/builtins.py +++ b/scrapy/addons/builtins.py @@ -1,23 +1,44 @@ import scrapy from scrapy.addons import Addon -__all__ = ['make_builtin_addon', - - 'depth', 'httperror', 'offsite', 'referer', 'urllength', - - 'ajaxcrawl', 'chunked', 'cookies', 'defaultheaders', - 'downloadtimeout', 'httpauth', 'httpcache', 'httpcompression', - 'httpproxy', 'metarefresh', 'redirect', 'retry', 'robotstxt', - 'stats', 'useragent', - - 'autothrottle', 'corestats', 'closespider', 'debugger', 'feedexport', - 'logstats', 'memdebug', 'memusage', 'spiderstate', 'stacktracedump', - 'statsmailer', 'telnetconsole', - ] +__all__ = [ + "make_builtin_addon", + "depth", + "httperror", + "offsite", + "referer", + "urllength", + "ajaxcrawl", + "chunked", + "cookies", + "defaultheaders", + "downloadtimeout", + "httpauth", + "httpcache", + "httpcompression", + "httpproxy", + "metarefresh", + "redirect", + "retry", + "robotstxt", + "stats", + "useragent", + "autothrottle", + "corestats", + "closespider", + "debugger", + "feedexport", + "logstats", + "memdebug", + "memusage", + "spiderstate", + "stacktracedump", + "statsmailer", + "telnetconsole", +] -def make_builtin_addon(addon_name, addon_default_config=None, - addon_version=None): +def make_builtin_addon(addon_name, addon_default_config=None, addon_version=None): class ThisAddon(Addon): name = addon_name version = addon_version or scrapy.__version__ @@ -33,59 +54,65 @@ def make_builtin_addon(addon_name, addon_default_config=None, # SPIDER MIDDLEWARES -depth = make_builtin_addon('depth') +depth = make_builtin_addon("depth") -httperror = make_builtin_addon('httperror') +httperror = make_builtin_addon("httperror") -offsite = make_builtin_addon('offsite') +offsite = make_builtin_addon("offsite") -referer = make_builtin_addon('referer') +referer = make_builtin_addon("referer") -urllength = make_builtin_addon('urllength') +urllength = make_builtin_addon("urllength") # DOWNLOADER MIDDLEWARES -ajaxcrawl = make_builtin_addon('ajaxcrawl', {'enabled': True}) +ajaxcrawl = make_builtin_addon("ajaxcrawl", {"enabled": True}) -chunked = make_builtin_addon('chunked') +chunked = make_builtin_addon("chunked") + +cookies = make_builtin_addon("cookies") + +defaultheaders = make_builtin_addon("defaultheaders") -cookies = make_builtin_addon('cookies') -defaultheaders = make_builtin_addon('defaultheaders') # Assume every config entry is a header def defaultheaders_export_config(self, config, settings): conf = self.default_config or {} conf.update(config) - settings.set('DEFAULT_REQUEST_HEADERS', conf, 'addon') + settings.set("DEFAULT_REQUEST_HEADERS", conf, "addon") + + defaultheaders.export_config = defaultheaders_export_config -downloadtimeout = make_builtin_addon('downloadtimeout') -downloadtimeout.config_mapping = {'timeout': 'DOWNLOAD_TIMEOUT', - 'download_timeout': 'DOWNLOAD_TIMEOUT'} +downloadtimeout = make_builtin_addon("downloadtimeout") +downloadtimeout.config_mapping = { + "timeout": "DOWNLOAD_TIMEOUT", + "download_timeout": "DOWNLOAD_TIMEOUT", +} -httpauth = make_builtin_addon('httpauth') +httpauth = make_builtin_addon("httpauth") -httpcache = make_builtin_addon('httpcache', {'enabled': True}) +httpcache = make_builtin_addon("httpcache", {"enabled": True}) -httpcompression = make_builtin_addon('httpcompression') -httpcompression.config_mapping = {'enabled': 'COMPRESSION_ENABLED'} +httpcompression = make_builtin_addon("httpcompression") +httpcompression.config_mapping = {"enabled": "COMPRESSION_ENABLED"} -httpproxy = make_builtin_addon('httpproxy') +httpproxy = make_builtin_addon("httpproxy") -metarefresh = make_builtin_addon('metarefresh') -metarefresh.config_mapping = {'max_times': 'REDIRECT_MAX_TIMES'} +metarefresh = make_builtin_addon("metarefresh") +metarefresh.config_mapping = {"max_times": "REDIRECT_MAX_TIMES"} -redirect = make_builtin_addon('redirect') +redirect = make_builtin_addon("redirect") -retry = make_builtin_addon('retry') +retry = make_builtin_addon("retry") -robotstxt = make_builtin_addon('robotstxt', {'obey': True}) +robotstxt = make_builtin_addon("robotstxt", {"obey": True}) -stats = make_builtin_addon('stats') +stats = make_builtin_addon("stats") -useragent = make_builtin_addon('useragent') -useragent.config_mapping = {'user_agent': 'USER_AGENT'} +useragent = make_builtin_addon("useragent") +useragent.config_mapping = {"user_agent": "USER_AGENT"} # ITEM PIPELINES @@ -93,27 +120,27 @@ useragent.config_mapping = {'user_agent': 'USER_AGENT'} # EXTENSIONS -autothrottle = make_builtin_addon('autothrottle', {'enabled': True}) +autothrottle = make_builtin_addon("autothrottle", {"enabled": True}) -corestats = make_builtin_addon('corestats') +corestats = make_builtin_addon("corestats") -closespider = make_builtin_addon('closespider') +closespider = make_builtin_addon("closespider") -debugger = make_builtin_addon('debugger') +debugger = make_builtin_addon("debugger") -feedexport = make_builtin_addon('feedexport') -feedexport.settings_prefix = 'FEED' +feedexport = make_builtin_addon("feedexport") +feedexport.settings_prefix = "FEED" -logstats = make_builtin_addon('logstats') +logstats = make_builtin_addon("logstats") -memdebug = make_builtin_addon('memdebug', {'enabled': True}) +memdebug = make_builtin_addon("memdebug", {"enabled": True}) -memusage = make_builtin_addon('memusage', {'enabled': True}) +memusage = make_builtin_addon("memusage", {"enabled": True}) -spiderstate = make_builtin_addon('spiderstate') +spiderstate = make_builtin_addon("spiderstate") -stacktracedump = make_builtin_addon('stacktracedump') +stacktracedump = make_builtin_addon("stacktracedump") -statsmailer = make_builtin_addon('statsmailer') +statsmailer = make_builtin_addon("statsmailer") -telnetconsole = make_builtin_addon('telnetconsole') +telnetconsole = make_builtin_addon("telnetconsole") diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 4bf7e7e66..8577cce02 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -85,7 +85,7 @@ def load_module_or_object(path): return load_object(path) except (ValueError, NameError, ImportError): pass - raise NameError("Could not load '%s'" % path) + raise NameError(f"Could not load '{path}'") def walk_modules(path): diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index a4e278fa5..741dd81cf 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -1,145 +1,143 @@ -from collections import OrderedDict import itertools -import os.path -import six -from tests import mock import unittest import warnings +from collections import OrderedDict +from unittest import mock from pkg_resources import VersionConflict -import zope.interface +from zope.interface import directlyProvides +from zope.interface.exceptions import BrokenImplementation, MultipleInvalid from zope.interface.verify import verifyObject -from zope.interface.exceptions import BrokenImplementation from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler from scrapy.interfaces import IAddon from scrapy.settings import BaseSettings -from . import addons -from . import addonmod +from . import addonmod, addons class AddonTest(unittest.TestCase): - def setUp(self): self.rawaddon = Addon() class AddonWithAttributes(Addon): - name = 'Test' - version = '1.0' + name = "Test" + version = "1.0" + self.testaddon = AddonWithAttributes() def test_interface(self): # Raw Addon should fail exactly b/c name and version are not given - self.assertFalse(hasattr(self.rawaddon, 'name')) - self.assertFalse(hasattr(self.rawaddon, 'version')) - self.assertRaises(BrokenImplementation, verifyObject, IAddon, - self.rawaddon) + self.assertFalse(hasattr(self.rawaddon, "name")) + self.assertFalse(hasattr(self.rawaddon, "version")) + self.assertRaises(MultipleInvalid, verifyObject, IAddon, self.rawaddon) verifyObject(IAddon, self.testaddon) def test_export_component(self): - settings = BaseSettings({'ITEM_PIPELINES': BaseSettings(), - 'DOWNLOAD_HANDLERS': BaseSettings()}, - 'default') + settings = BaseSettings( + {"ITEM_PIPELINES": BaseSettings(), "DOWNLOAD_HANDLERS": BaseSettings()}, + "default", + ) self.testaddon.component_type = None self.testaddon.export_component({}, settings) - self.assertEqual(len(settings['ITEM_PIPELINES']), 0) - self.testaddon.component_type = 'ITEM_PIPELINES' - self.testaddon.component = 'test.component' + self.assertEqual(len(settings["ITEM_PIPELINES"]), 0) + self.testaddon.component_type = "ITEM_PIPELINES" + self.testaddon.component = "test.component" self.testaddon.export_component({}, settings) - six.assertCountEqual(self, settings['ITEM_PIPELINES'], - ['test.component']) - self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 0) + self.assertCountEqual(settings["ITEM_PIPELINES"], ["test.component"]) + self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 0) self.testaddon.component_order = 313 self.testaddon.export_component({}, settings) - self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 313) - self.testaddon.component_type = 'DOWNLOAD_HANDLERS' - self.testaddon.component_key = 'http' + self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 313) + self.testaddon.component_type = "DOWNLOAD_HANDLERS" + self.testaddon.component_key = "http" self.testaddon.export_component({}, settings) - self.assertEqual(settings['DOWNLOAD_HANDLERS']['http'], - 'test.component') + self.assertEqual(settings["DOWNLOAD_HANDLERS"]["http"], "test.component") def test_export_basics(self): settings = BaseSettings() - self.testaddon.basic_settings = {'TESTKEY': 313, 'OTHERKEY': True} + self.testaddon.basic_settings = {"TESTKEY": 313, "OTHERKEY": True} self.testaddon.export_basics(settings) - self.assertEqual(settings['TESTKEY'], 313) - self.assertEqual(settings['OTHERKEY'], True) - self.assertEqual(settings.getpriority('TESTKEY'), 15) + self.assertEqual(settings["TESTKEY"], 313) + self.assertEqual(settings["OTHERKEY"], True) + self.assertEqual(settings.getpriority("TESTKEY"), 15) def test_export_config(self): settings = BaseSettings() self.testaddon.settings_prefix = None - self.testaddon.config_mapping = {'MAPPED_key': 'MAPPING_WORKED'} - self.testaddon.default_config = {'key': 55, 'defaultkey': 100} - self.testaddon.export_config({'key': 313, 'OTHERKEY': True, - 'mapped_KEY': 99}, settings) - self.assertEqual(settings['TEST_KEY'], 313) - self.assertEqual(settings['TEST_DEFAULTKEY'], 100) - self.assertEqual(settings['TEST_OTHERKEY'], True) - self.assertNotIn('MAPPED_key', settings) - self.assertNotIn('MAPPED_KEY', settings) - self.assertEqual(settings['MAPPING_WORKED'], 99) - self.assertEqual(settings.getpriority('TEST_KEY'), 15) + self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"} + self.testaddon.default_config = {"key": 55, "defaultkey": 100} + self.testaddon.export_config( + {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings + ) + self.assertEqual(settings["TEST_KEY"], 313) + self.assertEqual(settings["TEST_DEFAULTKEY"], 100) + self.assertEqual(settings["TEST_OTHERKEY"], True) + self.assertNotIn("MAPPED_key", settings) + self.assertNotIn("MAPPED_KEY", settings) + self.assertEqual(settings["MAPPING_WORKED"], 99) + self.assertEqual(settings.getpriority("TEST_KEY"), 15) - self.testaddon.settings_prefix = 'PREF' - self.testaddon.export_config({'newkey': 99}, settings) - self.assertEqual(settings['PREF_NEWKEY'], 99) + self.testaddon.settings_prefix = "PREF" + self.testaddon.export_config({"newkey": 99}, settings) + self.assertEqual(settings["PREF_NEWKEY"], 99) - with mock.patch.object(settings, 'set') as mock_set: + with mock.patch.object(settings, "set") as mock_set: self.testaddon.settings_prefix = False - self.testaddon.export_config({'thirdnewkey': 99}, settings) + self.testaddon.export_config({"thirdnewkey": 99}, settings) self.assertEqual(mock_set.call_count, 0) def test_update_settings(self): settings = BaseSettings() - settings.set('TEST_KEY1', 'default', priority='default') - settings.set('TEST_KEY2', 'project', priority='project') + settings.set("TEST_KEY1", "default", priority="default") + settings.set("TEST_KEY2", "project", priority="project") self.testaddon.settings_prefix = None - self.testaddon.basic_settings = {'OTHERTEST_KEY': 'addon'} - addon_config = {'key1': 'addon', 'key2': 'addon', 'key3': 'addon'} + self.testaddon.basic_settings = {"OTHERTEST_KEY": "addon"} + addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"} self.testaddon.update_settings(addon_config, settings) - self.assertEqual(settings['OTHERTEST_KEY'], 'addon') - self.assertEqual(settings['TEST_KEY1'], 'addon') - self.assertEqual(settings['TEST_KEY2'], 'project') - self.assertEqual(settings['TEST_KEY3'], 'addon') + self.assertEqual(settings["OTHERTEST_KEY"], "addon") + self.assertEqual(settings["TEST_KEY1"], "addon") + self.assertEqual(settings["TEST_KEY2"], "project") + self.assertEqual(settings["TEST_KEY3"], "addon") class AddonManagerTest(unittest.TestCase): - def setUp(self): self.manager = AddonManager() def test_add(self): manager = AddonManager() - manager.add(addonmod, {'key': 'val1'}) - manager.add('tests.test_addons.addons.GoodAddon') - six.assertCountEqual(self, manager, ['AddonModule', 'GoodAddon']) - self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['AddonModule'], ['key']) - self.assertEqual(manager.configs['AddonModule']['key'], 'val1') + manager.add(addonmod, {"key": "val1"}) + manager.add("tests.test_addons.addons.GoodAddon") + self.assertCountEqual(manager, ["AddonModule", "GoodAddon"]) + self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) + self.assertCountEqual(manager.configs["AddonModule"], ["key"]) + self.assertEqual(manager.configs["AddonModule"]["key"], "val1") self.assertRaises(ValueError, manager.add, addonmod) def test_add_dont_instantiate_providing_classes(self): class ProviderGoodAddon(addons.GoodAddon): pass - zope.interface.directlyProvides(ProviderGoodAddon, IAddon) + + directlyProvides(ProviderGoodAddon, IAddon) manager = AddonManager() manager.add(ProviderGoodAddon) - self.assertIs(manager['GoodAddon'], ProviderGoodAddon) + self.assertIs(manager["GoodAddon"], ProviderGoodAddon) def test_add_verifies(self): - brokenaddon = self.manager.get_addon( - 'tests.test_addons.addons.BrokenAddon') - self.assertRaises(zope.interface.exceptions.BrokenImplementation, - self.manager.add, - brokenaddon) + brokenaddon = self.manager.get_addon("tests.test_addons.addons.BrokenAddon") + self.assertRaises( + BrokenImplementation, + self.manager.add, + brokenaddon, + ) def test_add_adds_missing_interface_declaration(self): class GoodAddonWithoutDeclaration(object): - name = 'GoodAddonWithoutDeclaration' - version = '1.0' + name = "GoodAddonWithoutDeclaration" + version = "1.0" + self.manager.add(GoodAddonWithoutDeclaration) def test_remove(self): @@ -147,64 +145,66 @@ class AddonManagerTest(unittest.TestCase): def test_gets_removed(removearg): manager.add(addonmod) - self.assertIn('AddonModule', manager) + self.assertIn("AddonModule", manager) manager.remove(removearg) - self.assertNotIn('AddonModule', manager) + self.assertNotIn("AddonModule", manager) - test_gets_removed('AddonModule') + test_gets_removed("AddonModule") test_gets_removed(addonmod) - test_gets_removed('tests.test_addons.addonmod') - self.assertRaises(KeyError, manager.remove, 'nonexistent') + test_gets_removed("tests.test_addons.addonmod") + self.assertRaises(KeyError, manager.remove, "nonexistent") self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) def test_get_addon(self): - goodaddon = self.manager.get_addon('tests.test_addons.addons.GoodAddon') + goodaddon = self.manager.get_addon("tests.test_addons.addons.GoodAddon") self.assertIs(goodaddon, addons.GoodAddon) loaded_addonmod = self.manager.get_addon("tests.test_addons.addonmod") self.assertIs(loaded_addonmod, addonmod) - addonspath = os.path.join(os.path.dirname(__file__), 'addons.py') goodaddon = self.manager.get_addon("tests.test_addons.addons") self.assertIsInstance(goodaddon, addons.GoodAddon) - self.assertRaises(NameError, self.manager.get_addon, 'xy.n_onexistent') + self.assertRaises(NameError, self.manager.get_addon, "xy.n_onexistent") def test_get_addon_forward(self): class SomeCls(object): - _addon = 'tests.test_addons.addons.GoodAddon' + _addon = "tests.test_addons.addons.GoodAddon" + self.assertIs(self.manager.get_addon(SomeCls()), addons.GoodAddon) def test_get_addon_nested(self): - x = addons.GoodAddon('outer') - x._addon = addons.GoodAddon('middle') - x._addon._addon = addons.GoodAddon('inner') + x = addons.GoodAddon("outer") + x._addon = addons.GoodAddon("middle") + x._addon._addon = addons.GoodAddon("inner") self.assertIs(self.manager.get_addon(x), x._addon._addon) def test_load_dict_load_settings(self): def _test_load_method(func, *args, **kwargs): manager = AddonManager() getattr(manager, func)(*args, **kwargs) - six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule']) - self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon) - six.assertCountEqual(self, manager.configs['GoodAddon'], ['key']) - self.assertEqual(manager.configs['GoodAddon']['key'], 'val2') - self.assertEqual(manager['AddonModule'], addonmod) - self.assertIn('key', manager.configs['AddonModule']) - self.assertEqual(manager.configs['AddonModule']['key'], 'val1') + self.assertCountEqual(manager, ["GoodAddon", "AddonModule"]) + self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) + self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) + self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") + self.assertEqual(manager["AddonModule"], addonmod) + self.assertIn("key", manager.configs["AddonModule"]) + self.assertEqual(manager.configs["AddonModule"]["key"], "val1") addonsdict = { - "tests.test_addons.addonmod": {'key': 'val1'}, - 'tests.test_addons.addons.GoodAddon': {'key': 'val2'}, - } - _test_load_method('load_dict', addonsdict) + "tests.test_addons.addonmod": {"key": "val1"}, + "tests.test_addons.addons.GoodAddon": {"key": "val2"}, + } + _test_load_method("load_dict", addonsdict) settings = BaseSettings() - settings.set('ADDONS', {"tests.test_addons.addonmod": 0, - 'tests.test_addons.addons.GoodAddon': 0}) - settings.set('ADDONMODULE', {'key': 'val1'}) - settings.set('GOODADDON', {'key': 'val2'}) - _test_load_method('load_settings', settings) + settings.set( + "ADDONS", + {"tests.test_addons.addonmod": 0, "tests.test_addons.addons.GoodAddon": 0}, + ) + settings.set("ADDONMODULE", {"key": "val1"}) + settings.set("GOODADDON", {"key": "val2"}) + _test_load_method("load_settings", settings) def test_load_dict_load_settings_order(self): def _test_load_method(expected_order, func, *args, **kwargs): @@ -218,72 +218,71 @@ class AddonManagerTest(unittest.TestCase): for ordered_addons in itertools.permutations(addonlist): expected_order = [a.name for a in ordered_addons] addonsdict = OrderedDict((a, {}) for a in ordered_addons) - _test_load_method(expected_order, 'load_dict', addonsdict) - settings = BaseSettings({ - 'ADDONS': {a: i for i, a in enumerate(ordered_addons)} - }) - _test_load_method(expected_order, 'load_settings', settings) + _test_load_method(expected_order, "load_dict", addonsdict) + settings = BaseSettings( + {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} + ) + _test_load_method(expected_order, "load_settings", settings) def test_enabled_disabled(self): manager = AddonManager() - manager.add(addons.GoodAddon('FirstAddon')) - manager.add(addons.GoodAddon('SecondAddon')) - self.assertEqual(set(manager.enabled), - set(('FirstAddon', 'SecondAddon'))) + manager.add(addons.GoodAddon("FirstAddon")) + manager.add(addons.GoodAddon("SecondAddon")) + self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon"))) self.assertEqual(manager.disabled, []) - manager.disable('FirstAddon') - self.assertEqual(manager.enabled, ['SecondAddon']) - self.assertEqual(manager.disabled, ['FirstAddon']) - manager.enable('FirstAddon') - self.assertEqual(set(manager.enabled), - set(('FirstAddon', 'SecondAddon'))) + manager.disable("FirstAddon") + self.assertEqual(manager.enabled, ["SecondAddon"]) + self.assertEqual(manager.disabled, ["FirstAddon"]) + manager.enable("FirstAddon") + self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon"))) self.assertEqual(manager.disabled, []) def test_enable_before_add(self): manager = AddonManager() - self.assertRaises(ValueError, manager.enable, 'FirstAddon') - manager.disable('FirstAddon') - manager.enable('FirstAddon') - manager.add(addons.GoodAddon('FirstAddon')) - self.assertIn('FirstAddon', manager.enabled) + self.assertRaises(ValueError, manager.enable, "FirstAddon") + manager.disable("FirstAddon") + manager.enable("FirstAddon") + manager.add(addons.GoodAddon("FirstAddon")) + self.assertIn("FirstAddon", manager.enabled) def test_disable_before_add(self): manager = AddonManager() - manager.disable('FirstAddon') - manager.add(addons.GoodAddon('FirstAddon')) - self.assertEqual(manager.disabled, ['FirstAddon']) + manager.disable("FirstAddon") + manager.add(addons.GoodAddon("FirstAddon")) + self.assertEqual(manager.disabled, ["FirstAddon"]) def test_callbacks(self): - first_addon = addons.GoodAddon('FirstAddon') - second_addon = addons.GoodAddon('SecondAddon') + first_addon = addons.GoodAddon("FirstAddon") + second_addon = addons.GoodAddon("SecondAddon") manager = AddonManager() - manager.add(first_addon, {'test': 'first'}) - manager.add(second_addon, {'test': 'second'}) + manager.add(first_addon, {"test": "first"}) + manager.add(second_addon, {"test": "second"}) crawler = mock.create_autospec(Crawler) settings = BaseSettings() - with mock.patch.object(first_addon, 'update_addons') as ua_first, \ - mock.patch.object(second_addon, 'update_addons') as ua_second, \ - mock.patch.object(first_addon, 'update_settings') as us_first, \ - mock.patch.object(second_addon, 'update_settings') as us_second, \ - mock.patch.object(first_addon, 'check_configuration') as cc_first, \ - mock.patch.object(second_addon, 'check_configuration') as cc_second: + with mock.patch.object( + first_addon, "update_addons" + ) as ua_first, mock.patch.object( + second_addon, "update_addons" + ) as ua_second, mock.patch.object( + first_addon, "update_settings" + ) as us_first, mock.patch.object( + second_addon, "update_settings" + ) as us_second, mock.patch.object( + first_addon, "check_configuration" + ) as cc_first, mock.patch.object( + second_addon, "check_configuration" + ) as cc_second: manager.update_addons() - ua_first.assert_called_once_with(manager.configs['FirstAddon'], - manager) - ua_second.assert_called_once_with(manager.configs['SecondAddon'], - manager) + ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager) + ua_second.assert_called_once_with(manager.configs["SecondAddon"], manager) manager.update_settings(settings) - us_first.assert_called_once_with(manager.configs['FirstAddon'], - settings) - us_second.assert_called_once_with(manager.configs['SecondAddon'], - settings) + us_first.assert_called_once_with(manager.configs["FirstAddon"], settings) + us_second.assert_called_once_with(manager.configs["SecondAddon"], settings) manager.check_configuration(crawler) - cc_first.assert_called_once_with(manager.configs['FirstAddon'], - crawler) - cc_second.assert_called_once_with(manager.configs['SecondAddon'], - crawler) + cc_first.assert_called_once_with(manager.configs["FirstAddon"], crawler) + cc_second.assert_called_once_with(manager.configs["SecondAddon"], crawler) self.assertEqual(ua_first.call_count, 1) self.assertEqual(ua_second.call_count, 1) self.assertEqual(us_first.call_count, 1) @@ -291,10 +290,10 @@ class AddonManagerTest(unittest.TestCase): us_first.reset_mock() us_second.reset_mock() - manager.disable('FirstAddon') + manager.disable("FirstAddon") manager.update_settings(settings) self.assertEqual(us_first.call_count, 0) - manager.enable('FirstAddon') + manager.enable("FirstAddon") manager.update_settings(settings) self.assertEqual(us_first.call_count, 1) self.assertEqual(us_second.call_count, 2) @@ -302,44 +301,42 @@ class AddonManagerTest(unittest.TestCase): # This will become relevant when we let spiders implement the add-on # interface and should be replaced with a test where # AddonManager.spidercls = None then. - manager._call_if_exists(None, 'irrelevant') + manager._call_if_exists(None, "irrelevant") def test_update_addons_last_minute_add(self): class AddedAddon(addons.GoodAddon): - name = 'AddedAddon' + name = "AddedAddon" class FirstAddon(addons.GoodAddon): - name = 'FirstAddon' + name = "FirstAddon" def update_addons(self, config, addons): addons.add(AddedAddon()) manager = AddonManager() first_addon = FirstAddon() - with mock.patch.object(first_addon, 'update_addons', - wraps=first_addon.update_addons) as ua_first, \ - mock.patch.object(AddedAddon, 'update_addons') as ua_added: - manager.add(first_addon, {'non-empty': 'dict'}) + with mock.patch.object( + first_addon, "update_addons", wraps=first_addon.update_addons + ) as ua_first, mock.patch.object(AddedAddon, "update_addons") as ua_added: + manager.add(first_addon, {"non-empty": "dict"}) manager.update_addons() - six.assertCountEqual(self, manager, ['FirstAddon', 'AddedAddon']) - ua_first.assert_called_once_with(manager.configs['FirstAddon'], - manager) - ua_added.assert_called_once_with(manager.configs['AddedAddon'], - manager) + self.assertCountEqual(manager, ["FirstAddon", "AddedAddon"]) + ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager) + ua_added.assert_called_once_with(manager.configs["AddedAddon"], manager) def test_check_dependency_clashes_attributes(self): provides = addons.GoodAddon("ProvidesAddon") - provides.provides = ('test', ) + provides.provides = ("test",) provides2 = addons.GoodAddon("ProvidesAddon2") - provides2.provides = ('test', ) + provides2.provides = ("test",) requires = addons.GoodAddon("RequiresAddon") - requires.requires = ('test', ) + requires.requires = ("test",) requires_name = addons.GoodAddon("RequiresNameAddon") - requires_name.requires = ('ProvidesAddon', ) + requires_name.requires = ("ProvidesAddon",) requires_newer = addons.GoodAddon("RequiresNewerAddon") - requires_newer.requires = ('test>=2.0', ) + requires_newer.requires = ("test>=2.0",) modifies = addons.GoodAddon("ModifiesAddon") - modifies.modifies = ('test', ) + modifies.modifies = ("test",) def check_with(*addons): manager = AddonManager() diff --git a/tests/test_addons/addonmod.py b/tests/test_addons/addonmod.py index 8ecf4b81d..c59f6737f 100644 --- a/tests/test_addons/addonmod.py +++ b/tests/test_addons/addonmod.py @@ -9,8 +9,10 @@ FROM = "test_addons.addonmod" name = "AddonModule" version = "1.0" + def update_settings(config, settings): pass + def check_configuration(config, crawler): pass diff --git a/tests/test_addons/addons.py b/tests/test_addons/addons.py index f3442b192..4adb9fe8f 100644 --- a/tests/test_addons/addons.py +++ b/tests/test_addons/addons.py @@ -1,18 +1,16 @@ import zope.interface -from scrapy.addons import Addon from scrapy.interfaces import IAddon class Addon(object): - FROM = 'test_addons.addons' + FROM = "test_addons.addons" @zope.interface.declarations.implementer(IAddon) class GoodAddon(object): - - name = 'GoodAddon' - version = '1.0' + name = "GoodAddon" + version = "1.0" def __init__(self, name=None, version=None): if name is not None: @@ -32,8 +30,7 @@ class GoodAddon(object): @zope.interface.declarations.implementer(IAddon) class BrokenAddon(object): - - name = 'BrokenAddon' + name = "BrokenAddon" # No version diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.py index c89876950..1050cbbed 100644 --- a/tests/test_addons/test_builtins.py +++ b/tests/test_addons/test_builtins.py @@ -7,19 +7,17 @@ from scrapy.settings import Settings class BuiltinAddonsTest(unittest.TestCase): - def test_make_builtin_addon(self): - httpcache = make_builtin_addon('httpcache', {'enabled': True}) - self.assertEqual(httpcache.name, 'httpcache') - self.assertEqual(httpcache.default_config, {'enabled': True}) + httpcache = make_builtin_addon("httpcache", {"enabled": True}) + self.assertEqual(httpcache.name, "httpcache") + self.assertEqual(httpcache.default_config, {"enabled": True}) self.assertEqual(httpcache.version, scrapy.__version__) - httpcache = make_builtin_addon('httpcache', {'enabled': True}, '99.9') - self.assertEqual(httpcache.version, '99.9') + httpcache = make_builtin_addon("httpcache", {"enabled": True}, "99.9") + self.assertEqual(httpcache.version, "99.9") def test_defaultheaders_export_config(self): settings = Settings() dh = scrapy.addons.defaultheaders() - dh.export_config({'X-Test-Header': 'val'}, settings) - self.assertIn('X-Test-Header', settings['DEFAULT_REQUEST_HEADERS']) - self.assertEqual(settings['DEFAULT_REQUEST_HEADERS']['X-Test-Header'], - 'val') + dh.export_config({"X-Test-Header": "val"}, settings) + self.assertIn("X-Test-Header", settings["DEFAULT_REQUEST_HEADERS"]) + self.assertEqual(settings["DEFAULT_REQUEST_HEADERS"]["X-Test-Header"], "val") diff --git a/tests/test_crawl.py b/tests/test_crawl.py index e353f80cb..1e97863b0 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -419,7 +419,7 @@ with multiples lines addonmgr = AddonManager() addonmgr.add(FailedCheckAddon()) - crawler = self.runner.create_crawler(SimpleSpider) + crawler = get_crawler(SimpleSpider) crawler.addons = addonmgr # Doesn't work in 'precise' test environment: # with self.assertRaises(ValueError): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 7ddf952f7..fd57d846e 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -90,14 +90,12 @@ class MiddlewareManagerTest(unittest.TestCase): def test_instances_from_settings(self): settings = Settings() - myM3 = M3() class InstanceTestMiddlewareManager(MiddlewareManager): @classmethod def _get_mwlist_from_settings(cls, settings): - return ["tests.test_middleware.M1", M2, myM3] + return ["tests.test_middleware.M1", M2] mwman = InstanceTestMiddlewareManager.from_settings(settings) self.assertIsInstance(mwman.middlewares[0], M1) self.assertIsInstance(mwman.middlewares[1], M2) - self.assertIs(mwman.middlewares[2], myM3) From 55ac26228b853aec6d543bfaf3ab85ebbf5ab002 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 16:50:43 +0400 Subject: [PATCH 21/55] Remove builtin addons. --- scrapy/{addons/__init__.py => addons.py} | 3 - scrapy/addons/builtins.py | 146 ----------------------- sep/sep-021.rst | 25 ---- tests/test_addons/test_builtins.py | 23 ---- 4 files changed, 197 deletions(-) rename scrapy/{addons/__init__.py => addons.py} (99%) delete mode 100644 scrapy/addons/builtins.py delete mode 100644 tests/test_addons/test_builtins.py diff --git a/scrapy/addons/__init__.py b/scrapy/addons.py similarity index 99% rename from scrapy/addons/__init__.py rename to scrapy/addons.py index aad9ef789..c9b873039 100644 --- a/scrapy/addons/__init__.py +++ b/scrapy/addons.py @@ -456,6 +456,3 @@ class AddonManager(Mapping): """ for name in self: self._call_addon(name, "check_configuration", crawler) - - -from scrapy.addons.builtins import * # noqa diff --git a/scrapy/addons/builtins.py b/scrapy/addons/builtins.py deleted file mode 100644 index ea3afbf99..000000000 --- a/scrapy/addons/builtins.py +++ /dev/null @@ -1,146 +0,0 @@ -import scrapy -from scrapy.addons import Addon - -__all__ = [ - "make_builtin_addon", - "depth", - "httperror", - "offsite", - "referer", - "urllength", - "ajaxcrawl", - "chunked", - "cookies", - "defaultheaders", - "downloadtimeout", - "httpauth", - "httpcache", - "httpcompression", - "httpproxy", - "metarefresh", - "redirect", - "retry", - "robotstxt", - "stats", - "useragent", - "autothrottle", - "corestats", - "closespider", - "debugger", - "feedexport", - "logstats", - "memdebug", - "memusage", - "spiderstate", - "stacktracedump", - "statsmailer", - "telnetconsole", -] - - -def make_builtin_addon(addon_name, addon_default_config=None, addon_version=None): - class ThisAddon(Addon): - name = addon_name - version = addon_version or scrapy.__version__ - default_config = addon_default_config or {} - - return ThisAddon - - -# XXX: Below are CLASSES that have lowercase names. This is in line with the -# original SEP-021 but violates PEP8. -# We might consider prepending all built-in addon names with scrapy_ or similar -# to reduce the chance of name clashes. - -# SPIDER MIDDLEWARES - -depth = make_builtin_addon("depth") - -httperror = make_builtin_addon("httperror") - -offsite = make_builtin_addon("offsite") - -referer = make_builtin_addon("referer") - -urllength = make_builtin_addon("urllength") - - -# DOWNLOADER MIDDLEWARES - -ajaxcrawl = make_builtin_addon("ajaxcrawl", {"enabled": True}) - -chunked = make_builtin_addon("chunked") - -cookies = make_builtin_addon("cookies") - -defaultheaders = make_builtin_addon("defaultheaders") - - -# Assume every config entry is a header -def defaultheaders_export_config(self, config, settings): - conf = self.default_config or {} - conf.update(config) - settings.set("DEFAULT_REQUEST_HEADERS", conf, "addon") - - -defaultheaders.export_config = defaultheaders_export_config - -downloadtimeout = make_builtin_addon("downloadtimeout") -downloadtimeout.config_mapping = { - "timeout": "DOWNLOAD_TIMEOUT", - "download_timeout": "DOWNLOAD_TIMEOUT", -} - -httpauth = make_builtin_addon("httpauth") - -httpcache = make_builtin_addon("httpcache", {"enabled": True}) - -httpcompression = make_builtin_addon("httpcompression") -httpcompression.config_mapping = {"enabled": "COMPRESSION_ENABLED"} - -httpproxy = make_builtin_addon("httpproxy") - -metarefresh = make_builtin_addon("metarefresh") -metarefresh.config_mapping = {"max_times": "REDIRECT_MAX_TIMES"} - -redirect = make_builtin_addon("redirect") - -retry = make_builtin_addon("retry") - -robotstxt = make_builtin_addon("robotstxt", {"obey": True}) - -stats = make_builtin_addon("stats") - -useragent = make_builtin_addon("useragent") -useragent.config_mapping = {"user_agent": "USER_AGENT"} - - -# ITEM PIPELINES - - -# EXTENSIONS - -autothrottle = make_builtin_addon("autothrottle", {"enabled": True}) - -corestats = make_builtin_addon("corestats") - -closespider = make_builtin_addon("closespider") - -debugger = make_builtin_addon("debugger") - -feedexport = make_builtin_addon("feedexport") -feedexport.settings_prefix = "FEED" - -logstats = make_builtin_addon("logstats") - -memdebug = make_builtin_addon("memdebug", {"enabled": True}) - -memusage = make_builtin_addon("memusage", {"enabled": True}) - -spiderstate = make_builtin_addon("spiderstate") - -stacktracedump = make_builtin_addon("stacktracedump") - -statsmailer = make_builtin_addon("statsmailer") - -telnetconsole = make_builtin_addon("telnetconsole") diff --git a/sep/sep-021.rst b/sep/sep-021.rst index cb1701014..47cba004c 100644 --- a/sep/sep-021.rst +++ b/sep/sep-021.rst @@ -322,28 +322,3 @@ with ``scrapy.addons.`` prepended (i.e. pointing to Scrapy's ``addons`` submodule). If the object found has an ``_addon`` attribute, that attribute will be treated as the found add-on. This allows, for example, to change the add-on based on the Python version. - -Updating existing extensions ----------------------------- - -An ``Addon`` class is introduced that add-on developers may or may not subclass -depending on how much of the 'default functionality' they want. Naturally, it -does not provide ``NAME`` and ``VERSION``. Its default ``update_settings()`` -exposes the add-on configuration into the global settings namespace with an -appropriate name, e.g. this section from ``scrapy.cfg``:: - - [httpcache] - dir = /some/dir - -would expose ``HTTPCACHE_DIR``. - -Add-on modules will be written for all built-in extensions and placed in -``scrapy.addons``. For many default Scrapy components, it will be sufficient to -create a subclass of ``Addon`` with minor or no method modifications. The -component code remains where it is (i.e. in ``scrapy.pipelines``, etc.). - -Later, the global settings namespace could be cleaned up in a backwards --incompatible fashion by deprecating support for the global setting names, e.g. -``HTTPCACHE_DIR``, and instead instantiate the components with the add-on -configuration in ``update_settings()``. - diff --git a/tests/test_addons/test_builtins.py b/tests/test_addons/test_builtins.py deleted file mode 100644 index 1050cbbed..000000000 --- a/tests/test_addons/test_builtins.py +++ /dev/null @@ -1,23 +0,0 @@ -import unittest - -import scrapy -import scrapy.addons -from scrapy.addons.builtins import make_builtin_addon -from scrapy.settings import Settings - - -class BuiltinAddonsTest(unittest.TestCase): - def test_make_builtin_addon(self): - httpcache = make_builtin_addon("httpcache", {"enabled": True}) - self.assertEqual(httpcache.name, "httpcache") - self.assertEqual(httpcache.default_config, {"enabled": True}) - self.assertEqual(httpcache.version, scrapy.__version__) - httpcache = make_builtin_addon("httpcache", {"enabled": True}, "99.9") - self.assertEqual(httpcache.version, "99.9") - - def test_defaultheaders_export_config(self): - settings = Settings() - dh = scrapy.addons.defaultheaders() - dh.export_config({"X-Test-Header": "val"}, settings) - self.assertIn("X-Test-Header", settings["DEFAULT_REQUEST_HEADERS"]) - self.assertEqual(settings["DEFAULT_REQUEST_HEADERS"]["X-Test-Header"], "val") From fdbc141b23b95f8f744712f86c28a4d4d4d8c52f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 17:22:51 +0400 Subject: [PATCH 22/55] Replace pkg_resources with packaging. --- scrapy/addons.py | 28 ++++++++++++---------------- tests/test_addons/__init__.py | 3 +-- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index c9b873039..8f0f39889 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -2,9 +2,11 @@ import warnings from collections import OrderedDict, defaultdict from collections.abc import Mapping from inspect import isclass +from typing import Dict import zope.interface -from pkg_resources import Distribution, Requirement, WorkingSet +from packaging.requirements import Requirement +from packaging.version import Version from zope.interface.verify import verifyObject from scrapy.interfaces import IAddon @@ -303,7 +305,7 @@ class AddonManager(Mapping): for a, c in zip(addons, configs): self.add(a, c) - def check_dependency_clashes(self): + def check_dependency_clashes(self) -> None: """Check for incompatibilities in add-on dependencies. Add-ons can provide information about their dependencies in their @@ -320,24 +322,20 @@ class AddonManager(Mapping): add-on. """ # Collect all active add-ons and the components they provide - ws = WorkingSet("") + versions: Dict[str, Version] = {} - def add_dist(project_name, version, **kwargs): - if project_name in ws.entry_keys.get("scrapy", []): + def add_version(project_name, version): + if project_name in versions: raise ImportError( f"Component {project_name} provided by multiple add-ons" ) - else: - dist = Distribution( - project_name=project_name, version=version, **kwargs - ) - ws.add(dist, entry="scrapy") + versions[project_name] = Version(version) for name in self: ver = self[name].version - add_dist(name, ver) + add_version(name, ver) for provides_name in getattr(self[name], "provides", []): - add_dist(provides_name, ver) + add_version(provides_name, ver) # Collect all required and modified components def compile_attribute_dict(attribute_name): @@ -352,10 +350,8 @@ class AddonManager(Mapping): req_or_mod = set(required.keys()).union(modified.keys()) for reqstr in req_or_mod: - req = Requirement.parse(reqstr) - # May raise VersionConflict. Do we want to catch it and raise - # our own exception or is it helpful enough? - if ws.find(req) is None: + req = Requirement(reqstr) + if req.name not in versions or versions[req.name] not in req.specifier: raise ImportError( f"Add-ons {required[reqstr] + modified[reqstr]} require" f" or modify missing component {reqstr}" diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index 741dd81cf..451ffed0f 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -4,7 +4,6 @@ import warnings from collections import OrderedDict from unittest import mock -from pkg_resources import VersionConflict from zope.interface import directlyProvides from zope.interface.exceptions import BrokenImplementation, MultipleInvalid from zope.interface.verify import verifyObject @@ -347,7 +346,7 @@ class AddonManagerTest(unittest.TestCase): self.assertRaises(ImportError, check_with, requires) self.assertRaises(ImportError, check_with, modifies) self.assertRaises(ImportError, check_with, provides, provides2) - self.assertRaises(VersionConflict, check_with, provides, requires_newer) + self.assertRaises(ImportError, check_with, provides, requires_newer) with warnings.catch_warnings(record=True) as w: check_with(provides, modifies) check_with(provides) From 7ebb8256f029b2027a3603685569378ccce73070 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 17:52:11 +0400 Subject: [PATCH 23/55] Some cleanup. --- docs/topics/addons.rst | 3 --- scrapy/addons.py | 6 +++--- tests/test_addons/addonmod.py | 6 ++---- tests/test_addons/addons.py | 10 +++------- 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index ba6e839a5..b86058dc4 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -43,7 +43,6 @@ case with one requiring no configuration) are enabled/configured in a project's INSTALLED_ADDONS = ( 'httpcache', 'path.to.some.addon', - 'path/to/other/addon.py', ) HTTPCACHE = { @@ -73,8 +72,6 @@ dictionary keys. I.e., the configuration from above would look like this: [addon:path.to.some.addon] some_config = true - [addon:path/to/other/addon.py] - Enabling and configuring add-ons within Python code --------------------------------------------------- diff --git a/scrapy/addons.py b/scrapy/addons.py index 8f0f39889..43a36e5e5 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -156,10 +156,10 @@ class AddonManager(Mapping): addons = AddonManager() # ... load some add-ons here - print addons.enabled # prints names of all enabled add-ons - print addons['TestAddon'].version # prints version of add-on with name + print(addons.enabled) # prints names of all enabled add-ons + print(addons['TestAddon'].version) # prints version of add-on with name # 'TestAddon' - print addons.configs['TestAddon'] # prints configuration of 'TestAddon' + print(addons.configs['TestAddon']) # prints configuration of 'TestAddon' """ diff --git a/tests/test_addons/addonmod.py b/tests/test_addons/addonmod.py index c59f6737f..092c3c0eb 100644 --- a/tests/test_addons/addonmod.py +++ b/tests/test_addons/addonmod.py @@ -1,10 +1,8 @@ -import zope.interface +from zope.interface import moduleProvides from scrapy.interfaces import IAddon -zope.interface.moduleProvides(IAddon) - -FROM = "test_addons.addonmod" +moduleProvides(IAddon) name = "AddonModule" version = "1.0" diff --git a/tests/test_addons/addons.py b/tests/test_addons/addons.py index 4adb9fe8f..d878f37ea 100644 --- a/tests/test_addons/addons.py +++ b/tests/test_addons/addons.py @@ -1,13 +1,9 @@ -import zope.interface +from zope.interface import implementer from scrapy.interfaces import IAddon -class Addon(object): - FROM = "test_addons.addons" - - -@zope.interface.declarations.implementer(IAddon) +@implementer(IAddon) class GoodAddon(object): name = "GoodAddon" version = "1.0" @@ -28,7 +24,7 @@ class GoodAddon(object): pass -@zope.interface.declarations.implementer(IAddon) +@implementer(IAddon) class BrokenAddon(object): name = "BrokenAddon" # No version From 282fe3dd4fd4a87b27f8e9793760f877778d2914 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 17:52:25 +0400 Subject: [PATCH 24/55] Drop the SEP as we decided we don't want to update it. --- sep/sep-021.rst | 324 ------------------------------------------------ 1 file changed, 324 deletions(-) delete mode 100644 sep/sep-021.rst diff --git a/sep/sep-021.rst b/sep/sep-021.rst deleted file mode 100644 index 47cba004c..000000000 --- a/sep/sep-021.rst +++ /dev/null @@ -1,324 +0,0 @@ -======= =================== -SEP 21 -Title Add-ons -Author Pablo Hoffman -Created 2014-02-14 -Status Draft -======= =================== - -================ -SEP-021: Add-ons -================ - -This proposal introduces add-ons, a unified way to manage Scrapy extensions, -middlewares and pipelines. - -Scrapy currently supports many hooks and mechanisms for extending its -functionality, but no single entry point for enabling and configuring them. -Instead, the hooks are spread over: - -* Spider middlewares (``SPIDER_MIDDLEWARES``) -* Downloader middlewares (``DOWNLOADER_MIDDLEWARES``) -* Downloader handlers (``DOWNLOADER_HANDLERS``) -* Item pipelines (``ITEM_PIPELINES``) -* Feed exporters and storages (``FEED_EXPORTERS``, ``FEED_STORAGES``) -* Overridable components (``DUPEFILTER_CLASS``, ``STATS_CLASS``, - ``SCHEDULER``, ``SPIDER_MANAGER_CLASS``, ``ITEM_PROCESSOR``, etc.) -* Generic extensions (``EXTENSIONS``) -* CLI commands (``COMMANDS_MODULE``) - -This approach has several shortfalls: - -* Enabling an extension often requires modifying many settings, often in a - coordinated way, which is complex and error prone. -* Extension developers have little control over ensuring their library - dependencies and configuration requirements are met, especially since most - extensions never 'see' a fully-configured crawler before it starts running. -* The user is burdened with supervising potential interplay of extensions, - especially non-included ones, ranging from setting name clashes to mutually - excluding dependencies/configuration requirements. - -*Add-ons* search to remedy these shortcomings by enhancing Scrapy's extension -management, making it easy-to-use and transparent for users while giving more -configuration control to developers. - - -Design goals and non-goals -========================== - -Goals: - -* simple to manage: adding or removing extensions should be just a matter of - adding or removing lines in a configuration file -* backward compatibility with enabling extension the "old way" (i.e. modifying - settings directly) - -Non-goals: - -* a way to publish, distribute or discover extensions (use pypi for that) - - -User experience: managing add-ons -================================= - -Add-ons are enabled and configured either via Scrapy's settings, or (for add-ons -not bound to any project) in ``scrapy.cfg``. - -In the settings, add-ons can be enabled by adding either their name (for -built-in add-ons), their Python path, or their file path, to a -``INSTALLED_ADDONS`` setting. If necessary, each add-on can be configured by -providing a dictionary-valued setting with the uppercase add-on name. For -example, to enable and configure the built-in ``httpcache`` add-on and enable -(without configuring) two custom add-ons, one via Python path and one via file -path, add these entries to your settings module:: - - INSTALLED_ADDONS = ( - 'httpcache', - 'mymodule.filters.myfilter', - 'mymodule/filters/otherfilter.py', - ) - - HTTPCACHE = { - 'ignore_http_codes': [404, 503], - } - -In ``scrapy.cfg``, add-ons are enabled and configured with one section per -add-on. The section names correspond to the entries of ``INSTALLED_ADDONS``. -The configuration from above could look like this:: - - [addon:httpcache] - ignore_http_codes = 404,503 - - [addon:mymodule.filters.myfilter] - - [addon:mymodule/filters/otherfilter.py] - - -Developer experience: writing add-ons -===================================== - -Add-ons are (any) Python *objects* that implement Scrapy's *add-on interface*. -The interface is enforced through ``zope.interface``. This leaves the choice of -Python object up the developer. Examples: - -* for a small pipeline, the add-on interface could be implemented in the same - class that also implements the ``open/close_spider`` and ``process_item`` - callbacks -* for larger add-ons, or for clearer structure, the interface could be provided - by a stand-alone module - -The absolute minimum interface consists of just two attributes: - -* ``NAME``: string with add-on name -* ``VERSION``: PEP-440 style version string - -To be any useful, an add-on should implement at least one of the following -callback methods: - -* ``update_addons()``: adds and configures other add-ons -* ``update_settings()``: sets configuration (such as default values for this - add-on and required settings for other extensions) and enables needed - components. -* ``check_configuration()``: receives the fully-initialized ``Crawler`` - instance before it starts running, performs additional dependency and - configuration requirement checks - -Additionally, an add-on may (and should, where appropriate) provide one or more -variables that can be used for automated detection of possible dependency -clashes: - -* ``REQUIRES``: list of built-in or custom components required by this add-on, - as PEP-440 strings -* ``MODIFIES``: list of components whose functionality is affected or replaced - by this add-on (a custom HTTP cache should list ``httpcache`` here) -* ``PROVIDES``: list of components provided by this add-on (e.g. ``mongodb`` - for an extension that provides generic read/write access to a MongoDB - database, releasing other components from having to provide their own - database access methods) - -update_addons() ------------------ - -Called: -~~~~~~~ - -Shortly after initialisation of the ``Crawler`` object. - -Arguments: -~~~~~~~~~~ - -* ``config``: configuration of this add-on -* ``addons``: the add-on manager, providing methods to add and configure add-ons - -Purpose: -~~~~~~~~ - -* Configure and enable related add-ons, useful for 'umbrella add-ons' which - chain-load other add-ons based on the configuration - -Examples: -~~~~~~~~~ - -.. code-block:: python - - def update_addons(config, addons): - if "httpcache" not in addons.enabled: - addons.add("httpcache", {"expiration_secs": 60}) - -or: - -.. code-block:: python - - def update_addons(config, addons): - if "otheraddon" in addons.enabled: - addons.configs["otheraddon"]["some_config_name"] = True - -update_settings() ------------------ - -Called: -~~~~~~~ - -Directly after the ``update_addons()`` callback of all add-ons has been called. - -Arguments: -~~~~~~~~~~ - -* ``config``: configuration of this add-on -* ``settings``: the crawler's ``Settings`` instance containing all project - settings - -Purpose: -~~~~~~~~ - -* Modify ``settings`` to enable required components -* Expose some add-on specific configuration (``config``) into the global - settings namespace (``settings``) if necessary -* Raise exception if components can not be properly configured (e.g. on missing - dependencies); Scrapy will print this exception *and exit* (making users - explicitly acknowledge that the add-on does not work by forcing them to - disable it). - -Side note: -~~~~~~~~~~ - -The ``MiddlewareManager.from_settings()`` method will receive a slight -modification to allow directly placing Python objects instead of class paths -in the middleware dict settings. This way, add-ons can place already -instantiated components into the settings. This allows keeping configuration -as local to components as possible and avoids cluttering up the global -settings namespace. Furthermore, it allows reusing components (e.g. using -two instances of the same mongodb pipeline to write to different locations). - -Examples: -~~~~~~~~~ - -:: - - def update_settings(config, settings): - # Don't care where this module is located - settings.set['DOWNLADER_MIDDLEWARES']({ - __name__ + '.downloadermw.coolmw': 900, - }) - - # Instantiate components to not expose settings into - # the global namespace - from .pipelines import MySQLPipeline - mysqlpl = MySQLPipeline(password = config['password']) - settings.set['ITEM_PIPELINES']({ - mysqlpl: 200, - }) - -or:: - - def update_settings(config, settings): - # Assuming this class also has a process_item() method - settings.set['ITEM_PIPELINES']({ - self: 200, - }) - -or:: - - def update_settings(config, settings): - try: - import boto - except ImportError: - raise RuntimeError("boto library is required") - -check_configuration() ---------------------- - -Called: -~~~~~~~ - -Shortly before the crawler starts crawling. - -Arguments: -~~~~~~~~~~ - -* ``config``: configuration of this add-on -* ``crawler``: fully-initialized ``Crawler`` object, ready to start crawling - -Purpose: -~~~~~~~~ - -* Perform post-initialization checks like making sure the extension and its - dependencies were configured properly. -* Raise exception if a critical check failed; Scrapy will print this exception - *and exit* (see ``update_settings()`` purpose for rationale on this). - -Examples: -~~~~~~~~~ - -:: - - def check_configuration(config, crawler): - if 'some.other.addon' not in crawler.addons.enabled: - raise RuntimeError("Some other add-on required to use this add-on") - - -Implementation -============== - -A new core component, the *add-on manager*, is introduced to Scrapy. It -facilitates loading add-ons, gathering and providing information on them, -calling their callbacks at appropriate times, and performing basic checks for -dependency and configuration clashes. - -Layout ------- - -A new ``AddonManager`` class is introduced, providing methods to - -* add and remove add-ons, -* search for add-ons by name -* read enabled add-ons and their configurations from the settings module and - from ``settings.py``, -* enable and disable add-ons -* check for possible dependency incompatibilites by inspecting the collected - ``REQUIRES``, ``MODIFIES`` and ``PROVIDES`` add-on variables -* call the add-on callbacks - -Integration into start-up process ---------------------------------- - -The settings used to crawl are not complete until the spider-specific settings -have been loaded in ``Crawler.__init__()``. Add-on management follows this -approach and only starts loading add-ons when the crawler is initialised. - -Instantiation and the calls ``update_addons()`` and ``update_settings()`` happen -in ``Crawler.__init__()``. The final checks (i.e. the callback to -``check_configuration()``) is coded into the ``Crawler.crawl()`` method after -creating the engine. - -Finding add-ons ---------------- - -Add-on localisation is governed by the add-on paths given in -``INSTALLED_ADDONS`` (or by the section names if using ``scrapy.cfg``). If -nothing is found at the given path, it is tried again with ``addons.`` -prepended (i.e. pointing to the project's ``addons`` folder or module), then -with ``scrapy.addons.`` prepended (i.e. pointing to Scrapy's ``addons`` -submodule). If the object found has an ``_addon`` attribute, that attribute -will be treated as the found add-on. This allows, for example, to change the -add-on based on the Python version. From 22bd0d9a796cca980f3d2c2d951d0f3960ab8a18 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 17:58:11 +0400 Subject: [PATCH 25/55] Fix docs build. --- docs/topics/addons.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index b86058dc4..198c48bcc 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -236,6 +236,7 @@ convenience functions by overwriting :meth:`~scrapy.addons.Addon.update_settings`. .. module:: scrapy.addons + :noindex: .. autoclass:: Addon :members: From fdfab17438647b10a7cb74b808ddf4acfd89c6ab Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 18:00:18 +0400 Subject: [PATCH 26/55] Fix docs for the renamed ADDONS setting. --- docs/topics/addons.rst | 18 +++++++++--------- docs/topics/settings.rst | 20 ++++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 198c48bcc..25d7da50b 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -20,15 +20,15 @@ only then, the add-on manager will read a list of enabled add-ons and their configurations from your settings. There are two places where you can provide the paths to add-ons you want to enable: -* the ``INSTALLED_ADDONS`` setting, and +* the ``ADDONS`` setting, and * the ``scrapy.cfg`` file. As Scrapy settings can be modified from many places, e.g. in a project's ``settings.py``, in a Spider's ``custom_settings`` attribute, or from the -command line, using the ``INSTALLED_ADDONS`` setting is the preferred way to +command line, using the ``ADDONS`` setting is the preferred way to manage add-ons. -The ``INSTALLED_ADDONS`` setting a tuple in which every item is a path to an +The ``ADDONS`` setting a tuple in which every item is a path to an add-on. The path can be both a Python or a file path. While more precise, it is not necessary to specify the full add-on Python path if it is either built into Scrapy or lives in your project's ``addons`` submodule. @@ -40,7 +40,7 @@ This is an example where an internal add-on and two third-party add-ons (in this case with one requiring no configuration) are enabled/configured in a project's ``settings.py``:: - INSTALLED_ADDONS = ( + ADDONS = ( 'httpcache', 'path.to.some.addon', ) @@ -79,7 +79,7 @@ Enabling and configuring add-ons within Python code The :class:`~scrapy.addons.AddonManager` will only read from Scrapy's settings and from ``scrapy.cfg`` *at the beginning* of Scrapy's start-up process. Afterwards, i.e. as soon as the :class:`~scrapy.addons.AddonManager` is -populated, changing the ``INSTALLED_ADDONS`` setting or any of the add-on +populated, changing the ``ADDONS`` setting or any of the add-on configuration dictionary settings will have no effect. If you want to enable, disable, or configure add-ons in Python code, for example @@ -202,7 +202,7 @@ specify :pep:`440`-style information about required versions. Examples:: requires = ['otheraddon >= 2.0', 'yetanotheraddon'] The Python object or module that is pointed to by an add-on path (e.g. given in -the ``INSTALLED_ADDONS`` setting, or given to +the ``ADDONS`` setting, or given to :meth:`~scrapy.addons.AddonManager.add`) does not necessarily have to be an add-on. Instead, it can provide an ``_addon`` attribute. This attribute can be either an add-on or another add-on path. @@ -401,7 +401,7 @@ of placing this in your ``settings.py``:: you can also use the add-on framework:: - INSTALLED_ADDONS = ( + ADDONS = ( # ..., 'httpcache', ) @@ -412,9 +412,9 @@ you can also use the add-on framework:: } Note that you *must* enable built-in addons by placing them in your -``INSTALLED_ADDONS`` setting before you can use them for configuring built-in +``ADDONS`` setting before you can use them for configuring built-in components. I.e., configuring the ``HTTPCACHE`` setting will have no effect -when ``httpcache`` is not listed in ``INSTALLED_ADDONS``. +when ``httpcache`` is not listed in ``ADDONS``. In general, the add-on names match the lowercase name of the component, with its type suffix removed (i.e. the add-on configuring the diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 9e8c30efe..aa09abbd5 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -201,6 +201,16 @@ to any particular component. In that case the module of that component will be shown, typically an extension, middleware or pipeline. It also means that the component must be enabled in order for the setting to have any effect. +.. setting:: ADDONS + +ADDONS +------ + +Default: ``()`` + +A tuple containing paths to the add-ons enabled in your project. For more +information, see :ref:`topics-addons`. + .. setting:: AWS_ACCESS_KEY_ID AWS_ACCESS_KEY_ID @@ -964,16 +974,6 @@ some of them need to be enabled through a setting. For more information See the :ref:`extensions user guide ` and the :ref:`list of available extensions `. -.. setting:: INSTALLED_ADDONS - -INSTALLED_ADDONS ----------------- - -Default: ``()`` - -A tuple containing paths to the add-ons enabled in your project. For more -information, see :ref:`topics-addons`. - .. setting:: FEED_TEMPDIR FEED_TEMPDIR From 5e76464fbf1338bb791b4ae2ad97dd33f72dd0a1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 18:01:41 +0400 Subject: [PATCH 27/55] Fix a merge error. --- scrapy/interfaces.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index d8e5f9866..b8aa77ced 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -17,11 +17,6 @@ class ISpiderLoader(Interface): """Return the list of spiders names that can handle the given request""" -# ISpiderManager is deprecated, don't use it! -# An alias is kept for backwards compatibility. -ISpiderManager = ISpiderLoader - - class IAddon(Interface): """Scrapy add-on""" From 815af431209686018b1bc2cbb80fd946cfe93614 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 18:11:14 +0400 Subject: [PATCH 28/55] Remove load_module_or_object. --- scrapy/addons.py | 6 +++--- scrapy/utils/misc.py | 16 ---------------- tests/test_utils_misc/__init__.py | 13 ++++++------- 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 43a36e5e5..153de66d2 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -11,7 +11,7 @@ from zope.interface.verify import verifyObject from scrapy.interfaces import IAddon from scrapy.utils.conf import build_component_list -from scrapy.utils.misc import load_module_or_object +from scrapy.utils.misc import load_object @zope.interface.implementer(IAddon) @@ -252,8 +252,8 @@ class AddonManager(Mapping): """ if isinstance(path, str): try: - obj = load_module_or_object(path) - except NameError: + obj = load_object(path) + except (ValueError, NameError, ImportError): raise NameError(f"Could not find add-on '{path}'") else: obj = path diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 8577cce02..d861c9ab6 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -72,22 +72,6 @@ def load_object(path: Union[str, Callable]) -> Any: return obj -def load_module_or_object(path): - """Load python module or (non-module) object from given path. - - Path can be both a Python or a file path. - """ - try: - return import_module(path) - except ImportError: - pass - try: - return load_object(path) - except (ValueError, NameError, ImportError): - pass - raise NameError(f"Could not load '{path}'") - - def walk_modules(path): """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 7932ca04c..4f6e0d02c 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -8,7 +8,6 @@ from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, create_instance, - load_module_or_object, load_object, rel_has_nofollow, set_environ, @@ -31,17 +30,17 @@ class UtilsMiscTestCase(unittest.TestCase): obj = load_object("scrapy.utils.misc.load_object") self.assertIs(obj, load_object) + def test_load_object_module(self): + testmod = load_object(__name__ + ".testmod") + self.assertTrue(hasattr(testmod, "TESTVAR")) + obj = load_object("scrapy.utils.misc.load_object") + self.assertIs(obj, load_object) + def test_load_object_exceptions(self): self.assertRaises(ImportError, load_object, "nomodule999.mod.function") self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999") self.assertRaises(TypeError, load_object, {}) - def test_load_module_or_object(self): - testmod = load_module_or_object(__name__ + ".testmod") - self.assertTrue(hasattr(testmod, "TESTVAR")) - obj = load_object("scrapy.utils.misc.load_object") - self.assertIs(obj, load_object) - def test_walk_modules(self): mods = walk_modules("tests.test_utils_misc.test_walk_modules") expected = [ From 27f5f3513437466d4e67df7dc3e0e57f959cc03b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 18:30:33 +0400 Subject: [PATCH 29/55] More quick doc fixes. --- docs/topics/addons.rst | 131 ++------------------------------------- docs/topics/settings.rst | 6 +- 2 files changed, 7 insertions(+), 130 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 25d7da50b..523f6e86e 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -40,10 +40,10 @@ This is an example where an internal add-on and two third-party add-ons (in this case with one requiring no configuration) are enabled/configured in a project's ``settings.py``:: - ADDONS = ( - 'httpcache', - 'path.to.some.addon', - ) + ADDONS = { + 'httpcache': 0, + 'path.to.some.addon': 0, + } HTTPCACHE = { 'expiration_secs': 60, @@ -385,126 +385,3 @@ Forward to other add-ons depending on Python version:: _addon = 'path.to.addon' else: _addon = 'path.to.other.addon' - - -Built-in add-on reference -========================= - -Scrapy comes with gateway add-ons that you can use to configure the built-in -middlewares and extensions. For example, to activate and configure the -:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`, instead -of placing this in your ``settings.py``:: - - HTTPCACHE_ENABLED = True - HTTPCACHE_EXPIRATION_SECS = 60 - HTTPCACHE_IGNORE_HTTP_CODES = [404] - -you can also use the add-on framework:: - - ADDONS = ( - # ..., - 'httpcache', - ) - - HTTPCACHE = { - 'expiration_secs': 60, - 'ignore_http_codes': [404], - } - -Note that you *must* enable built-in addons by placing them in your -``ADDONS`` setting before you can use them for configuring built-in -components. I.e., configuring the ``HTTPCACHE`` setting will have no effect -when ``httpcache`` is not listed in ``ADDONS``. - -In general, the add-on names match the lowercase name of the component, with its -type suffix removed (i.e. the add-on configuring the -:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` is called -``httpcache``), and the configuration option names match the names of the -settings they map to, with the component prefix removed (i.e. -``expiration_secs`` maps to :setting:`HTTPCACHE_EXPIRATION_SECS`, as above). -The available add-ons are: - - -+--------------------------------------+--------------------------------------+ -| Add-on | Notes | -+======================================+======================================+ -| **Spider middlewares** | -+--------------------------------------+--------------------------------------+ -| depth (:class:`~scrapy.spidermi\ | | -| ddlewares.depth.DepthMiddleware`) | | -+--------------------------------------+--------------------------------------+ -| httperror (:class:`~scrapy.spid\ | | -| ermiddlewares.httperror.HttpErrorMi\ | | -| ddleware`) | | -+--------------------------------------+--------------------------------------+ -| offsite (:class:`~scrapy.spid\ | | -| ermiddlewares.offsite.OffsiteMiddle\ | | -| ware`) | | -+--------------------------------------+--------------------------------------+ -| referer (:class:`~scrapy.spid\ | | -| ermiddlewares.referer.RefererMiddle\ | | -| ware`) | | -+--------------------------------------+--------------------------------------+ -| urllength (:class:`~scrapy.spid\ | | -| ermiddlewares.urllength.UrlLengthMi\ | | -| ddleware`) | | -+--------------------------------------+--------------------------------------+ -| **Downloader middlewares** | -+--------------------------------------+--------------------------------------+ -| ajaxcrawl (:class:`~scrapy.download\ | | -| ermiddlewares.ajaxcrawl.AjaxCrawlMi\ | | -| ddleware`) | | -+--------------------------------------+--------------------------------------+ -| chunked (:class:`~scrapy.download\ | | -| ermiddlewares.chunked.ChunkedTrans\ | | -| ferMiddleware`) | | -+--------------------------------------+--------------------------------------+ -| cookies (:class:`~scrapy.download\ | | -| ermiddlewares.cookies.CookiesMiddle\ | | -| ware`) | | -+--------------------------------------+--------------------------------------+ -| defaultheaders (:class:`~scrapy.down\| Every configuration entry is treated | -| loadermiddlewares.defaultheaders.Def\| as a default header. | -| aultHeadersMiddleware`) | | -+--------------------------------------+--------------------------------------+ -| **Extensions** | -+--------------------------------------+--------------------------------------+ -| autothrottle | Installing sets | -| (:ref:`topics-autothrottle`) | :setting:`AUTOTHROTTLE_ENABLED` to | -| | ``True``. | -+--------------------------------------+--------------------------------------+ -| corestats (:class:`~scrapy.exten\ | | -| sions.corestats.CoreStats`) | | -+--------------------------------------+--------------------------------------+ -| closespider (:class:`~scrapy.exten\ | | -| sions.closespider.CloseSpider`) | | -+--------------------------------------+--------------------------------------+ -| debugger (:class:`~scrapy.exten\ | | -| sions.debug.Debugger`) | | -+--------------------------------------+--------------------------------------+ -| feedexport (:ref:`topics-feed-expor\ | | -| ts`) | | -+--------------------------------------+--------------------------------------+ -| logstats (:class:`~scrapy.exten\ | | -| sions.logstats.LogStats`) | | -+--------------------------------------+--------------------------------------+ -| memdebug (:class:`~scrapy.exten\ | Installing sets | -| sions.memdebug.MemoryDebugger`) | :setting:`MEMDEBUG_ENABLED` to | -| | ``True``. | -+--------------------------------------+--------------------------------------+ -| memusage (:class:`~scrapy.exten\ | Installing sets | -| sions.memusage.MemoryUsage`) | :setting:`MEMUSAGE_ENABLED` to | -| | ``True``. | -+--------------------------------------+--------------------------------------+ -| spiderstate (:class:`~scrapy.exten\ | | -| sions.spiderstate.SpiderState`) | | -+--------------------------------------+--------------------------------------+ -| stacktracedump (:class:`~scrapy.ext\ | | -| ensions.debug.StackTraceDump`) | | -+--------------------------------------+--------------------------------------+ -| statsmailer (:class:`~scrapy.exten\ | | -| sions.statsmailer.StatsMailer`) | | -+--------------------------------------+--------------------------------------+ -| telnetconsole (:ref:`topics-telnet\ | | -| console`) | | -+--------------------------------------+--------------------------------------+ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index aa09abbd5..143002360 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -206,10 +206,10 @@ component must be enabled in order for the setting to have any effect. ADDONS ------ -Default: ``()`` +Default: ``{}`` -A tuple containing paths to the add-ons enabled in your project. For more -information, see :ref:`topics-addons`. +A dict containing paths to the add-ons enabled in your project and their +priorities. For more information, see :ref:`topics-addons`. .. setting:: AWS_ACCESS_KEY_ID From c7f78a8305fd7c08705cc0125e0f6e99dbaa0d10 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 14 Jun 2023 20:43:18 +0400 Subject: [PATCH 30/55] Revert "Remove load_module_or_object." This reverts commit 815af431209686018b1bc2cbb80fd946cfe93614. --- scrapy/addons.py | 6 +++--- scrapy/utils/misc.py | 16 ++++++++++++++++ tests/test_utils_misc/__init__.py | 13 +++++++------ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 153de66d2..43a36e5e5 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -11,7 +11,7 @@ from zope.interface.verify import verifyObject from scrapy.interfaces import IAddon from scrapy.utils.conf import build_component_list -from scrapy.utils.misc import load_object +from scrapy.utils.misc import load_module_or_object @zope.interface.implementer(IAddon) @@ -252,8 +252,8 @@ class AddonManager(Mapping): """ if isinstance(path, str): try: - obj = load_object(path) - except (ValueError, NameError, ImportError): + obj = load_module_or_object(path) + except NameError: raise NameError(f"Could not find add-on '{path}'") else: obj = path diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index d861c9ab6..8577cce02 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -72,6 +72,22 @@ def load_object(path: Union[str, Callable]) -> Any: return obj +def load_module_or_object(path): + """Load python module or (non-module) object from given path. + + Path can be both a Python or a file path. + """ + try: + return import_module(path) + except ImportError: + pass + try: + return load_object(path) + except (ValueError, NameError, ImportError): + pass + raise NameError(f"Could not load '{path}'") + + def walk_modules(path): """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 4f6e0d02c..7932ca04c 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -8,6 +8,7 @@ from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, create_instance, + load_module_or_object, load_object, rel_has_nofollow, set_environ, @@ -30,17 +31,17 @@ class UtilsMiscTestCase(unittest.TestCase): obj = load_object("scrapy.utils.misc.load_object") self.assertIs(obj, load_object) - def test_load_object_module(self): - testmod = load_object(__name__ + ".testmod") - self.assertTrue(hasattr(testmod, "TESTVAR")) - obj = load_object("scrapy.utils.misc.load_object") - self.assertIs(obj, load_object) - def test_load_object_exceptions(self): self.assertRaises(ImportError, load_object, "nomodule999.mod.function") self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999") self.assertRaises(TypeError, load_object, {}) + def test_load_module_or_object(self): + testmod = load_module_or_object(__name__ + ".testmod") + self.assertTrue(hasattr(testmod, "TESTVAR")) + obj = load_object("scrapy.utils.misc.load_object") + self.assertIs(obj, load_object) + def test_walk_modules(self): mods = walk_modules("tests.test_utils_misc.test_walk_modules") expected = [ From 2f9ebb66c32fc11eee86261d55e232b7e12f6a09 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 15 Jun 2023 17:00:38 +0400 Subject: [PATCH 31/55] Remove some dead code/docs. --- docs/topics/addons.rst | 32 ++------------------------ scrapy/addons.py | 26 --------------------- tests/test_addons/__init__.py | 43 ++++++++++++----------------------- 3 files changed, 16 insertions(+), 85 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 523f6e86e..35e40ca15 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -17,16 +17,7 @@ Activating and configuring add-ons Add-ons and their configuration live in Scrapy's :class:`~scrapy.addons.AddonManager`. During Scrapy's start-up process, and only then, the add-on manager will read a list of enabled add-ons and their -configurations from your settings. There are two places where you can provide -the paths to add-ons you want to enable: - -* the ``ADDONS`` setting, and -* the ``scrapy.cfg`` file. - -As Scrapy settings can be modified from many places, e.g. in a project's -``settings.py``, in a Spider's ``custom_settings`` attribute, or from the -command line, using the ``ADDONS`` setting is the preferred way to -manage add-ons. +configurations from your ``ADDONS`` setting. The ``ADDONS`` setting a tuple in which every item is a path to an add-on. The path can be both a Python or a file path. While more precise, it is @@ -54,30 +45,11 @@ case with one requiring no configuration) are enabled/configured in a project's 'some_config': True, } -It is also possible to manage add-ons from ``scrapy.cfg``. While the syntax is -a little friendlier, be aware that this file, and therefore the configuration in -it, is not bound to a particular Scrapy project. While this should not pose a -problem when you use the project on your development machine only, a common -stumbling block is that ``scrapy.cfg`` is not deployed via ``scrapyd-deploy``. - -In ``scrapy.cfg``, section names, prepended with ``addon:``, replace the -dictionary keys. I.e., the configuration from above would look like this: - -.. code-block:: cfg - - [addon:httpcache] - expiration_secs = 60 - ignore_http_codes = 404,405 - - [addon:path.to.some.addon] - some_config = true - - Enabling and configuring add-ons within Python code --------------------------------------------------- The :class:`~scrapy.addons.AddonManager` will only read from Scrapy's settings -and from ``scrapy.cfg`` *at the beginning* of Scrapy's start-up process. +*at the beginning* of Scrapy's start-up process. Afterwards, i.e. as soon as the :class:`~scrapy.addons.AddonManager` is populated, changing the ``ADDONS`` setting or any of the add-on configuration dictionary settings will have no effect. diff --git a/scrapy/addons.py b/scrapy/addons.py index 43a36e5e5..66463afdb 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -261,32 +261,6 @@ class AddonManager(Mapping): obj = AddonManager.get_addon(obj._addon) return obj - def load_dict(self, addonsdict): - """Load add-ons and configurations from given dictionary. - - Each add-on should be an entry in the dictionary, where the key - corresponds to the add-on path. The value should be a dictionary - representing the add-on configuration. - - Example add-on dictionary:: - - addonsdict = { - 'path.to.addon1': { - 'setting1': 'value', - 'setting2': 42, - }, - 'path/to/addon2.py': { - 'addon2setting': True, - }, - } - - :param addonsdict: dictionary where keys correspond to add-on paths \ - and values correspond to their configuration - :type addonsdict: ``dict`` - """ - for addonpath, addoncfg in addonsdict.items(): - self.add(addonpath, addoncfg) - def load_settings(self, settings): """Load add-ons and configurations from settings object. diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index 451ffed0f..d1c17a044 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -1,7 +1,6 @@ import itertools import unittest import warnings -from collections import OrderedDict from unittest import mock from zope.interface import directlyProvides @@ -178,24 +177,7 @@ class AddonManagerTest(unittest.TestCase): x._addon._addon = addons.GoodAddon("inner") self.assertIs(self.manager.get_addon(x), x._addon._addon) - def test_load_dict_load_settings(self): - def _test_load_method(func, *args, **kwargs): - manager = AddonManager() - getattr(manager, func)(*args, **kwargs) - self.assertCountEqual(manager, ["GoodAddon", "AddonModule"]) - self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) - self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) - self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") - self.assertEqual(manager["AddonModule"], addonmod) - self.assertIn("key", manager.configs["AddonModule"]) - self.assertEqual(manager.configs["AddonModule"]["key"], "val1") - - addonsdict = { - "tests.test_addons.addonmod": {"key": "val1"}, - "tests.test_addons.addons.GoodAddon": {"key": "val2"}, - } - _test_load_method("load_dict", addonsdict) - + def test_load_settings(self): settings = BaseSettings() settings.set( "ADDONS", @@ -203,25 +185,28 @@ class AddonManagerTest(unittest.TestCase): ) settings.set("ADDONMODULE", {"key": "val1"}) settings.set("GOODADDON", {"key": "val2"}) - _test_load_method("load_settings", settings) - - def test_load_dict_load_settings_order(self): - def _test_load_method(expected_order, func, *args, **kwargs): - manager = AddonManager() - getattr(manager, func)(*args, **kwargs) - self.assertEqual(list(manager.keys()), expected_order) + manager = AddonManager() + manager.load_settings(settings) + self.assertCountEqual(manager, ["GoodAddon", "AddonModule"]) + self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) + self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) + self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") + self.assertEqual(manager["AddonModule"], addonmod) + self.assertIn("key", manager.configs["AddonModule"]) + self.assertEqual(manager.configs["AddonModule"]["key"], "val1") + def test_load_settings_order(self): # Get three addons named 0, 1, 2 addonlist = [addons.GoodAddon(str(x)) for x in range(3)] # Test both methods for every possible mutation for ordered_addons in itertools.permutations(addonlist): expected_order = [a.name for a in ordered_addons] - addonsdict = OrderedDict((a, {}) for a in ordered_addons) - _test_load_method(expected_order, "load_dict", addonsdict) settings = BaseSettings( {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} ) - _test_load_method(expected_order, "load_settings", settings) + manager = AddonManager() + manager.load_settings(settings) + self.assertEqual(list(manager.keys()), expected_order) def test_enabled_disabled(self): manager = AddonManager() From 0258c87dab6d7336263ff53c7747f6400d8cd527 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 15 Jun 2023 18:40:59 +0400 Subject: [PATCH 32/55] Add typing for Crawler.addons. --- scrapy/crawler.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 5e7499f9c..fdcc6354d 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,7 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Optional, Type, Union +from typing import TYPE_CHECKING, Optional, Set, Type, Union from twisted.internet import defer from zope.interface.exceptions import DoesNotImplement @@ -57,7 +57,7 @@ class Crawler: spidercls: Type[Spider], settings: Union[None, dict, Settings] = None, init_reactor: bool = False, - addons=None, + addons: Optional[AddonManager] = None, ): if isinstance(spidercls, Spider): raise ValueError("The spidercls argument must be a class, not an object") @@ -69,7 +69,7 @@ class Crawler: self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) - self.addons = addons if addons is not None else AddonManager() + self.addons: AddonManager = addons if addons is not None else AddonManager() self.addons.load_settings(self.settings) self.addons.update_addons() self.addons.check_dependency_clashes() @@ -199,14 +199,14 @@ class CrawlerRunner: ) return loader_cls.from_settings(settings.frozencopy()) - def __init__(self, settings=None, addons=None): + def __init__(self, settings=None, addons: Optional[AddonManager] = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings - self.addons = addons + self.addons: Optional[AddonManager] = addons self.spider_loader = self._get_spider_loader(settings) - self._crawlers = set() - self._active = set() + self._crawlers: Set[Crawler] = set() + self._active: Set[defer.Deferred] = set() self.bootstrap_failed = False @property @@ -285,7 +285,7 @@ class CrawlerRunner: def _create_crawler(self, spidercls): if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings, self.addons) + return Crawler(spidercls, self.settings, addons=self.addons) def stop(self): """ @@ -331,7 +331,12 @@ class CrawlerProcess(CrawlerRunner): process. See :ref:`run-from-script` for an example. """ - def __init__(self, settings=None, install_root_handler=True, addons=None): + def __init__( + self, + settings=None, + install_root_handler: bool = True, + addons: Optional[AddonManager] = None, + ): super().__init__(settings, addons) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) From f582246d7b0a70c9ace5642002c30040f7182d7c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 15 Jun 2023 18:49:08 +0400 Subject: [PATCH 33/55] More doc fixes. --- docs/index.rst | 2 +- docs/topics/addons.rst | 33 +++++++++++---------------------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index ace5a2eb7..8798aebd1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -237,7 +237,7 @@ Extending Scrapy Understand the Scrapy architecture. :doc:`topics/addons` - Enable and configure built-in and third-party extensions. + Enable and configure third-party extensions. :doc:`topics/downloader-middleware` Customize how pages get requested and downloaded. diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 35e40ca15..5d1a4f753 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -19,26 +19,18 @@ Add-ons and their configuration live in Scrapy's only then, the add-on manager will read a list of enabled add-ons and their configurations from your ``ADDONS`` setting. -The ``ADDONS`` setting a tuple in which every item is a path to an -add-on. The path can be both a Python or a file path. While more precise, it is -not necessary to specify the full add-on Python path if it is either built into -Scrapy or lives in your project's ``addons`` submodule. +The ``ADDONS`` setting is a dict in which every key is an addon class or its +import path and the vaoue is its priority. The configuration of an add-on, if necessary at all, is stored as a dictionary setting whose name is the uppercase add-on name. -This is an example where an internal add-on and two third-party add-ons (in this -case with one requiring no configuration) are enabled/configured in a project's -``settings.py``:: +This is an example where two add-ons (in this case with one requiring no +configuration) are enabled/configured in a project's ``settings.py``:: ADDONS = { - 'httpcache': 0, - 'path.to.some.addon': 0, - } - - HTTPCACHE = { - 'expiration_secs': 60, - 'ignore_http_codes': [404, 405], + 'path.to.someaddon': 0, + path.to.someaddon2: 1, } SOMEADDON = { @@ -67,7 +59,7 @@ add-ons framework, e.g.: * :meth:`~scrapy.addons.AddonManager.enable` and :meth:`~scrapy.addons.AddonManager.disable` methods, * the :attr:`~scrapy.addons.AddonManager.configs` dictionary which holds the - configuration of all add-ons + configuration of all add-ons. In this example, we ensure that the ``httpcache`` add-on is loaded, and that its ``expiration_secs`` configuration is set to ``60``:: @@ -88,9 +80,9 @@ Python object up the developer. Examples: * for a small pipeline, the add-on interface could be implemented in the same class that also implements the ``open/close_spider`` and ``process_item`` - callbacks + callbacks, * for larger add-ons, or for clearer structure, the interface could be provided - by a stand-alone module + by a stand-alone module. The absolute minimum interface consists of two attributes: @@ -137,9 +129,7 @@ crawling process: This method is called immediately before :meth:`update_settings`, and should be used to enable and configure other *add-ons* only. - When using this callback, be aware that there is no guarantee in which order - the :meth:`update_addons` callbacks of enabled add-ons will be called. - Add-ons that are added to the :class:`~scrapy.addons.AddonManager` during + Add-ons that are added to the :class:`~scrapy.addons.AddonManager` during this callback will also have their :meth:`update_addons` method called. :param config: Configuration of this add-on @@ -244,8 +234,7 @@ Check dependencies:: import boto except ImportError: raise RuntimeError("myaddon requires the boto library") - else: - self.export_config(config, settings) + self.export_config(config, settings) Enable a component that lives relative to the add-on (see :ref:`topics-api-settings`):: From 79bf8b1f2e8eae5dfb02443b4be5d474502793ca Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 15 Jun 2023 19:23:28 +0400 Subject: [PATCH 34/55] Test cleanup. --- tests/test_addons/__init__.py | 2 +- tests/test_crawl.py | 6 ++---- tests/test_middleware.py | 12 ------------ 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py index d1c17a044..fa3f706e5 100644 --- a/tests/test_addons/__init__.py +++ b/tests/test_addons/__init__.py @@ -198,7 +198,7 @@ class AddonManagerTest(unittest.TestCase): def test_load_settings_order(self): # Get three addons named 0, 1, 2 addonlist = [addons.GoodAddon(str(x)) for x in range(3)] - # Test both methods for every possible mutation + # Test for every possible ordering for ordered_addons in itertools.permutations(addonlist): expected_order = [a.name for a in ordered_addons] settings = BaseSettings( diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 1e97863b0..920e5f4ae 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -421,10 +421,8 @@ with multiples lines addonmgr.add(FailedCheckAddon()) crawler = get_crawler(SimpleSpider) crawler.addons = addonmgr - # Doesn't work in 'precise' test environment: - # with self.assertRaises(ValueError): - # yield crawler.crawl() - yield self.assertFailure(crawler.crawl(), ValueError) + with self.assertRaises(ValueError): + yield crawler.crawl() class CrawlSpiderTestCase(TestCase): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index fd57d846e..00ff746ee 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -87,15 +87,3 @@ class MiddlewareManagerTest(unittest.TestCase): mwman = TestMiddlewareManager.from_settings(settings) classes = [x.__class__ for x in mwman.middlewares] self.assertEqual(classes, [M1, M3]) - - def test_instances_from_settings(self): - settings = Settings() - - class InstanceTestMiddlewareManager(MiddlewareManager): - @classmethod - def _get_mwlist_from_settings(cls, settings): - return ["tests.test_middleware.M1", M2] - - mwman = InstanceTestMiddlewareManager.from_settings(settings) - self.assertIsInstance(mwman.middlewares[0], M1) - self.assertIsInstance(mwman.middlewares[1], M2) From 54287f733972dcaf13706664ba89ab1c52e3a290 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 15 Jun 2023 19:53:34 +0400 Subject: [PATCH 35/55] Docs cleanup. --- scrapy/addons.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 66463afdb..aced6092a 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -30,9 +30,7 @@ class Addon(object): component_type = None """Component setting into which to export via :meth:`export_component`. Can - be any of the dictionary-like component setting names (e.g. - ``DOWNLOADER_MIDDLEWARES``) or any of their abbreviations in - :attr:`~scrapy.addons.COMPONENT_TYPE_ABBR`. If ``None``, + be any of the dictionary-like component setting names. If ``None``, :meth:`export_component` will do nothing. """ @@ -50,9 +48,8 @@ class Addon(object): component = None """Component to be inserted via :meth:`export_component`. This can be anything that can be used in the dictionary-like component settings, i.e. - a class path, a class, or an instance. If ``None``, it is assumed that the - add-on itself is also provides the component interface, and ``self`` will be - used. + a class path or a class. If ``None``, it is assumed that the add-on itself + also provides the component interface, and ``self`` will be used. """ settings_prefix = None @@ -241,13 +238,13 @@ class AddonManager(Mapping): def get_addon(path): """Get an add-on object by its Python or file path. - ``path`` is assumed to be either a Python or a file path of a Scrapy - add-on. If the object or module pointed to by ``path`` has an attribute - named ``_addon`` that attribute will be assumed to be the add-on. - :meth:`get_addon` will keep following ``_addon`` attributes until it - finds an object that does not have an attribute named ``_addon``. + ``path`` is assumed to be an import path of an add-on. If the object or + module pointed to by ``path`` has an attribute named ``_addon`` that + attribute will be assumed to be the add-on. :meth:`get_addon` will keep + following ``_addon`` attributes until it finds an object that does not + have an attribute named ``_addon``. - :param path: Python or file path to an add-on + :param path: Import path of an add-on :type path: ``str`` """ if isinstance(path, str): @@ -366,7 +363,7 @@ class AddonManager(Mapping): elif addon in self._disable_on_add: self._disable_on_add.remove(addon) else: - raise ValueError("Add-ons need to be added before they can be " "enabled") + raise ValueError("Add-ons need to be added before they can be enabled") @property def disabled(self): From e7124447f7e86ee93ad78ffd0ebfa6acbebc73ce Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 26 Jun 2023 16:57:46 +0400 Subject: [PATCH 36/55] Remove unneeded code. --- docs/topics/addons.rst | 241 ++------------------- scrapy/addons.py | 347 ++++-------------------------- scrapy/cmdline.py | 5 +- scrapy/crawler.py | 23 +- scrapy/interfaces.py | 21 +- scrapy/utils/conf.py | 12 +- scrapy/utils/misc.py | 18 +- scrapy/utils/test.py | 3 +- tests/test_addons.py | 72 +++++++ tests/test_addons/__init__.py | 342 ----------------------------- tests/test_addons/addonmod.py | 16 -- tests/test_addons/addons.py | 33 --- tests/test_crawl.py | 1 - tests/test_crawler.py | 27 --- tests/test_utils_misc/__init__.py | 7 - tests/test_utils_misc/testmod.py | 1 - 16 files changed, 140 insertions(+), 1029 deletions(-) create mode 100644 tests/test_addons.py delete mode 100644 tests/test_addons/__init__.py delete mode 100644 tests/test_addons/addonmod.py delete mode 100644 tests/test_addons/addons.py delete mode 100644 tests/test_utils_misc/testmod.py diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 5d1a4f753..6a9590f33 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -15,12 +15,13 @@ Activating and configuring add-ons ================================== Add-ons and their configuration live in Scrapy's -:class:`~scrapy.addons.AddonManager`. During Scrapy's start-up process, and -only then, the add-on manager will read a list of enabled add-ons and their -configurations from your ``ADDONS`` setting. +:class:`~scrapy.addons.AddonManager`. During a :class:`~scrapy.crawler.Crawler` +initialization the add-on manager will read a list of enabled add-ons from your +``ADDONS`` setting and their optional configuration from the respective +settings. The ``ADDONS`` setting is a dict in which every key is an addon class or its -import path and the vaoue is its priority. +import path and the value is its priority. The configuration of an add-on, if necessary at all, is stored as a dictionary setting whose name is the uppercase add-on name. @@ -37,66 +38,17 @@ configuration) are enabled/configured in a project's ``settings.py``:: 'some_config': True, } -Enabling and configuring add-ons within Python code ---------------------------------------------------- - -The :class:`~scrapy.addons.AddonManager` will only read from Scrapy's settings -*at the beginning* of Scrapy's start-up process. -Afterwards, i.e. as soon as the :class:`~scrapy.addons.AddonManager` is -populated, changing the ``ADDONS`` setting or any of the add-on -configuration dictionary settings will have no effect. - -If you want to enable, disable, or configure add-ons in Python code, for example -when writing your own add-on, you will have to use the -:class:`~scrapy.addons.AddonManager`. You can access the add-on manager through -either ``crawler.addons`` or, if you are writing an add-on, through the -``addons`` argument of the :meth:`update_addons` callback. The add-on manager -provides many useful methods and attributes to facilitate interacting with the -add-ons framework, e.g.: - -* an :meth:`~scrapy.addons.AddonManager.add` method to load add-ons, -* the :attr:`~scrapy.addons.AddonManager.enabled` list of enabled add-ons, -* :meth:`~scrapy.addons.AddonManager.enable` and - :meth:`~scrapy.addons.AddonManager.disable` methods, -* the :attr:`~scrapy.addons.AddonManager.configs` dictionary which holds the - configuration of all add-ons. - -In this example, we ensure that the ``httpcache`` add-on is loaded, and that -its ``expiration_secs`` configuration is set to ``60``:: - - # addons is an instance of AddonManager - if 'httpcache' not in addons: - addons.add('httpcache', {'expiration_secs': 60}) - else: - addons.configs['httpcache']['expiration_secs'] = 60 - Writing your own add-ons ======================== -Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*. -The interface is enforced through ``zope.interface``. This leaves the choice of -Python object up the developer. Examples: - -* for a small pipeline, the add-on interface could be implemented in the same - class that also implements the ``open/close_spider`` and ``process_item`` - callbacks, -* for larger add-ons, or for clearer structure, the interface could be provided - by a stand-alone module. - -The absolute minimum interface consists of two attributes: +Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*: .. attribute:: name string with add-on name -.. attribute:: version - - version string (PEP-404, e.g. ``'1.0.1'``) - -Of course, stating just these two attributes will not get you very far. Add-ons -can provide three callback methods that are called at various stages before the -crawling process: + :type: ``str`` .. method:: update_settings(config, settings) @@ -124,75 +76,18 @@ crawling process: :param crawler: Fully initialized Scrapy crawler :type crawler: :class:`~scrapy.crawler.Crawler` -.. method:: update_addons(config, addons) - - This method is called immediately before :meth:`update_settings`, and should - be used to enable and configure other *add-ons* only. - - Add-ons that are added to the :class:`~scrapy.addons.AddonManager` during - this callback will also have their :meth:`update_addons` method called. - - :param config: Configuration of this add-on - :type config: ``dict`` - - :param addons: Add-on manager holding all loaded add-ons - :type addons: :class:`~scrapy.addons.AddonManager` - -Additionally, add-ons may (and should, where appropriate) provide one or more -attributes that can be used for limited automated detection of possible -dependency clashes: - -.. attribute:: requires - - list of built-in or custom components needed by this add-on, as strings. - -.. attribute:: modifies - - list of built-in or custom components whose functionality is affected or - replaced by this add-on (a custom HTTP cache should list ``httpcache`` here) - -.. attribute:: provides - - list of components provided by this add-on (e.g. ``mongodb`` for an - extension that provides generic read/write access to a MongoDB database) - -The entries in the :attr:`requires` and :attr:`modifies` attributes can be add-on -names or components from other add-ons' :attr:`provides` attribute. You can -specify :pep:`440`-style information about required versions. Examples:: - - requires = ['httpcache'] - requires = ['otheraddon >= 2.0', 'yetanotheraddon'] - -The Python object or module that is pointed to by an add-on path (e.g. given in -the ``ADDONS`` setting, or given to -:meth:`~scrapy.addons.AddonManager.add`) does not necessarily have to be an -add-on. Instead, it can provide an ``_addon`` attribute. This attribute can be -either an add-on or another add-on path. - Add-on base class ================= Scrapy comes with a built-in base class for add-ons which provides some -convenience functionality: - -* basic settings can be exported via :meth:`~scrapy.addons.Addon.export_basics`, - configurable via :attr:`~scrapy.addons.Addon.basic_settings`. -* a single component (e.g. an item pipeline or a downloader middleware) can be - inserted into Scrapy's settings via - :meth:`~scrapy.addons.Addon.export_component`, configurable via - :attr:`~scrapy.addons.Addon.component_type`, - :attr:`~scrapy.addons.Addon.component_key`, - :attr:`~scrapy.addons.Addon.component`, and the ``order`` key in - :attr:`~scrapy.addons.Addon.default_config`. -* the add-on configuration can be exposed into Scrapy's settings via - :meth:`~scrapy.addons.Addon.export_config`, configurable via - :attr:`~scrapy.addons.Addon.default_config`, - :attr:`~scrapy.addons.Addon.config_mapping`, and - :attr:`~scrapy.addons.Addon.settings_prefix`. +convenience functionality: the add-on configuration can be exposed into +Scrapy's settings via :meth:`~scrapy.addons.Addon.export_config`, configurable +via :attr:`~scrapy.addons.Addon.default_config` and +:attr:`~scrapy.addons.Addon.config_mapping`. By default, the base add-on class will expose the add-on configuration into -Scrapy's settings namespace, in caps and with the add-on name prepended. It is +Scrapy's settings namespace, in upper case. It is easy to write your own functionality while still being able to use the convenience functions by overwriting :meth:`~scrapy.addons.Addon.update_settings`. @@ -213,13 +108,11 @@ Set some basic configuration using the :class:`Addon` base class:: class MyAddon(Addon): name = 'myaddon' - version = '1.0' - component = 'path.to.mypipeline' - component_type = 'ITEM_PIPELINES' - component_order = 200 - basic_settings = { - 'DNSCACHE_ENABLED': False, - } + + def update_settings(self, config, settings): + super().update_settings(settings) + settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 + settings["DNSCACHE_ENABLED"] = True Check dependencies:: @@ -227,78 +120,22 @@ Check dependencies:: class MyAddon(Addon): name = 'myaddon' - version = '1.0' def update_settings(self, config, settings): try: import boto except ImportError: raise RuntimeError("myaddon requires the boto library") - self.export_config(config, settings) - -Enable a component that lives relative to the add-on (see -:ref:`topics-api-settings`):: - - from scrapy.addons import Addon - - class MyAddon(Addon): - name = 'myaddon' - version = '1.0' - component = __name__ + '.downloadermw.coolmw' - component_type = 'DOWNLOADER_MIDDLEWARES' - component_order = 900 - -Instantiate components ad hoc:: - - from path.to.my.pipelines import MySQLPipeline - - class MyAddon(object): - name = 'myaddon' - version = '1.0' - - def update_settings(self, config, settings): - mysqlpl = MySQLPipeline(password=config['password']) - settings.set( - 'ITEM_PIPELINES', - {mysqlpl: 200}, - priority='addon', - ) - -Provide add-on interface along component interface:: - - class MyPipeline(object): - name = 'mypipeline' - version = '1.0' - - def process_item(self, item, spider): - # Do some processing here - return item - - def update_settings(self, config, settings): - settings.set( - 'ITEM_PIPELINES', - {self: 200}, - priority='addon', - ) - -Enable another addon (see :ref:`topics-api-addonmanager`):: - - class MyAddon(object): - name = 'myaddon' - version = '1.0' - - def update_addons(self, config, addons): - if 'httpcache' not in addons.enabled: - addons.add('httpcache', {'expiration_secs': 60}) + super().update_settings(settings) Check configuration of fully initialized crawler (see :ref:`topics-api-crawler`):: class MyAddon(object): name = 'myaddon' - version = '1.0' def update_settings(self, config, settings): + super().update_settings(settings) settings.set('DNSCACHE_ENABLED', False, priority='addon') def check_configuration(self, config, crawler): @@ -306,43 +143,3 @@ Check configuration of fully initialized crawler (see # The spider, some other add-on, or the user messed with the # DNS cache setting raise ValueError("myaddon is incompatible with DNS cache") - -Provide add-on interface through a module: - -.. code-block:: python - - name = "AddonModule" - version = "1.0" - - - class MyPipeline(object): - ... - - - class MyDownloaderMiddleware(object): - ... - - - def update_settings(config, settings): - settings.set( - "ITEM_PIPELINES", - {MyPipeline(): 200}, - priority="addon", - ) - settings.set( - "DOWNLOADER_MIDDLEWARES", - {MyDownloaderMiddleware(): 800}, - priority="addon", - ) - -Forward to other add-ons depending on Python version:: - - # This could be a Python module, say project/pipelines/mypipeline.py, but - # could also be done inside a class, etc. - import six - - if six.PY3: - # We're running Python 3 - _addon = 'path.to.addon' - else: - _addon = 'path.to.other.addon' diff --git a/scrapy/addons.py b/scrapy/addons.py index aced6092a..a54086fda 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,108 +1,27 @@ -import warnings -from collections import OrderedDict, defaultdict from collections.abc import Mapping -from inspect import isclass -from typing import Dict +from typing import Any, Dict, Iterator, Optional, OrderedDict -import zope.interface -from packaging.requirements import Requirement -from packaging.version import Version -from zope.interface.verify import verifyObject - -from scrapy.interfaces import IAddon from scrapy.utils.conf import build_component_list -from scrapy.utils.misc import load_module_or_object +from scrapy.utils.misc import load_object -@zope.interface.implementer(IAddon) class Addon(object): - basic_settings = None - """``dict`` of settings that will be exported via :meth:`export_basics`.""" + name: str default_config = None """``dict`` with default configuration.""" config_mapping = None """``dict`` with mappings from config names to setting names. The given - setting names will be taken as given, i.e. they will be neither prefixed - nor uppercased. + setting names will be taken as given, not uppercased. """ - component_type = None - """Component setting into which to export via :meth:`export_component`. Can - be any of the dictionary-like component setting names. If ``None``, - :meth:`export_component` will do nothing. - """ - - component_key = None - """Key to be used in the component dictionary setting when exporting via - :meth:`export_component`. This is only useful for the settings that have - no order, e.g. ``DOWNLOAD_HANDLERS`` or ``FEED_EXPORTERS``. - """ - - component_order = 0 - """Component order to use when not given in the add-on configuration. Has - no effect for component types that use :attr:`component_key`. - """ - - component = None - """Component to be inserted via :meth:`export_component`. This can be - anything that can be used in the dictionary-like component settings, i.e. - a class path or a class. If ``None``, it is assumed that the add-on itself - also provides the component interface, and ``self`` will be used. - """ - - settings_prefix = None - """Prefix with which the add-on configuration will be exported into the - global settings namespace via :meth:`export_config`. If ``None``, - :attr:`name` will be used. If ``False``, no configuration will be exported. - """ - - def export_component(self, config, settings): - """Export the component in :attr:`component` into the dictionary-like - component setting derived from :attr:`component_type`. - - Where applicable, the order parameter of the component (i.e. the - dictionary value) will be retrieved from the ``order`` add-on - configuration value. - - :param config: Add-on configuration from which to read component order - :type config: ``dict`` - - :param settings: Settings object into which to export component - :type settings: :class:`~scrapy.settings.Settings` - """ - if self.component_type: - comp = self.component or self - if self.component_key: - # e.g. for DOWNLOAD_HANDLERS: {'http': 'myclass'} - k = self.component_key - v = comp - else: - # e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100} - k = comp - v = config.get("order", self.component_order) - settings[self.component_type].update({k: v}, "addon") - - def export_basics(self, settings): - """Export the :attr:`basic_settings` attribute into the settings object. - - All settings will be exported with ``addon`` priority (see - :ref:`topics-api-settings`). - - :param settings: Settings object into which to expose the basic settings - :type settings: :class:`~scrapy.settings.Settings` - """ - for setting, value in (self.basic_settings or {}).items(): - settings.set(setting, value, "addon") - def export_config(self, config, settings): - """Export the add-on configuration, all keys in caps and with - :attr:`settings_prefix` or :attr:`name` prepended, into the settings + """Export the add-on configuration, all keys in caps, into the settings object. For example, the add-on configuration ``{'key': 'value'}`` will export - the setting ``ADDONNAME_KEY`` with a value of ``value``. All settings + the setting ``KEY`` with a value of ``value``. All settings will be exported with ``addon`` priority (see :ref:`topics-api-settings`). @@ -112,11 +31,8 @@ class Addon(object): :param settings: Settings object into which to export the configuration :type settings: :class:`~scrapy.settings.Settings` """ - if self.settings_prefix is False: - return conf = self.default_config or {} conf.update(config) - prefix = self.settings_prefix or self.name # Since default exported config is case-insensitive (everything will be # uppercased), make mapped config case-insensitive as well conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()} @@ -124,14 +40,11 @@ class Addon(object): if key.lower() in conf_mapping: key = conf_mapping[key.lower()] else: - key = (prefix + "_" + key).upper() + key = key.upper() settings.set(key, val, "addon") def update_settings(self, config, settings): - """Export both the basic settings and the add-on configuration. I.e., - call :meth:`export_basics` and :meth:`export_config`. - - For more advanced add-ons, you may want to override this callback. + """Modifiy `settings` to enable and configure required components. :param config: Add-on configuration :type config: ``dict`` @@ -139,12 +52,21 @@ class Addon(object): :param settings: Crawler settings object :type settings: :class:`~scrapy.settings.Settings` """ - self.export_component(config, settings) - self.export_basics(settings) self.export_config(config, settings) + def check_configuration(self, config, crawler): + """Perform post-initialization checks on fully configured `crawler`. -class AddonManager(Mapping): + :param config: Add-on configuration + :type config: ``dict`` + + :param crawler: the fully-initialized crawler + :type crawler: :class:`~scrapy.crawler.Crawler` + """ + pass + + +class AddonManager(Mapping[str, Addon]): """This class facilitates loading and storing :ref:`topics-addons`. You can treat it like a read-only dictionary in which keys correspond to @@ -160,108 +82,42 @@ class AddonManager(Mapping): """ - def __init__(self): - self._addons = OrderedDict() - self.configs = {} - self._disable_on_add = [] + def __init__(self) -> None: + self._addons: OrderedDict[str, Addon] = OrderedDict[str, Addon]() + self.configs: Dict[str, Dict[str, Any]] = {} - def __getitem__(self, name): + def __getitem__(self, name: str) -> Addon: return self._addons[name] - def __delitem__(self, name): - del self._addons[name] - del self.configs[name] - - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self._addons) - def __len__(self): + def __len__(self) -> int: return len(self._addons) - def add(self, addon, config=None): + def add(self, addon: Any, config: Optional[Dict[str, Any]] = None): """Store an add-on. - If ``addon`` is a string, it will be treated as add-on path and passed - to :meth:`get_addon`. Otherwise, ``addon`` must be a Python object - implementing or providing Scrapy's add-on interface. The interface - will be enforced through ``zope.interface``'s ``verifyObject()``. - - If ``addon`` is a class, it will be instantiated. You can avoid this - (for example if you have implemented the add-on callbacks as class - methods) by declaring -- via ``zope.interface`` -- that your class - directly *provides* ``scrapy.interfaces.IAddon``. - :param addon: The add-on object (or path) to be stored - :type addon: Any Python object providing the add-on interface or ``str`` + :type addon: Python object, class or ``str`` :param config: The add-on configuration dictionary :type config: ``dict`` """ - addon = self.get_addon(addon) - if isclass(addon) and not IAddon.providedBy(addon): + if isinstance(addon, (type, str)): + addon = load_object(addon) + if isinstance(addon, type): addon = addon() - if not IAddon.providedBy(addon): - zope.interface.alsoProvides(addon, IAddon) - # zope.interface's exceptions are already quite helpful. Still, should - # we catch them and log an error message? - verifyObject(IAddon, addon) name = addon.name if name in self: raise ValueError(f"Addon '{name}' already loaded") self._addons[name] = addon self.configs[name] = config or {} - if name in self._disable_on_add: - self.configs[name]["_enabled"] = False - self._disable_on_add.remove(name) - - def remove(self, addon): - """Remove an add-on. - - If ``addon`` is the name of a stored add-on, that add-on will be - removed. Otherwise, you can use the argument in the same fashion as - in :meth:`add`. - - :param addon: The add-on name, object, or path to be removed - :type addon: Any Python object providing the add-on interface or ``str`` - """ - if addon in self: - del self[addon] - elif hasattr(addon, "name") and addon.name in self: - del self[addon.name] - else: - try: - del self[self.get_addon(addon).name] - except NameError: - raise KeyError - - @staticmethod - def get_addon(path): - """Get an add-on object by its Python or file path. - - ``path`` is assumed to be an import path of an add-on. If the object or - module pointed to by ``path`` has an attribute named ``_addon`` that - attribute will be assumed to be the add-on. :meth:`get_addon` will keep - following ``_addon`` attributes until it finds an object that does not - have an attribute named ``_addon``. - - :param path: Import path of an add-on - :type path: ``str`` - """ - if isinstance(path, str): - try: - obj = load_module_or_object(path) - except NameError: - raise NameError(f"Could not find add-on '{path}'") - else: - obj = path - if hasattr(obj, "_addon"): - obj = AddonManager.get_addon(obj._addon) - return obj def load_settings(self, settings): """Load add-ons and configurations from settings object. - This will invoke :meth:`get_addon` for every add-on path in the + This will load the addon for every add-on path in the ``ADDONS`` setting. For each of these add-ons, the configuration will be read from the dictionary setting whose name matches the uppercase add-on name. @@ -271,141 +127,12 @@ class AddonManager(Mapping): :type settings: :class:`~scrapy.settings.Settings` """ paths = build_component_list(settings["ADDONS"]) - addons = [self.get_addon(path) for path in paths] + addons = [load_object(path) for path in paths] configs = [settings.getdict(addon.name.upper()) for addon in addons] for a, c in zip(addons, configs): self.add(a, c) - def check_dependency_clashes(self) -> None: - """Check for incompatibilities in add-on dependencies. - - Add-ons can provide information about their dependencies in their - ``provides``, ``modifies`` and ``requires`` attributes. This method will - raise an ``ImportError`` if - - * a component required by an add-on is not provided by any other add-on, - or - * a component modified by an add-on is not provided by any other add-on, - or - * the same component is provided by more than one add-on, - - and warn when a component required by an add-on is modified by any other - add-on. - """ - # Collect all active add-ons and the components they provide - versions: Dict[str, Version] = {} - - def add_version(project_name, version): - if project_name in versions: - raise ImportError( - f"Component {project_name} provided by multiple add-ons" - ) - versions[project_name] = Version(version) - - for name in self: - ver = self[name].version - add_version(name, ver) - for provides_name in getattr(self[name], "provides", []): - add_version(provides_name, ver) - - # Collect all required and modified components - def compile_attribute_dict(attribute_name): - attrs = defaultdict(list) - for name in self: - for entry in getattr(self[name], attribute_name, []): - attrs[entry].append(name) - return attrs - - modified = compile_attribute_dict("modifies") - required = compile_attribute_dict("requires") - - req_or_mod = set(required.keys()).union(modified.keys()) - for reqstr in req_or_mod: - req = Requirement(reqstr) - if req.name not in versions or versions[req.name] not in req.specifier: - raise ImportError( - f"Add-ons {required[reqstr] + modified[reqstr]} require" - f" or modify missing component {reqstr}" - ) - - mod_and_req = set(required.keys()).intersection(modified.keys()) - for conflict in mod_and_req: - warnings.warn( - f"Component '{conflict}', required by add-ons {required[conflict]}," - f" is modified by add-ons {modified[conflict]}" - ) - - def disable(self, addon): - """Disable an add-on, i.e. prevent its callbacks from being called. - - If you disable an add-on before it is loaded, it will be disabled as - soon as it is added to the :class:`AddonManager`. - - :param addon: Name of the add-on to be disabled - :type addon: ``str`` - """ - if addon in self: - self.configs[addon]["_enabled"] = False - else: - self._disable_on_add.append(addon) - - def enable(self, addon): - """Re-enable a disabled add-on. - - Will raise ``ValueError`` if the add-on is neither already loaded nor - marked for being disabled on adding. - - :param addon: Name of the add-on to be enabled - :type addon: ``str`` - """ - if addon in self: - self.configs[addon]["_enabled"] = True - elif addon in self._disable_on_add: - self._disable_on_add.remove(addon) - else: - raise ValueError("Add-ons need to be added before they can be enabled") - - @property - def disabled(self): - """Names of disabled add-ons""" - return [ - a for a in self if not self.configs[a].get("_enabled", True) - ] + self._disable_on_add - - @property - def enabled(self): - """Names of enabled add-ons""" - return [a for a in self if self.configs[a].get("_enabled", True)] - - def _call_if_exists(self, obj, cbname, *args, **kwargs): - if obj is None: - return - try: - cb = getattr(obj, cbname) - except AttributeError: - return - else: - cb(*args, **kwargs) - - def _call_addon(self, addonname, cbname, *args, **kwargs): - if self.configs[addonname].get("_enabled", True): - self._call_if_exists( - self[addonname], cbname, self.configs[addonname], *args, **kwargs - ) - - def update_addons(self): - """Call ``update_addons()`` of all held add-ons. - - This will also call ``update_addons()`` of all add-ons that are added - last minute during the ``update_addons()`` routine of other add-ons. - """ - called_addons = set() - while called_addons != set(self): - for name in set(self).difference(called_addons): - called_addons.add(name) - self._call_addon(name, "update_addons", self) - - def update_settings(self, settings): + def update_settings(self, settings) -> None: """Call ``update_settings()`` of all held add-ons. :param settings: The :class:`~scrapy.settings.Settings` object to be \ @@ -413,13 +140,13 @@ class AddonManager(Mapping): :type settings: :class:`~scrapy.settings.Settings` """ for name in self: - self._call_addon(name, "update_settings", settings) + self[name].update_settings(self.configs[name], settings) - def check_configuration(self, crawler): + def check_configuration(self, crawler) -> None: """Call ``check_configuration()`` of all held add-ons. :param crawler: the fully-initialized crawler :type crawler: :class:`~scrapy.crawler.Crawler` """ for name in self: - self._call_addon(name, "check_configuration", crawler) + self[name].check_configuration(self.configs[name], crawler) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 1b579f10e..efc9b36ea 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -6,7 +6,6 @@ import sys from importlib.metadata import entry_points import scrapy -from scrapy.addons import AddonManager from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter from scrapy.crawler import CrawlerProcess from scrapy.exceptions import UsageError @@ -131,8 +130,6 @@ def execute(argv=None, settings=None): else: settings["EDITOR"] = editor - addons = AddonManager() - inproject = inside_project() cmds = _get_commands_dict(settings, inproject) cmdname = _pop_command_name(argv) @@ -156,7 +153,7 @@ def execute(argv=None, settings=None): opts, args = parser.parse_known_args(args=argv[1:]) _run_print_help(parser, cmd.process_options, args, opts) - cmd.crawler_process = CrawlerProcess(settings, addons=addons) + cmd.crawler_process = CrawlerProcess(settings) _run_print_help(parser, _run_command, cmd, args, opts) sys.exit(cmd.exitcode) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 0c9861dbd..a0cb368ed 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -58,7 +58,6 @@ class Crawler: spidercls: Type[Spider], settings: Union[None, dict, Settings] = None, init_reactor: bool = False, - addons: Optional[AddonManager] = None, ): if isinstance(spidercls, Spider): raise ValueError("The spidercls argument must be a class, not an object") @@ -70,10 +69,8 @@ class Crawler: self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) - self.addons: AddonManager = addons if addons is not None else AddonManager() + self.addons: AddonManager = AddonManager() self.addons.load_settings(self.settings) - self.addons.update_addons() - self.addons.check_dependency_clashes() self.addons.update_settings(self.settings) self.signals: SignalManager = SignalManager(self) @@ -200,15 +197,10 @@ class CrawlerRunner: ) return loader_cls.from_settings(settings.frozencopy()) - def __init__( - self, - settings: Union[Dict[str, Any], Settings, None] = None, - addons: Optional[AddonManager] = None, - ): + def __init__(self, settings: Union[Dict[str, Any], Settings, None] = None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings - self.addons: Optional[AddonManager] = addons self.spider_loader = self._get_spider_loader(settings) self._crawlers: Set[Crawler] = set() self._active: Set[defer.Deferred] = set() @@ -292,7 +284,7 @@ class CrawlerRunner: def _create_crawler(self, spidercls: Union[str, Type[Spider]]) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings, addons=self.addons) + return Crawler(spidercls, self.settings) def stop(self): """ @@ -338,13 +330,8 @@ class CrawlerProcess(CrawlerRunner): process. See :ref:`run-from-script` for an example. """ - def __init__( - self, - settings=None, - install_root_handler: bool = True, - addons: Optional[AddonManager] = None, - ): - super().__init__(settings, addons) + def __init__(self, settings=None, install_root_handler=True): + super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) self._initialized_reactor = False diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index b8aa77ced..9a2c5f170 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,4 +1,4 @@ -from zope.interface import Attribute, Interface +from zope.interface import Interface class ISpiderLoader(Interface): @@ -15,22 +15,3 @@ class ISpiderLoader(Interface): def find_by_request(request): """Return the list of spiders names that can handle the given request""" - - -class IAddon(Interface): - """Scrapy add-on""" - - name = Attribute("""Add-on name""") - version = Attribute("""Add-on version string (PEP440)""") - - # XXX: Can methods be declared optional? I.e., can I enforce the signature - # but not the existence of a method? - - # def update_addons(config, addons): - # """Enables and configures other add-ons""" - - # def update_settings(config, settings): - # """Modifies `settings` to enable and configure required components""" - - # def check_configuration(config, crawler): - # """Performs post-initialization checks on fully configured `crawler`""" diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 8d1544c68..3ade1d105 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -99,18 +99,12 @@ def init_env(project="default", set_syspath=True): sys.path.append(projdir) -def config_from_filepath(sources): - """Create a ConfigParser and read in the given `sources`, which can be - either a filename or a list of filenames.""" - cfg = ConfigParser() - cfg.read(sources) - return cfg - - def get_config(use_closest=True): """Get Scrapy config file as a ConfigParser""" sources = get_sources(use_closest) - return config_from_filepath(sources) + cfg = ConfigParser() + cfg.read(sources) + return cfg def get_sources(use_closest=True) -> List[str]: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 4e23b01c3..b3c28da92 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -67,7 +67,7 @@ def load_object(path: Union[str, Callable]) -> Any: if callable(path): return path raise TypeError( - "Unexpected argument type, expected string " f"or object, got: {type(path)}" + f"Unexpected argument type, expected string or object, got: {type(path)}" ) try: @@ -86,22 +86,6 @@ def load_object(path: Union[str, Callable]) -> Any: return obj -def load_module_or_object(path): - """Load python module or (non-module) object from given path. - - Path can be both a Python or a file path. - """ - try: - return import_module(path) - except ImportError: - pass - try: - return load_object(path) - except (ValueError, NameError, ImportError): - pass - raise NameError(f"Could not load '{path}'") - - def walk_modules(path: str) -> List[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index fe26e1708..97de8d25a 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -73,7 +73,6 @@ def get_crawler( spidercls: Optional[Type[Spider]] = None, settings_dict: Optional[Dict[str, Any]] = None, prevent_warnings: bool = True, - addons=None, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level @@ -87,7 +86,7 @@ def get_crawler( if prevent_warnings: settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7" settings.update(settings_dict or {}) - runner = CrawlerRunner(settings, addons) + runner = CrawlerRunner(settings) return runner.create_crawler(spidercls or Spider) diff --git a/tests/test_addons.py b/tests/test_addons.py new file mode 100644 index 000000000..ecdc0426c --- /dev/null +++ b/tests/test_addons.py @@ -0,0 +1,72 @@ +import unittest + +from scrapy.addons import Addon, AddonManager +from scrapy.settings import BaseSettings + + +class GoodAddon(object): + name = "GoodAddon" + + def update_settings(self, config, settings): + pass + + def check_configuration(self, config, crawler): + pass + + +class AddonTest(unittest.TestCase): + def setUp(self): + class AddonWithAttributes(Addon): + name = "Test" + + self.testaddon = AddonWithAttributes() + + def test_export_config(self): + settings = BaseSettings() + self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"} + self.testaddon.default_config = {"key": 55, "defaultkey": 100} + self.testaddon.export_config( + {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings + ) + self.assertEqual(settings["KEY"], 313) + self.assertEqual(settings["DEFAULTKEY"], 100) + self.assertEqual(settings["OTHERKEY"], True) + self.assertNotIn("MAPPED_key", settings) + self.assertNotIn("MAPPED_KEY", settings) + self.assertEqual(settings["MAPPING_WORKED"], 99) + self.assertEqual(settings.getpriority("KEY"), 15) + + def test_update_settings(self): + settings = BaseSettings() + settings.set("KEY1", "default", priority="default") + settings.set("KEY2", "project", priority="project") + addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"} + self.testaddon.update_settings(addon_config, settings) + self.assertEqual(settings["KEY1"], "addon") + self.assertEqual(settings["KEY2"], "project") + self.assertEqual(settings["KEY3"], "addon") + + +class AddonManagerTest(unittest.TestCase): + def setUp(self): + self.manager = AddonManager() + + def test_add(self): + manager = AddonManager() + manager.add("tests.test_addons.GoodAddon") + self.assertCountEqual(manager, ["GoodAddon"]) + self.assertIsInstance(manager["GoodAddon"], GoodAddon) + + def test_load_settings(self): + settings = BaseSettings() + settings.set( + "ADDONS", + {"tests.test_addons.GoodAddon": 0}, + ) + settings.set("GOODADDON", {"key": "val2"}) + manager = AddonManager() + manager.load_settings(settings) + self.assertCountEqual(manager, ["GoodAddon"]) + self.assertIsInstance(manager["GoodAddon"], GoodAddon) + self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) + self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") diff --git a/tests/test_addons/__init__.py b/tests/test_addons/__init__.py deleted file mode 100644 index fa3f706e5..000000000 --- a/tests/test_addons/__init__.py +++ /dev/null @@ -1,342 +0,0 @@ -import itertools -import unittest -import warnings -from unittest import mock - -from zope.interface import directlyProvides -from zope.interface.exceptions import BrokenImplementation, MultipleInvalid -from zope.interface.verify import verifyObject - -from scrapy.addons import Addon, AddonManager -from scrapy.crawler import Crawler -from scrapy.interfaces import IAddon -from scrapy.settings import BaseSettings - -from . import addonmod, addons - - -class AddonTest(unittest.TestCase): - def setUp(self): - self.rawaddon = Addon() - - class AddonWithAttributes(Addon): - name = "Test" - version = "1.0" - - self.testaddon = AddonWithAttributes() - - def test_interface(self): - # Raw Addon should fail exactly b/c name and version are not given - self.assertFalse(hasattr(self.rawaddon, "name")) - self.assertFalse(hasattr(self.rawaddon, "version")) - self.assertRaises(MultipleInvalid, verifyObject, IAddon, self.rawaddon) - verifyObject(IAddon, self.testaddon) - - def test_export_component(self): - settings = BaseSettings( - {"ITEM_PIPELINES": BaseSettings(), "DOWNLOAD_HANDLERS": BaseSettings()}, - "default", - ) - self.testaddon.component_type = None - self.testaddon.export_component({}, settings) - self.assertEqual(len(settings["ITEM_PIPELINES"]), 0) - self.testaddon.component_type = "ITEM_PIPELINES" - self.testaddon.component = "test.component" - self.testaddon.export_component({}, settings) - self.assertCountEqual(settings["ITEM_PIPELINES"], ["test.component"]) - self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 0) - self.testaddon.component_order = 313 - self.testaddon.export_component({}, settings) - self.assertEqual(settings["ITEM_PIPELINES"]["test.component"], 313) - self.testaddon.component_type = "DOWNLOAD_HANDLERS" - self.testaddon.component_key = "http" - self.testaddon.export_component({}, settings) - self.assertEqual(settings["DOWNLOAD_HANDLERS"]["http"], "test.component") - - def test_export_basics(self): - settings = BaseSettings() - self.testaddon.basic_settings = {"TESTKEY": 313, "OTHERKEY": True} - self.testaddon.export_basics(settings) - self.assertEqual(settings["TESTKEY"], 313) - self.assertEqual(settings["OTHERKEY"], True) - self.assertEqual(settings.getpriority("TESTKEY"), 15) - - def test_export_config(self): - settings = BaseSettings() - self.testaddon.settings_prefix = None - self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"} - self.testaddon.default_config = {"key": 55, "defaultkey": 100} - self.testaddon.export_config( - {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings - ) - self.assertEqual(settings["TEST_KEY"], 313) - self.assertEqual(settings["TEST_DEFAULTKEY"], 100) - self.assertEqual(settings["TEST_OTHERKEY"], True) - self.assertNotIn("MAPPED_key", settings) - self.assertNotIn("MAPPED_KEY", settings) - self.assertEqual(settings["MAPPING_WORKED"], 99) - self.assertEqual(settings.getpriority("TEST_KEY"), 15) - - self.testaddon.settings_prefix = "PREF" - self.testaddon.export_config({"newkey": 99}, settings) - self.assertEqual(settings["PREF_NEWKEY"], 99) - - with mock.patch.object(settings, "set") as mock_set: - self.testaddon.settings_prefix = False - self.testaddon.export_config({"thirdnewkey": 99}, settings) - self.assertEqual(mock_set.call_count, 0) - - def test_update_settings(self): - settings = BaseSettings() - settings.set("TEST_KEY1", "default", priority="default") - settings.set("TEST_KEY2", "project", priority="project") - self.testaddon.settings_prefix = None - self.testaddon.basic_settings = {"OTHERTEST_KEY": "addon"} - addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"} - self.testaddon.update_settings(addon_config, settings) - self.assertEqual(settings["OTHERTEST_KEY"], "addon") - self.assertEqual(settings["TEST_KEY1"], "addon") - self.assertEqual(settings["TEST_KEY2"], "project") - self.assertEqual(settings["TEST_KEY3"], "addon") - - -class AddonManagerTest(unittest.TestCase): - def setUp(self): - self.manager = AddonManager() - - def test_add(self): - manager = AddonManager() - manager.add(addonmod, {"key": "val1"}) - manager.add("tests.test_addons.addons.GoodAddon") - self.assertCountEqual(manager, ["AddonModule", "GoodAddon"]) - self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) - self.assertCountEqual(manager.configs["AddonModule"], ["key"]) - self.assertEqual(manager.configs["AddonModule"]["key"], "val1") - self.assertRaises(ValueError, manager.add, addonmod) - - def test_add_dont_instantiate_providing_classes(self): - class ProviderGoodAddon(addons.GoodAddon): - pass - - directlyProvides(ProviderGoodAddon, IAddon) - manager = AddonManager() - manager.add(ProviderGoodAddon) - self.assertIs(manager["GoodAddon"], ProviderGoodAddon) - - def test_add_verifies(self): - brokenaddon = self.manager.get_addon("tests.test_addons.addons.BrokenAddon") - self.assertRaises( - BrokenImplementation, - self.manager.add, - brokenaddon, - ) - - def test_add_adds_missing_interface_declaration(self): - class GoodAddonWithoutDeclaration(object): - name = "GoodAddonWithoutDeclaration" - version = "1.0" - - self.manager.add(GoodAddonWithoutDeclaration) - - def test_remove(self): - manager = AddonManager() - - def test_gets_removed(removearg): - manager.add(addonmod) - self.assertIn("AddonModule", manager) - manager.remove(removearg) - self.assertNotIn("AddonModule", manager) - - test_gets_removed("AddonModule") - test_gets_removed(addonmod) - test_gets_removed("tests.test_addons.addonmod") - self.assertRaises(KeyError, manager.remove, "nonexistent") - self.assertRaises(KeyError, manager.remove, addons.GoodAddon()) - - def test_get_addon(self): - goodaddon = self.manager.get_addon("tests.test_addons.addons.GoodAddon") - self.assertIs(goodaddon, addons.GoodAddon) - - loaded_addonmod = self.manager.get_addon("tests.test_addons.addonmod") - self.assertIs(loaded_addonmod, addonmod) - - goodaddon = self.manager.get_addon("tests.test_addons.addons") - self.assertIsInstance(goodaddon, addons.GoodAddon) - - self.assertRaises(NameError, self.manager.get_addon, "xy.n_onexistent") - - def test_get_addon_forward(self): - class SomeCls(object): - _addon = "tests.test_addons.addons.GoodAddon" - - self.assertIs(self.manager.get_addon(SomeCls()), addons.GoodAddon) - - def test_get_addon_nested(self): - x = addons.GoodAddon("outer") - x._addon = addons.GoodAddon("middle") - x._addon._addon = addons.GoodAddon("inner") - self.assertIs(self.manager.get_addon(x), x._addon._addon) - - def test_load_settings(self): - settings = BaseSettings() - settings.set( - "ADDONS", - {"tests.test_addons.addonmod": 0, "tests.test_addons.addons.GoodAddon": 0}, - ) - settings.set("ADDONMODULE", {"key": "val1"}) - settings.set("GOODADDON", {"key": "val2"}) - manager = AddonManager() - manager.load_settings(settings) - self.assertCountEqual(manager, ["GoodAddon", "AddonModule"]) - self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon) - self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) - self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") - self.assertEqual(manager["AddonModule"], addonmod) - self.assertIn("key", manager.configs["AddonModule"]) - self.assertEqual(manager.configs["AddonModule"]["key"], "val1") - - def test_load_settings_order(self): - # Get three addons named 0, 1, 2 - addonlist = [addons.GoodAddon(str(x)) for x in range(3)] - # Test for every possible ordering - for ordered_addons in itertools.permutations(addonlist): - expected_order = [a.name for a in ordered_addons] - settings = BaseSettings( - {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} - ) - manager = AddonManager() - manager.load_settings(settings) - self.assertEqual(list(manager.keys()), expected_order) - - def test_enabled_disabled(self): - manager = AddonManager() - manager.add(addons.GoodAddon("FirstAddon")) - manager.add(addons.GoodAddon("SecondAddon")) - self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon"))) - self.assertEqual(manager.disabled, []) - manager.disable("FirstAddon") - self.assertEqual(manager.enabled, ["SecondAddon"]) - self.assertEqual(manager.disabled, ["FirstAddon"]) - manager.enable("FirstAddon") - self.assertEqual(set(manager.enabled), set(("FirstAddon", "SecondAddon"))) - self.assertEqual(manager.disabled, []) - - def test_enable_before_add(self): - manager = AddonManager() - self.assertRaises(ValueError, manager.enable, "FirstAddon") - manager.disable("FirstAddon") - manager.enable("FirstAddon") - manager.add(addons.GoodAddon("FirstAddon")) - self.assertIn("FirstAddon", manager.enabled) - - def test_disable_before_add(self): - manager = AddonManager() - manager.disable("FirstAddon") - manager.add(addons.GoodAddon("FirstAddon")) - self.assertEqual(manager.disabled, ["FirstAddon"]) - - def test_callbacks(self): - first_addon = addons.GoodAddon("FirstAddon") - second_addon = addons.GoodAddon("SecondAddon") - - manager = AddonManager() - manager.add(first_addon, {"test": "first"}) - manager.add(second_addon, {"test": "second"}) - crawler = mock.create_autospec(Crawler) - settings = BaseSettings() - - with mock.patch.object( - first_addon, "update_addons" - ) as ua_first, mock.patch.object( - second_addon, "update_addons" - ) as ua_second, mock.patch.object( - first_addon, "update_settings" - ) as us_first, mock.patch.object( - second_addon, "update_settings" - ) as us_second, mock.patch.object( - first_addon, "check_configuration" - ) as cc_first, mock.patch.object( - second_addon, "check_configuration" - ) as cc_second: - manager.update_addons() - ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager) - ua_second.assert_called_once_with(manager.configs["SecondAddon"], manager) - manager.update_settings(settings) - us_first.assert_called_once_with(manager.configs["FirstAddon"], settings) - us_second.assert_called_once_with(manager.configs["SecondAddon"], settings) - manager.check_configuration(crawler) - cc_first.assert_called_once_with(manager.configs["FirstAddon"], crawler) - cc_second.assert_called_once_with(manager.configs["SecondAddon"], crawler) - self.assertEqual(ua_first.call_count, 1) - self.assertEqual(ua_second.call_count, 1) - self.assertEqual(us_first.call_count, 1) - self.assertEqual(us_second.call_count, 1) - - us_first.reset_mock() - us_second.reset_mock() - manager.disable("FirstAddon") - manager.update_settings(settings) - self.assertEqual(us_first.call_count, 0) - manager.enable("FirstAddon") - manager.update_settings(settings) - self.assertEqual(us_first.call_count, 1) - self.assertEqual(us_second.call_count, 2) - - # This will become relevant when we let spiders implement the add-on - # interface and should be replaced with a test where - # AddonManager.spidercls = None then. - manager._call_if_exists(None, "irrelevant") - - def test_update_addons_last_minute_add(self): - class AddedAddon(addons.GoodAddon): - name = "AddedAddon" - - class FirstAddon(addons.GoodAddon): - name = "FirstAddon" - - def update_addons(self, config, addons): - addons.add(AddedAddon()) - - manager = AddonManager() - first_addon = FirstAddon() - with mock.patch.object( - first_addon, "update_addons", wraps=first_addon.update_addons - ) as ua_first, mock.patch.object(AddedAddon, "update_addons") as ua_added: - manager.add(first_addon, {"non-empty": "dict"}) - manager.update_addons() - self.assertCountEqual(manager, ["FirstAddon", "AddedAddon"]) - ua_first.assert_called_once_with(manager.configs["FirstAddon"], manager) - ua_added.assert_called_once_with(manager.configs["AddedAddon"], manager) - - def test_check_dependency_clashes_attributes(self): - provides = addons.GoodAddon("ProvidesAddon") - provides.provides = ("test",) - provides2 = addons.GoodAddon("ProvidesAddon2") - provides2.provides = ("test",) - requires = addons.GoodAddon("RequiresAddon") - requires.requires = ("test",) - requires_name = addons.GoodAddon("RequiresNameAddon") - requires_name.requires = ("ProvidesAddon",) - requires_newer = addons.GoodAddon("RequiresNewerAddon") - requires_newer.requires = ("test>=2.0",) - modifies = addons.GoodAddon("ModifiesAddon") - modifies.modifies = ("test",) - - def check_with(*addons): - manager = AddonManager() - for a in addons: - manager.add(a) - return manager.check_dependency_clashes() - - self.assertRaises(ImportError, check_with, requires) - self.assertRaises(ImportError, check_with, modifies) - self.assertRaises(ImportError, check_with, provides, provides2) - self.assertRaises(ImportError, check_with, provides, requires_newer) - with warnings.catch_warnings(record=True) as w: - check_with(provides, modifies) - check_with(provides) - check_with(provides, requires) - check_with(provides, requires_name) - self.assertEqual(len(w), 0) - check_with(requires, provides, modifies) - self.assertEqual(len(w), 1) diff --git a/tests/test_addons/addonmod.py b/tests/test_addons/addonmod.py deleted file mode 100644 index 092c3c0eb..000000000 --- a/tests/test_addons/addonmod.py +++ /dev/null @@ -1,16 +0,0 @@ -from zope.interface import moduleProvides - -from scrapy.interfaces import IAddon - -moduleProvides(IAddon) - -name = "AddonModule" -version = "1.0" - - -def update_settings(config, settings): - pass - - -def check_configuration(config, crawler): - pass diff --git a/tests/test_addons/addons.py b/tests/test_addons/addons.py deleted file mode 100644 index d878f37ea..000000000 --- a/tests/test_addons/addons.py +++ /dev/null @@ -1,33 +0,0 @@ -from zope.interface import implementer - -from scrapy.interfaces import IAddon - - -@implementer(IAddon) -class GoodAddon(object): - name = "GoodAddon" - version = "1.0" - - def __init__(self, name=None, version=None): - if name is not None: - self.name = name - if version is not None: - self.version = version - - def update_addons(self, config, addons): - pass - - def update_settings(self, config, settings): - pass - - def check_configuration(self, config, crawler): - pass - - -@implementer(IAddon) -class BrokenAddon(object): - name = "BrokenAddon" - # No version - - -_addon = GoodAddon() diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 920e5f4ae..d844a645f 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -412,7 +412,6 @@ with multiples lines def test_abort_on_addon_failed_check(self): class FailedCheckAddon(Addon): name = "FailedCheckAddon" - version = "1.0" def check_configuration(self, config, crawler): raise ValueError diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ed34d9e58..d54a2cb7e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -15,7 +15,6 @@ from twisted.trial import unittest from w3lib import __version__ as w3lib_version import scrapy -from scrapy.addons import Addon, AddonManager from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions import telnet @@ -56,32 +55,6 @@ class CrawlerTestCase(BaseCrawlerTest): self.assertFalse(settings.frozen) self.assertTrue(crawler.settings.frozen) - def test_populate_addons_settings(self): - class TestAddon(Addon): - name = "TestAddon" - version = "1.0" - - addonconfig = {"TEST1": "addon", "TEST2": "addon", "TEST3": "addon"} - - class TestAddon2(Addon): - name = "testAddon2" - version = "1.0" - - addonconfig2 = {"TEST": "addon2"} - - settings = Settings() - settings.set("TESTADDON_TEST1", "project", priority="project") - settings.set("TESTADDON_TEST2", "default", priority="default") - addonmgr = AddonManager() - addonmgr.add(TestAddon(), addonconfig) - addonmgr.add(TestAddon2(), addonconfig2) - crawler = Crawler(DefaultSpider, settings, addons=addonmgr) - - self.assertEqual(crawler.settings["TESTADDON_TEST1"], "project") - self.assertEqual(crawler.settings["TESTADDON_TEST2"], "addon") - self.assertEqual(crawler.settings["TESTADDON_TEST3"], "addon") - self.assertEqual(crawler.settings["TESTADDON2_TEST"], "addon2") - def test_crawler_accepts_dict(self): crawler = get_crawler(DefaultSpider, {"foo": "bar"}) self.assertEqual(crawler.settings["foo"], "bar") diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 7932ca04c..69793ee75 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -8,7 +8,6 @@ from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, create_instance, - load_module_or_object, load_object, rel_has_nofollow, set_environ, @@ -36,12 +35,6 @@ class UtilsMiscTestCase(unittest.TestCase): self.assertRaises(NameError, load_object, "scrapy.utils.misc.load_object999") self.assertRaises(TypeError, load_object, {}) - def test_load_module_or_object(self): - testmod = load_module_or_object(__name__ + ".testmod") - self.assertTrue(hasattr(testmod, "TESTVAR")) - obj = load_object("scrapy.utils.misc.load_object") - self.assertIs(obj, load_object) - def test_walk_modules(self): mods = walk_modules("tests.test_utils_misc.test_walk_modules") expected = [ diff --git a/tests/test_utils_misc/testmod.py b/tests/test_utils_misc/testmod.py deleted file mode 100644 index eb540335f..000000000 --- a/tests/test_utils_misc/testmod.py +++ /dev/null @@ -1 +0,0 @@ -TESTVAR = True From 760c0db094b3147236294f656f4168c16502de97 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 26 Jun 2023 17:15:13 +0400 Subject: [PATCH 37/55] Fix typing on 3.8. --- scrapy/addons.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index a54086fda..523c36e9c 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,5 +1,4 @@ -from collections.abc import Mapping -from typing import Any, Dict, Iterator, Optional, OrderedDict +from typing import Any, Dict, Iterator, Mapping, Optional, OrderedDict from scrapy.utils.conf import build_component_list from scrapy.utils.misc import load_object From 9612ae3e93239b86cedcd124073de6fff2736e99 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 28 Jun 2023 12:41:33 +0400 Subject: [PATCH 38/55] Remove more code. --- docs/topics/addons.rst | 100 ++++----------------------------- scrapy/addons.py | 124 ++++------------------------------------- tests/test_addons.py | 48 +++++----------- tests/test_crawl.py | 16 ------ 4 files changed, 37 insertions(+), 251 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 6a9590f33..3421864fa 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -17,40 +17,26 @@ Activating and configuring add-ons Add-ons and their configuration live in Scrapy's :class:`~scrapy.addons.AddonManager`. During a :class:`~scrapy.crawler.Crawler` initialization the add-on manager will read a list of enabled add-ons from your -``ADDONS`` setting and their optional configuration from the respective -settings. +``ADDONS`` setting. The ``ADDONS`` setting is a dict in which every key is an addon class or its import path and the value is its priority. -The configuration of an add-on, if necessary at all, is stored as a dictionary -setting whose name is the uppercase add-on name. - -This is an example where two add-ons (in this case with one requiring no -configuration) are enabled/configured in a project's ``settings.py``:: +This is an example where two add-ons are enabled in a project's +``settings.py``:: ADDONS = { 'path.to.someaddon': 0, path.to.someaddon2: 1, } - SOMEADDON = { - 'some_config': True, - } - Writing your own add-ons ======================== -Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*: +Add-ons are (any) Python *objects* that include the following method: -.. attribute:: name - - string with add-on name - - :type: ``str`` - -.. method:: update_settings(config, settings) +.. method:: update_settings(settings) This method is called during the initialization of the :class:`~scrapy.crawler.Crawler`. Here, you should perform dependency checks @@ -58,88 +44,26 @@ Add-ons are (any) Python *objects* that provide Scrapy's *add-on interface*: :class:`~scrapy.settings.Settings` object as wished, e.g. enable components for this add-on or set required configuration of other extensions. - :param config: Configuration of this add-on - :type config: ``dict`` - :param settings: The settings object storing Scrapy/component configuration :type settings: :class:`~scrapy.settings.Settings` -.. method:: check_configuration(config, crawler) - - This method is called when the :class:`~scrapy.crawler.Crawler` has been - fully initialized, immediately before it starts crawling. You can perform - additional dependency and configuration checks here. - - :param config: Configuration of this add-on - :type config: ``dict`` - - :param crawler: Fully initialized Scrapy crawler - :type crawler: :class:`~scrapy.crawler.Crawler` - - -Add-on base class -================= - -Scrapy comes with a built-in base class for add-ons which provides some -convenience functionality: the add-on configuration can be exposed into -Scrapy's settings via :meth:`~scrapy.addons.Addon.export_config`, configurable -via :attr:`~scrapy.addons.Addon.default_config` and -:attr:`~scrapy.addons.Addon.config_mapping`. - -By default, the base add-on class will expose the add-on configuration into -Scrapy's settings namespace, in upper case. It is -easy to write your own functionality while still being able to use the -convenience functions by overwriting -:meth:`~scrapy.addons.Addon.update_settings`. - -.. module:: scrapy.addons - :noindex: - -.. autoclass:: Addon - :members: - Add-on examples =============== -Set some basic configuration using the :class:`Addon` base class:: +Set some basic configuration:: - from scrapy.addons import Addon - - class MyAddon(Addon): - name = 'myaddon' - - def update_settings(self, config, settings): - super().update_settings(settings) + class MyAddon: + def update_settings(self, settings): settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 settings["DNSCACHE_ENABLED"] = True Check dependencies:: - from scrapy.addons import Addon - - class MyAddon(Addon): - name = 'myaddon' - - def update_settings(self, config, settings): + class MyAddon: + def update_settings(self, settings): try: import boto except ImportError: - raise RuntimeError("myaddon requires the boto library") - super().update_settings(settings) - -Check configuration of fully initialized crawler (see -:ref:`topics-api-crawler`):: - - class MyAddon(object): - name = 'myaddon' - - def update_settings(self, config, settings): - super().update_settings(settings) - settings.set('DNSCACHE_ENABLED', False, priority='addon') - - def check_configuration(self, config, crawler): - if crawler.settings.getbool('DNSCACHE_ENABLED'): - # The spider, some other add-on, or the user messed with the - # DNS cache setting - raise ValueError("myaddon is incompatible with DNS cache") + raise RuntimeError("MyAddon requires the boto library") + ... diff --git a/scrapy/addons.py b/scrapy/addons.py index 523c36e9c..bb4664d8e 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,100 +1,16 @@ -from typing import Any, Dict, Iterator, Mapping, Optional, OrderedDict +from typing import Any, List from scrapy.utils.conf import build_component_list from scrapy.utils.misc import load_object -class Addon(object): - name: str - - default_config = None - """``dict`` with default configuration.""" - - config_mapping = None - """``dict`` with mappings from config names to setting names. The given - setting names will be taken as given, not uppercased. - """ - - def export_config(self, config, settings): - """Export the add-on configuration, all keys in caps, into the settings - object. - - For example, the add-on configuration ``{'key': 'value'}`` will export - the setting ``KEY`` with a value of ``value``. All settings - will be exported with ``addon`` priority (see - :ref:`topics-api-settings`). - - :param config: Add-on configuration to be exposed - :type config: ``dict`` - - :param settings: Settings object into which to export the configuration - :type settings: :class:`~scrapy.settings.Settings` - """ - conf = self.default_config or {} - conf.update(config) - # Since default exported config is case-insensitive (everything will be - # uppercased), make mapped config case-insensitive as well - conf_mapping = {k.lower(): v for k, v in (self.config_mapping or {}).items()} - for key, val in conf.items(): - if key.lower() in conf_mapping: - key = conf_mapping[key.lower()] - else: - key = key.upper() - settings.set(key, val, "addon") - - def update_settings(self, config, settings): - """Modifiy `settings` to enable and configure required components. - - :param config: Add-on configuration - :type config: ``dict`` - - :param settings: Crawler settings object - :type settings: :class:`~scrapy.settings.Settings` - """ - self.export_config(config, settings) - - def check_configuration(self, config, crawler): - """Perform post-initialization checks on fully configured `crawler`. - - :param config: Add-on configuration - :type config: ``dict`` - - :param crawler: the fully-initialized crawler - :type crawler: :class:`~scrapy.crawler.Crawler` - """ - pass - - -class AddonManager(Mapping[str, Addon]): - """This class facilitates loading and storing :ref:`topics-addons`. - - You can treat it like a read-only dictionary in which keys correspond to - add-on names and values correspond to the add-on objects. Add-on - configurations are saved in the :attr:`config` dictionary attribute:: - - addons = AddonManager() - # ... load some add-ons here - print(addons.enabled) # prints names of all enabled add-ons - print(addons['TestAddon'].version) # prints version of add-on with name - # 'TestAddon' - print(addons.configs['TestAddon']) # prints configuration of 'TestAddon' - - """ +class AddonManager: + """This class facilitates loading and storing :ref:`topics-addons`.""" def __init__(self) -> None: - self._addons: OrderedDict[str, Addon] = OrderedDict[str, Addon]() - self.configs: Dict[str, Dict[str, Any]] = {} + self.addons: List[Any] = [] - def __getitem__(self, name: str) -> Addon: - return self._addons[name] - - def __iter__(self) -> Iterator[str]: - return iter(self._addons) - - def __len__(self) -> int: - return len(self._addons) - - def add(self, addon: Any, config: Optional[Dict[str, Any]] = None): + def add(self, addon: Any) -> None: """Store an add-on. :param addon: The add-on object (or path) to be stored @@ -107,19 +23,13 @@ class AddonManager(Mapping[str, Addon]): addon = load_object(addon) if isinstance(addon, type): addon = addon() - name = addon.name - if name in self: - raise ValueError(f"Addon '{name}' already loaded") - self._addons[name] = addon - self.configs[name] = config or {} + self.addons.append(addon) - def load_settings(self, settings): + def load_settings(self, settings) -> None: """Load add-ons and configurations from settings object. This will load the addon for every add-on path in the - ``ADDONS`` setting. For each of these add-ons, the configuration will be - read from the dictionary setting whose name matches the uppercase add-on - name. + ``ADDONS`` setting. :param settings: The :class:`~scrapy.settings.Settings` object from \ which to read the add-on configuration @@ -127,9 +37,8 @@ class AddonManager(Mapping[str, Addon]): """ paths = build_component_list(settings["ADDONS"]) addons = [load_object(path) for path in paths] - configs = [settings.getdict(addon.name.upper()) for addon in addons] - for a, c in zip(addons, configs): - self.add(a, c) + for a in addons: + self.add(a) def update_settings(self, settings) -> None: """Call ``update_settings()`` of all held add-ons. @@ -138,14 +47,5 @@ class AddonManager(Mapping[str, Addon]): updated :type settings: :class:`~scrapy.settings.Settings` """ - for name in self: - self[name].update_settings(self.configs[name], settings) - - def check_configuration(self, crawler) -> None: - """Call ``check_configuration()`` of all held add-ons. - - :param crawler: the fully-initialized crawler - :type crawler: :class:`~scrapy.crawler.Crawler` - """ - for name in self: - self[name].check_configuration(self.configs[name], crawler) + for addon in self.addons: + addon.update_settings(settings) diff --git a/tests/test_addons.py b/tests/test_addons.py index ecdc0426c..8ba27236d 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,47 +1,29 @@ import unittest +from typing import Any, Dict, Optional -from scrapy.addons import Addon, AddonManager +from scrapy.addons import AddonManager from scrapy.settings import BaseSettings -class GoodAddon(object): +class GoodAddon: name = "GoodAddon" - def update_settings(self, config, settings): - pass + def __init__(self, config: Optional[Dict[str, Any]] = None) -> None: + super().__init__() + self.config = config or {} - def check_configuration(self, config, crawler): - pass + def update_settings(self, settings): + settings.update(self.config, "addon") class AddonTest(unittest.TestCase): - def setUp(self): - class AddonWithAttributes(Addon): - name = "Test" - - self.testaddon = AddonWithAttributes() - - def test_export_config(self): - settings = BaseSettings() - self.testaddon.config_mapping = {"MAPPED_key": "MAPPING_WORKED"} - self.testaddon.default_config = {"key": 55, "defaultkey": 100} - self.testaddon.export_config( - {"key": 313, "OTHERKEY": True, "mapped_KEY": 99}, settings - ) - self.assertEqual(settings["KEY"], 313) - self.assertEqual(settings["DEFAULTKEY"], 100) - self.assertEqual(settings["OTHERKEY"], True) - self.assertNotIn("MAPPED_key", settings) - self.assertNotIn("MAPPED_KEY", settings) - self.assertEqual(settings["MAPPING_WORKED"], 99) - self.assertEqual(settings.getpriority("KEY"), 15) - def test_update_settings(self): settings = BaseSettings() settings.set("KEY1", "default", priority="default") settings.set("KEY2", "project", priority="project") - addon_config = {"key1": "addon", "key2": "addon", "key3": "addon"} - self.testaddon.update_settings(addon_config, settings) + addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"} + testaddon = GoodAddon(addon_config) + testaddon.update_settings(settings) self.assertEqual(settings["KEY1"], "addon") self.assertEqual(settings["KEY2"], "project") self.assertEqual(settings["KEY3"], "addon") @@ -54,8 +36,7 @@ class AddonManagerTest(unittest.TestCase): def test_add(self): manager = AddonManager() manager.add("tests.test_addons.GoodAddon") - self.assertCountEqual(manager, ["GoodAddon"]) - self.assertIsInstance(manager["GoodAddon"], GoodAddon) + self.assertIsInstance(manager.addons[0], GoodAddon) def test_load_settings(self): settings = BaseSettings() @@ -66,7 +47,4 @@ class AddonManagerTest(unittest.TestCase): settings.set("GOODADDON", {"key": "val2"}) manager = AddonManager() manager.load_settings(settings) - self.assertCountEqual(manager, ["GoodAddon"]) - self.assertIsInstance(manager["GoodAddon"], GoodAddon) - self.assertCountEqual(manager.configs["GoodAddon"], ["key"]) - self.assertEqual(manager.configs["GoodAddon"]["key"], "val2") + self.assertIsInstance(manager.addons[0], GoodAddon) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d844a645f..ca9084294 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -13,7 +13,6 @@ from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from scrapy import signals -from scrapy.addons import Addon, AddonManager from scrapy.crawler import CrawlerRunner from scrapy.exceptions import StopDownload from scrapy.http import Request @@ -408,21 +407,6 @@ with multiples lines self._assert_retried(log) self.assertIn("Got response 200", str(log)) - @defer.inlineCallbacks - def test_abort_on_addon_failed_check(self): - class FailedCheckAddon(Addon): - name = "FailedCheckAddon" - - def check_configuration(self, config, crawler): - raise ValueError - - addonmgr = AddonManager() - addonmgr.add(FailedCheckAddon()) - crawler = get_crawler(SimpleSpider) - crawler.addons = addonmgr - with self.assertRaises(ValueError): - yield crawler.crawl() - class CrawlSpiderTestCase(TestCase): def setUp(self): From f1ed5598f4d312337a9dd76e0c1a5856cabea8bc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 28 Jun 2023 19:13:46 +0400 Subject: [PATCH 39/55] Remove the check_configuration call. --- scrapy/crawler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index a0cb368ed..7a26cd2e4 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -133,7 +133,6 @@ class Crawler: try: self.spider = self._create_spider(*args, **kwargs) self.engine = self._create_engine() - self.addons.check_configuration(self) start_requests = iter(self.spider.start_requests()) yield self.engine.open_spider(self.spider, start_requests) yield defer.maybeDeferred(self.engine.start) From c92c9af075217c2296af364a4d9ea55463dbe7f0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 Jun 2023 15:04:46 +0400 Subject: [PATCH 40/55] Add create_instance support to addons. --- docs/topics/addons.rst | 30 +++++++++++++++++++++++++- scrapy/addons.py | 12 +++++++---- scrapy/crawler.py | 2 +- tests/test_addons.py | 48 +++++++++++++++++++++++++++++------------- 4 files changed, 71 insertions(+), 21 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 3421864fa..f1cc070ad 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -34,7 +34,7 @@ This is an example where two add-ons are enabled in a project's Writing your own add-ons ======================== -Add-ons are (any) Python *objects* that include the following method: +Add-ons are (any) Python objects that include the following method: .. method:: update_settings(settings) @@ -47,6 +47,20 @@ Add-ons are (any) Python *objects* that include the following method: :param settings: The settings object storing Scrapy/component configuration :type settings: :class:`~scrapy.settings.Settings` +They can also have the following method: + +.. classmethod:: from_crawler(cls, crawler) + :noindex: + + If present, this class method is called to create an addon instance + from a :class:`~scrapy.crawler.Crawler`. It must return a new instance + of the addon. Crawler object provides access to all Scrapy core + components like settings and signals; it is a way for pipeline to + access them and hook its functionality into Scrapy. + + :param crawler: The crawler that uses this addon + :type crawler: :class:`~scrapy.crawler.Crawler` + Add-on examples =============== @@ -67,3 +81,17 @@ Check dependencies:: except ImportError: raise RuntimeError("MyAddon requires the boto library") ... + +Access the crawler instance:: + + class MyAddon: + def __init__(self, crawler) -> None: + super().__init__() + self.crawler = crawler + + @classmethod + def from_crawler(cls, crawler: Crawler): + return cls(crawler) + + def update_settings(self, settings): + ... diff --git a/scrapy/addons.py b/scrapy/addons.py index bb4664d8e..ba33f1865 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,13 +1,17 @@ -from typing import Any, List +from typing import TYPE_CHECKING, Any, List from scrapy.utils.conf import build_component_list -from scrapy.utils.misc import load_object +from scrapy.utils.misc import create_instance, load_object + +if TYPE_CHECKING: + from scrapy.crawler import Crawler class AddonManager: """This class facilitates loading and storing :ref:`topics-addons`.""" - def __init__(self) -> None: + def __init__(self, crawler: "Crawler") -> None: + self.crawler: "Crawler" = crawler self.addons: List[Any] = [] def add(self, addon: Any) -> None: @@ -22,7 +26,7 @@ class AddonManager: if isinstance(addon, (type, str)): addon = load_object(addon) if isinstance(addon, type): - addon = addon() + addon = create_instance(addon, settings=None, crawler=self.crawler) self.addons.append(addon) def load_settings(self, settings) -> None: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 7a26cd2e4..12256440b 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -69,7 +69,7 @@ class Crawler: self.settings: Settings = settings.copy() self.spidercls.update_settings(self.settings) - self.addons: AddonManager = AddonManager() + self.addons: AddonManager = AddonManager(self) self.addons.load_settings(self.settings) self.addons.update_settings(self.settings) diff --git a/tests/test_addons.py b/tests/test_addons.py index 8ba27236d..d52665869 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,13 +1,12 @@ import unittest from typing import Any, Dict, Optional -from scrapy.addons import AddonManager +from scrapy.crawler import Crawler from scrapy.settings import BaseSettings +from scrapy.utils.test import get_crawler class GoodAddon: - name = "GoodAddon" - def __init__(self, config: Optional[Dict[str, Any]] = None) -> None: super().__init__() self.config = config or {} @@ -16,6 +15,20 @@ class GoodAddon: settings.update(self.config, "addon") +class CreateInstanceAddon: + def __init__(self, crawler: Crawler) -> None: + super().__init__() + self.crawler = crawler + self.config = crawler.settings.getdict("MYADDON") + + @classmethod + def from_crawler(cls, crawler: Crawler): + return cls(crawler) + + def update_settings(self, settings): + settings.update(self.config, "addon") + + class AddonTest(unittest.TestCase): def test_update_settings(self): settings = BaseSettings() @@ -30,21 +43,26 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): - def setUp(self): - self.manager = AddonManager() - def test_add(self): - manager = AddonManager() + crawler = get_crawler() + manager = crawler.addons manager.add("tests.test_addons.GoodAddon") self.assertIsInstance(manager.addons[0], GoodAddon) def test_load_settings(self): - settings = BaseSettings() - settings.set( - "ADDONS", - {"tests.test_addons.GoodAddon": 0}, - ) - settings.set("GOODADDON", {"key": "val2"}) - manager = AddonManager() - manager.load_settings(settings) + settings_dict = { + "ADDONS": {"tests.test_addons.GoodAddon": 0}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons self.assertIsInstance(manager.addons[0], GoodAddon) + + def test_create_instance(self): + settings_dict = { + "ADDONS": {"tests.test_addons.CreateInstanceAddon": 0}, + "MYADDON": {"MYADDON_KEY": "val"}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + self.assertIsInstance(manager.addons[0], CreateInstanceAddon) + self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val") From d5f74c72247e8fd4775e380569e0b809ad1cc6b5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 Jun 2023 20:43:46 +0400 Subject: [PATCH 41/55] Log the enabled addons. --- scrapy/addons.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scrapy/addons.py b/scrapy/addons.py index ba33f1865..612c7effc 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,3 +1,4 @@ +import logging from typing import TYPE_CHECKING, Any, List from scrapy.utils.conf import build_component_list @@ -6,6 +7,8 @@ from scrapy.utils.misc import create_instance, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler +logger = logging.getLogger(__name__) + class AddonManager: """This class facilitates loading and storing :ref:`topics-addons`.""" @@ -43,6 +46,13 @@ class AddonManager: addons = [load_object(path) for path in paths] for a in addons: self.add(a) + logger.info( + "Enabled addons:\n%(addons)s", + { + "addons": addons, + }, + extra={"crawler": self.crawler}, + ) def update_settings(self, settings) -> None: """Call ``update_settings()`` of all held add-ons. From a2264d3b8b70455f0ed481a9e76abc96056bc546 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Jul 2023 18:57:27 +0400 Subject: [PATCH 42/55] Improve docs about setting settings in addons. --- docs/topics/addons.rst | 10 +++++++++- docs/topics/api.rst | 1 + docs/topics/settings.rst | 15 +++++++++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index f1cc070ad..901a8bf8f 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -61,6 +61,14 @@ They can also have the following method: :param crawler: The crawler that uses this addon :type crawler: :class:`~scrapy.crawler.Crawler` +The settings set by the addon should use the ``addon`` priority (see +:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`). This +allows users to override these settings in the project or spider configuration. +This is not possible with settings that are mutable objects, such as the dict +that is a value of :setting:`ITEM_PIPELINES`. In these cases you can provide an +addon-specific setting that governs whether the addon will modify +:setting:`ITEM_PIPELINES`. + Add-on examples =============== @@ -70,7 +78,7 @@ Set some basic configuration:: class MyAddon: def update_settings(self, settings): settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 - settings["DNSCACHE_ENABLED"] = True + settings.set("DNSCACHE_ENABLED", True, "addon") Check dependencies:: diff --git a/docs/topics/api.rst b/docs/topics/api.rst index bb46b2b7d..d1a5497fb 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -137,6 +137,7 @@ Settings API SETTINGS_PRIORITIES = { "default": 0, "command": 10, + "addon": 15, "project": 20, "spider": 30, "cmdline": 40, diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 143002360..139e0a35f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -40,8 +40,9 @@ precedence: 1. Command line options (most precedence) 2. Settings per-spider 3. Project settings module - 4. Default settings per-command - 5. Default global settings (less precedence) + 4. Settings set by addons + 5. Default settings per-command + 6. Default global settings (less precedence) The population of these settings sources is taken care of internally, but a manual handling is possible using API calls. See the @@ -89,7 +90,13 @@ project, it's where most of your custom settings will be populated. For a standard Scrapy project, this means you'll be adding or changing the settings in the ``settings.py`` file created for your project. -4. Default settings per-command +4. Settings set by addons +------------------------- + +:ref:`Addons ` can modify settings. They should do this with +this priority, though this is not enforced. + +5. Default settings per-command ------------------------------- Each :doc:`Scrapy tool ` command can have its own default @@ -97,7 +104,7 @@ settings, which override the global default settings. Those custom command settings are specified in the ``default_settings`` attribute of the command class. -5. Default global settings +6. Default global settings -------------------------- The global defaults are located in the ``scrapy.settings.default_settings`` From cdda8ad46dc064229b5d875fb7685fdb32ebfb3f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 17 Jul 2023 21:05:58 +0400 Subject: [PATCH 43/55] Add docs about fallbacks in addons. --- docs/topics/addons.rst | 89 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 6 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 901a8bf8f..cc96207bd 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -62,12 +62,53 @@ They can also have the following method: :type crawler: :class:`~scrapy.crawler.Crawler` The settings set by the addon should use the ``addon`` priority (see -:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`). This -allows users to override these settings in the project or spider configuration. -This is not possible with settings that are mutable objects, such as the dict -that is a value of :setting:`ITEM_PIPELINES`. In these cases you can provide an -addon-specific setting that governs whether the addon will modify -:setting:`ITEM_PIPELINES`. +:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: + + class MyAddon: + def update_settings(self, settings): + settings.set("DNSCACHE_ENABLED", True, "addon") + +This allows users to override these settings in the project or spider +configuration. This is not possible with settings that are mutable objects, +such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases +you can provide an addon-specific setting that governs whether the addon will +modify :setting:`ITEM_PIPELINES`:: + + class MyAddon: + def update_settings(self, settings): + if settings.getbool("MYADDON_ENABLE_PIPELINE"): + settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 + +Fallbacks +--------- + +Some components provided by addons need to fallback to "default" +implementations, e.g. a custom download handler needs to send the request that +it doesn't handle via the default download handler, or a stats collector that +includes some additional processing but otherwise uses the default stats +collector. And it's possible that a project needs to use several custom +components of the same type, e.g. two custom download handlers that support +different kinds of custom requests and still need to use the default download +handler for other requests. To make such use cases easier to configure, we +recommend that such custom components should be written in the following way: + +1. The custom component (e.g. ``MyDownloadHandler``) shouldn't inherit from the + default Scrapy one (e.g. + ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler``), but instead + be able to load the class of the fallback component from a special setting + (e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use + it. +2. The addons that include these components should read the current value of + the default setting (e.g. ``DOWNLOAD_HANDLERS``) in their + ``update_settings()`` methods, save that value into the fallback setting + (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default + setting to the component provided byt the addon (e.g. + ``MyDownloadHandler``). If the fallback setting is already set by the user, + they shouldn't change it. +3. This way, if there are several addons that want to modify the same setting, + all of them will fallback to the component from the previous one and then to + the Scrapy default. The order of that depends on the priority order in the + ``ADDONS`` setting. Add-on examples @@ -103,3 +144,39 @@ Access the crawler instance:: def update_settings(self, settings): ... + +Use a fallback component:: + + from scrapy.core.downloader.handlers.http import HTTPDownloadHandler + + + fallback_setting = "MY_FALLBACK_DOWNLOAD_HANDLER" + + + class MyHandler: + lazy = False + + def __init__(self, settings, crawler): + dhcls = load_object(settings.get(fallback_setting)) + self._fallback_handler = create_instance( + dhcls, + settings=None, + crawler=crawler, + ) + + def download_request(self, request, spider): + if request.meta.get("my_params"): + # handle the request + ... + else: + return self._fallback_handler.download_request(request, spider) + + + class MyAddon: + def update_settings(self, settings): + if not settings.get(fallback_setting): + settings.set( + fallback_setting, + settings.getwithbase("DOWNLOAD_HANDLERS")["https"], + "addon", + ) From 5c34f34ecbc18c0829cd04a621b1d8239c25642b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 18 Jul 2023 19:50:08 +0400 Subject: [PATCH 44/55] Make AddonManager.add() private. --- scrapy/addons.py | 20 ++++++-------------- tests/test_addons.py | 6 ------ 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 612c7effc..812892613 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -17,15 +17,8 @@ class AddonManager: self.crawler: "Crawler" = crawler self.addons: List[Any] = [] - def add(self, addon: Any) -> None: - """Store an add-on. - - :param addon: The add-on object (or path) to be stored - :type addon: Python object, class or ``str`` - - :param config: The add-on configuration dictionary - :type config: ``dict`` - """ + def _add(self, addon: Any) -> None: + """Store an add-on.""" if isinstance(addon, (type, str)): addon = load_object(addon) if isinstance(addon, type): @@ -33,7 +26,7 @@ class AddonManager: self.addons.append(addon) def load_settings(self, settings) -> None: - """Load add-ons and configurations from settings object. + """Load add-ons and configurations from a settings object. This will load the addon for every add-on path in the ``ADDONS`` setting. @@ -42,10 +35,9 @@ class AddonManager: which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - paths = build_component_list(settings["ADDONS"]) - addons = [load_object(path) for path in paths] - for a in addons: - self.add(a) + addons = build_component_list(settings["ADDONS"]) + for addon in build_component_list(settings["ADDONS"]): + self._add(addon) logger.info( "Enabled addons:\n%(addons)s", { diff --git a/tests/test_addons.py b/tests/test_addons.py index d52665869..95377d718 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -43,12 +43,6 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): - def test_add(self): - crawler = get_crawler() - manager = crawler.addons - manager.add("tests.test_addons.GoodAddon") - self.assertIsInstance(manager.addons[0], GoodAddon) - def test_load_settings(self): settings_dict = { "ADDONS": {"tests.test_addons.GoodAddon": 0}, From 90dae3ee60b1f5ab5ea83d82ac8c00f1be54723e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 18 Jul 2023 19:52:27 +0400 Subject: [PATCH 45/55] Doc fixes. --- docs/topics/addons.rst | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index cc96207bd..c432c64d2 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -34,7 +34,7 @@ This is an example where two add-ons are enabled in a project's Writing your own add-ons ======================== -Add-ons are (any) Python objects that include the following method: +Add-ons are Python classes that include the following method: .. method:: update_settings(settings) @@ -54,9 +54,9 @@ They can also have the following method: If present, this class method is called to create an addon instance from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the addon. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for pipeline to - access them and hook its functionality into Scrapy. + of the addon. The crawler object provides access to all Scrapy core + components like settings and signals; it is a way for the addon to access + them and hook its functionality into Scrapy. :param crawler: The crawler that uses this addon :type crawler: :class:`~scrapy.crawler.Crawler` @@ -82,7 +82,7 @@ modify :setting:`ITEM_PIPELINES`:: Fallbacks --------- -Some components provided by addons need to fallback to "default" +Some components provided by addons need to fall back to "default" implementations, e.g. a custom download handler needs to send the request that it doesn't handle via the default download handler, or a stats collector that includes some additional processing but otherwise uses the default stats @@ -102,7 +102,7 @@ recommend that such custom components should be written in the following way: the default setting (e.g. ``DOWNLOAD_HANDLERS``) in their ``update_settings()`` methods, save that value into the fallback setting (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default - setting to the component provided byt the addon (e.g. + setting to the component provided by the addon (e.g. ``MyDownloadHandler``). If the fallback setting is already set by the user, they shouldn't change it. 3. This way, if there are several addons that want to modify the same setting, @@ -114,14 +114,18 @@ recommend that such custom components should be written in the following way: Add-on examples =============== -Set some basic configuration:: +Set some basic configuration: + +.. code-block:: python class MyAddon: def update_settings(self, settings): settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 settings.set("DNSCACHE_ENABLED", True, "addon") -Check dependencies:: +Check dependencies: + +.. code-block:: python class MyAddon: def update_settings(self, settings): @@ -131,7 +135,9 @@ Check dependencies:: raise RuntimeError("MyAddon requires the boto library") ... -Access the crawler instance:: +Access the crawler instance: + +.. code-block:: python class MyAddon: def __init__(self, crawler) -> None: @@ -145,7 +151,9 @@ Access the crawler instance:: def update_settings(self, settings): ... -Use a fallback component:: +Use a fallback component: + +.. code-block:: python from scrapy.core.downloader.handlers.http import HTTPDownloadHandler From 0a25a300cfc8281c355a6cfcd8605418347ed324 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 18 Jul 2023 20:29:42 +0400 Subject: [PATCH 46/55] Fix docs. --- docs/topics/addons.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index c432c64d2..36c7e0fa0 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -145,7 +145,7 @@ Access the crawler instance: self.crawler = crawler @classmethod - def from_crawler(cls, crawler: Crawler): + def from_crawler(cls, crawler): return cls(crawler) def update_settings(self, settings): From 005c8cc5f00f41ad7d836eccf45dbaf39c03ebca Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 19 Jul 2023 13:15:35 +0400 Subject: [PATCH 47/55] Unify the "add-on" spelling. --- docs/topics/addons.rst | 22 +++++++++++----------- docs/topics/settings.rst | 8 ++++---- scrapy/addons.py | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 36c7e0fa0..8733f9bde 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -19,7 +19,7 @@ Add-ons and their configuration live in Scrapy's initialization the add-on manager will read a list of enabled add-ons from your ``ADDONS`` setting. -The ``ADDONS`` setting is a dict in which every key is an addon class or its +The ``ADDONS`` setting is a dict in which every key is an add-on class or its import path and the value is its priority. This is an example where two add-ons are enabled in a project's @@ -52,16 +52,16 @@ They can also have the following method: .. classmethod:: from_crawler(cls, crawler) :noindex: - If present, this class method is called to create an addon instance + If present, this class method is called to create an add-on instance from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the addon. The crawler object provides access to all Scrapy core - components like settings and signals; it is a way for the addon to access + of the add-on. The crawler object provides access to all Scrapy core + components like settings and signals; it is a way for the add-on to access them and hook its functionality into Scrapy. - :param crawler: The crawler that uses this addon + :param crawler: The crawler that uses this add-on :type crawler: :class:`~scrapy.crawler.Crawler` -The settings set by the addon should use the ``addon`` priority (see +The settings set by the add-on should use the ``addon`` priority (see :ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: class MyAddon: @@ -71,7 +71,7 @@ The settings set by the addon should use the ``addon`` priority (see This allows users to override these settings in the project or spider configuration. This is not possible with settings that are mutable objects, such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases -you can provide an addon-specific setting that governs whether the addon will +you can provide an add-on-specific setting that governs whether the add-on will modify :setting:`ITEM_PIPELINES`:: class MyAddon: @@ -82,7 +82,7 @@ modify :setting:`ITEM_PIPELINES`:: Fallbacks --------- -Some components provided by addons need to fall back to "default" +Some components provided by add-ons need to fall back to "default" implementations, e.g. a custom download handler needs to send the request that it doesn't handle via the default download handler, or a stats collector that includes some additional processing but otherwise uses the default stats @@ -98,14 +98,14 @@ recommend that such custom components should be written in the following way: be able to load the class of the fallback component from a special setting (e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use it. -2. The addons that include these components should read the current value of +2. The add-ons that include these components should read the current value of the default setting (e.g. ``DOWNLOAD_HANDLERS``) in their ``update_settings()`` methods, save that value into the fallback setting (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default - setting to the component provided by the addon (e.g. + setting to the component provided by the add-on (e.g. ``MyDownloadHandler``). If the fallback setting is already set by the user, they shouldn't change it. -3. This way, if there are several addons that want to modify the same setting, +3. This way, if there are several add-ons that want to modify the same setting, all of them will fallback to the component from the previous one and then to the Scrapy default. The order of that depends on the priority order in the ``ADDONS`` setting. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 139e0a35f..602ab587d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -40,7 +40,7 @@ precedence: 1. Command line options (most precedence) 2. Settings per-spider 3. Project settings module - 4. Settings set by addons + 4. Settings set by add-ons 5. Default settings per-command 6. Default global settings (less precedence) @@ -90,10 +90,10 @@ project, it's where most of your custom settings will be populated. For a standard Scrapy project, this means you'll be adding or changing the settings in the ``settings.py`` file created for your project. -4. Settings set by addons -------------------------- +4. Settings set by add-ons +-------------------------- -:ref:`Addons ` can modify settings. They should do this with +:ref:`Add-ons ` can modify settings. They should do this with this priority, though this is not enforced. 5. Default settings per-command diff --git a/scrapy/addons.py b/scrapy/addons.py index 812892613..e72c5da98 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -28,7 +28,7 @@ class AddonManager: def load_settings(self, settings) -> None: """Load add-ons and configurations from a settings object. - This will load the addon for every add-on path in the + This will load the add-on for every add-on path in the ``ADDONS`` setting. :param settings: The :class:`~scrapy.settings.Settings` object from \ From 3ba2dc4d682c459ca3fa4419a68ae18b52a47642 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 24 Jul 2023 17:49:04 +0400 Subject: [PATCH 48/55] Fixes for addon docs. --- docs/topics/addons.rst | 17 ++++++++--------- docs/topics/api.rst | 11 ----------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 8733f9bde..f02399671 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -14,10 +14,8 @@ developers. Activating and configuring add-ons ================================== -Add-ons and their configuration live in Scrapy's -:class:`~scrapy.addons.AddonManager`. During a :class:`~scrapy.crawler.Crawler` -initialization the add-on manager will read a list of enabled add-ons from your -``ADDONS`` setting. +During :class:`~scrapy.crawler.Crawler` initialization, the list of enabled +add-ons is read from your ``ADDONS`` setting. The ``ADDONS`` setting is a dict in which every key is an add-on class or its import path and the value is its priority. @@ -27,7 +25,7 @@ This is an example where two add-ons are enabled in a project's ADDONS = { 'path.to.someaddon': 0, - path.to.someaddon2: 1, + SomeAddonClass: 1, } @@ -158,14 +156,14 @@ Use a fallback component: from scrapy.core.downloader.handlers.http import HTTPDownloadHandler - fallback_setting = "MY_FALLBACK_DOWNLOAD_HANDLER" + FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" class MyHandler: lazy = False def __init__(self, settings, crawler): - dhcls = load_object(settings.get(fallback_setting)) + dhcls = load_object(settings.get(FALLBACK_SETTING)) self._fallback_handler = create_instance( dhcls, settings=None, @@ -182,9 +180,10 @@ Use a fallback component: class MyAddon: def update_settings(self, settings): - if not settings.get(fallback_setting): + if not settings.get(FALLBACK_SETTING): settings.set( - fallback_setting, + FALLBACK_SETTING, settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "addon", ) + settings["DOWNLOAD_HANDLERS"]["http"] = "path.to.MyHandler" diff --git a/docs/topics/api.rst b/docs/topics/api.rst index d1a5497fb..16c28405c 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -155,17 +155,6 @@ Settings API .. autoclass:: BaseSettings :members: -.. _topics-api-addonmanager: - -AddonManager API -================ - -.. module:: scrapy.addons - :synopsis: Add-on manager - -.. autoclass:: AddonManager - :members: - .. _topics-api-spiderloader: SpiderLoader API From d8c5b415597f9a58e5bf10ab1d8838bd7f730949 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 31 Jul 2023 19:09:21 +0400 Subject: [PATCH 49/55] Add more addon tests. --- docs/topics/addons.rst | 2 +- tests/test_addons.py | 107 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 96 insertions(+), 13 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index f02399671..edb1e7883 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -186,4 +186,4 @@ Use a fallback component: settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "addon", ) - settings["DOWNLOAD_HANDLERS"]["http"] = "path.to.MyHandler" + settings["DOWNLOAD_HANDLERS"]["https"] = "path.to.MyHandler" diff --git a/tests/test_addons.py b/tests/test_addons.py index 95377d718..4156762d3 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,18 +1,24 @@ +import itertools import unittest -from typing import Any, Dict, Optional +from typing import Any, Dict -from scrapy.crawler import Crawler -from scrapy.settings import BaseSettings +from scrapy import Spider +from scrapy.crawler import Crawler, CrawlerRunner +from scrapy.settings import BaseSettings, Settings from scrapy.utils.test import get_crawler -class GoodAddon: - def __init__(self, config: Optional[Dict[str, Any]] = None) -> None: - super().__init__() - self.config = config or {} - +class SimpleAddon: def update_settings(self, settings): - settings.update(self.config, "addon") + pass + + +def get_addon_cls(config: Dict[str, Any]) -> type: + class AddonWithConfig: + def update_settings(self, settings: BaseSettings): + settings.update(config, priority="addon") + + return AddonWithConfig class CreateInstanceAddon: @@ -35,7 +41,7 @@ class AddonTest(unittest.TestCase): settings.set("KEY1", "default", priority="default") settings.set("KEY2", "project", priority="project") addon_config = {"KEY1": "addon", "KEY2": "addon", "KEY3": "addon"} - testaddon = GoodAddon(addon_config) + testaddon = get_addon_cls(addon_config)() testaddon.update_settings(settings) self.assertEqual(settings["KEY1"], "addon") self.assertEqual(settings["KEY2"], "project") @@ -45,11 +51,27 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): def test_load_settings(self): settings_dict = { - "ADDONS": {"tests.test_addons.GoodAddon": 0}, + "ADDONS": {"tests.test_addons.SimpleAddon": 0}, } crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons - self.assertIsInstance(manager.addons[0], GoodAddon) + self.assertIsInstance(manager.addons[0], SimpleAddon) + + def test_load_settings_order(self): + # Get three addons with different settings + addonlist = [] + for i in range(3): + addon = get_addon_cls({"KEY1": i}) + addon.number = i + addonlist.append(addon) + # Test for every possible ordering + for ordered_addons in itertools.permutations(addonlist): + expected_order = [a.number for a in ordered_addons] + settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} + crawler = get_crawler(settings_dict=settings) + manager = crawler.addons + self.assertEqual([a.number for a in manager.addons], expected_order) + self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1]) def test_create_instance(self): settings_dict = { @@ -60,3 +82,64 @@ class AddonManagerTest(unittest.TestCase): manager = crawler.addons self.assertIsInstance(manager.addons[0], CreateInstanceAddon) self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val") + + def test_settings_priority(self): + config = { + "KEY": 15, # priority=addon + } + settings_dict = { + "ADDONS": {get_addon_cls(config): 1}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual(crawler.settings.getint("KEY"), 15) + + settings = Settings(settings_dict) + settings.set("KEY", 0, priority="default") + runner = CrawlerRunner(settings) + crawler = runner.create_crawler(Spider) + self.assertEqual(crawler.settings.getint("KEY"), 15) + + settings_dict = { + "KEY": 20, # priority=project + "ADDONS": {get_addon_cls(config): 1}, + } + settings = Settings(settings_dict) + settings.set("KEY", 0, priority="default") + runner = CrawlerRunner(settings) + crawler = runner.create_crawler(Spider) + self.assertEqual(crawler.settings.getint("KEY"), 20) + + def test_fallback_workflow(self): + FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" + + class AddonWithFallback: + def update_settings(self, settings): + if not settings.get(FALLBACK_SETTING): + settings.set( + FALLBACK_SETTING, + settings.getwithbase("DOWNLOAD_HANDLERS")["https"], + "addon", + ) + settings["DOWNLOAD_HANDLERS"]["https"] = "AddonHandler" + + settings_dict = { + "ADDONS": {AddonWithFallback: 1}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual( + crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" + ) + self.assertEqual( + crawler.settings.get(FALLBACK_SETTING), + "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + ) + + settings_dict = { + "ADDONS": {AddonWithFallback: 1}, + "DOWNLOAD_HANDLERS": {"https": "UserHandler"}, + } + crawler = get_crawler(settings_dict=settings_dict) + self.assertEqual( + crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" + ) + self.assertEqual(crawler.settings.get(FALLBACK_SETTING), "UserHandler") From b67a81b81dd7c2a1b2017482551b3d818026f04f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 31 Jul 2023 20:25:56 +0400 Subject: [PATCH 50/55] Use the MyHandler class directly. --- docs/topics/addons.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index edb1e7883..8dddb7c91 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -186,4 +186,4 @@ Use a fallback component: settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "addon", ) - settings["DOWNLOAD_HANDLERS"]["https"] = "path.to.MyHandler" + settings["DOWNLOAD_HANDLERS"]["https"] = MyHandler From 41a4a163e3716bf0c14160373844bcded0e38abd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 31 Jul 2023 21:09:18 +0400 Subject: [PATCH 51/55] Handle NotConfigured in add-ons. --- docs/topics/addons.rst | 6 +++++- scrapy/addons.py | 26 +++++++++++--------------- scrapy/crawler.py | 1 - tests/test_addons.py | 13 +++++++++++++ 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 8dddb7c91..32c085079 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -77,6 +77,10 @@ modify :setting:`ITEM_PIPELINES`:: if settings.getbool("MYADDON_ENABLE_PIPELINE"): settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 +If the ``update_settings`` method raises +:exc:`scrapy.exceptions.NotConfigured`, the add-on will not be skipped. This +makes it easy to enable an add-on only when some conditions are met. + Fallbacks --------- @@ -130,7 +134,7 @@ Check dependencies: try: import boto except ImportError: - raise RuntimeError("MyAddon requires the boto library") + raise NotConfigured("MyAddon requires the boto library") ... Access the crawler instance: diff --git a/scrapy/addons.py b/scrapy/addons.py index e72c5da98..072143107 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -1,6 +1,8 @@ import logging from typing import TYPE_CHECKING, Any, List +from scrapy.exceptions import NotConfigured +from scrapy.settings import Settings from scrapy.utils.conf import build_component_list from scrapy.utils.misc import create_instance, load_object @@ -17,19 +19,23 @@ class AddonManager: self.crawler: "Crawler" = crawler self.addons: List[Any] = [] - def _add(self, addon: Any) -> None: + def _add(self, addon: Any, settings: Settings) -> None: """Store an add-on.""" if isinstance(addon, (type, str)): addon = load_object(addon) if isinstance(addon, type): addon = create_instance(addon, settings=None, crawler=self.crawler) - self.addons.append(addon) + try: + addon.update_settings(settings) + self.addons.append(addon) + except NotConfigured: + pass - def load_settings(self, settings) -> None: + def load_settings(self, settings: Settings) -> None: """Load add-ons and configurations from a settings object. This will load the add-on for every add-on path in the - ``ADDONS`` setting. + ``ADDONS`` setting and execute their ``update_settings`` methods. :param settings: The :class:`~scrapy.settings.Settings` object from \ which to read the add-on configuration @@ -37,7 +43,7 @@ class AddonManager: """ addons = build_component_list(settings["ADDONS"]) for addon in build_component_list(settings["ADDONS"]): - self._add(addon) + self._add(addon, settings) logger.info( "Enabled addons:\n%(addons)s", { @@ -45,13 +51,3 @@ class AddonManager: }, extra={"crawler": self.crawler}, ) - - def update_settings(self, settings) -> None: - """Call ``update_settings()`` of all held add-ons. - - :param settings: The :class:`~scrapy.settings.Settings` object to be \ - updated - :type settings: :class:`~scrapy.settings.Settings` - """ - for addon in self.addons: - addon.update_settings(settings) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 12256440b..bf69cee26 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -71,7 +71,6 @@ class Crawler: self.addons: AddonManager = AddonManager(self) self.addons.load_settings(self.settings) - self.addons.update_settings(self.settings) self.signals: SignalManager = SignalManager(self) diff --git a/tests/test_addons.py b/tests/test_addons.py index 4156762d3..5d053ed52 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -4,6 +4,7 @@ from typing import Any, Dict from scrapy import Spider from scrapy.crawler import Crawler, CrawlerRunner +from scrapy.exceptions import NotConfigured from scrapy.settings import BaseSettings, Settings from scrapy.utils.test import get_crawler @@ -57,6 +58,18 @@ class AddonManagerTest(unittest.TestCase): manager = crawler.addons self.assertIsInstance(manager.addons[0], SimpleAddon) + def test_notconfigured(self): + class NotConfiguredAddon: + def update_settings(self, settings): + raise NotConfigured() + + settings_dict = { + "ADDONS": {NotConfiguredAddon: 0}, + } + crawler = get_crawler(settings_dict=settings_dict) + manager = crawler.addons + self.assertFalse(manager.addons) + def test_load_settings_order(self): # Get three addons with different settings addonlist = [] From cf55eb05f59d7003bfb1491198e46853976dfb9e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 1 Aug 2023 13:30:56 +0400 Subject: [PATCH 52/55] Fix a typo. --- docs/topics/addons.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 32c085079..1bf2172bd 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -78,8 +78,8 @@ modify :setting:`ITEM_PIPELINES`:: settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 If the ``update_settings`` method raises -:exc:`scrapy.exceptions.NotConfigured`, the add-on will not be skipped. This -makes it easy to enable an add-on only when some conditions are met. +:exc:`scrapy.exceptions.NotConfigured`, the add-on will be skipped. This makes +it easy to enable an add-on only when some conditions are met. Fallbacks --------- From 7fdeb5c5c100fd94fd0fe2b8abc0c3fc0ba8c54d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 1 Aug 2023 16:58:18 +0400 Subject: [PATCH 53/55] Improve NotConfigured handling in add-ons. --- scrapy/addons.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 072143107..cd0cfb7de 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -19,18 +19,6 @@ class AddonManager: self.crawler: "Crawler" = crawler self.addons: List[Any] = [] - def _add(self, addon: Any, settings: Settings) -> None: - """Store an add-on.""" - if isinstance(addon, (type, str)): - addon = load_object(addon) - if isinstance(addon, type): - addon = create_instance(addon, settings=None, crawler=self.crawler) - try: - addon.update_settings(settings) - self.addons.append(addon) - except NotConfigured: - pass - def load_settings(self, settings: Settings) -> None: """Load add-ons and configurations from a settings object. @@ -41,13 +29,29 @@ class AddonManager: which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - addons = build_component_list(settings["ADDONS"]) - for addon in build_component_list(settings["ADDONS"]): - self._add(addon, settings) + enabled = [] + for clspath in build_component_list(settings["ADDONS"]): + try: + addoncls = load_object(clspath) + addon = create_instance( + addoncls, settings=settings, crawler=self.crawler + ) + addon.update_settings(settings) + self.addons.append(addon) + except NotConfigured as e: + if e.args: + clsname = ( + clspath.split(".")[-1] if isinstance(clspath, str) else clspath + ) + logger.warning( + "Disabled %(clsname)s: %(eargs)s", + {"clsname": clsname, "eargs": e.args[0]}, + extra={"crawler": self.crawler}, + ) logger.info( "Enabled addons:\n%(addons)s", { - "addons": addons, + "addons": enabled, }, extra={"crawler": self.crawler}, ) From f803ad63f32b488b3b89a81c6069d3575910e28e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 1 Aug 2023 17:23:09 +0400 Subject: [PATCH 54/55] Fix a typing issue. --- scrapy/addons.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index cd0cfb7de..f1ddfd211 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -29,7 +29,7 @@ class AddonManager: which to read the add-on configuration :type settings: :class:`~scrapy.settings.Settings` """ - enabled = [] + enabled: List[Any] = [] for clspath in build_component_list(settings["ADDONS"]): try: addoncls = load_object(clspath) From e58b8078f0d51148c1866d1da3f6e36609b5e2a5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 1 Aug 2023 20:20:15 +0400 Subject: [PATCH 55/55] Improve NotConfigured handling when clspath is a class. --- scrapy/addons.py | 7 ++----- scrapy/middleware.py | 5 ++--- tests/test_middleware.py | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index f1ddfd211..02dd4fde8 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -40,12 +40,9 @@ class AddonManager: self.addons.append(addon) except NotConfigured as e: if e.args: - clsname = ( - clspath.split(".")[-1] if isinstance(clspath, str) else clspath - ) logger.warning( - "Disabled %(clsname)s: %(eargs)s", - {"clsname": clsname, "eargs": e.args[0]}, + "Disabled %(clspath)s: %(eargs)s", + {"clspath": clspath, "eargs": e.args[0]}, extra={"crawler": self.crawler}, ) logger.info( diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 03e92b565..04b838d2d 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -46,10 +46,9 @@ class MiddlewareManager: enabled.append(clspath) except NotConfigured as e: if e.args: - clsname = clspath.split(".")[-1] logger.warning( - "Disabled %(clsname)s: %(eargs)s", - {"clsname": clsname, "eargs": e.args[0]}, + "Disabled %(clspath)s: %(eargs)s", + {"clspath": clspath, "eargs": e.args[0]}, extra={"crawler": crawler}, ) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 00ff746ee..a42c7b3d1 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -39,7 +39,7 @@ class MOff: pass def __init__(self): - raise NotConfigured + raise NotConfigured("foo") class TestMiddlewareManager(MiddlewareManager):