Introduce add-ons via AddonManager and Addon base class

docs/topics/api.rst

@@ -149,6 +149,17 @@ Settings API
 .. autoclass:: BaseSettings
    :members:
 
+.. _topics-api-addonmanager:
+
+AddonManager API
+================
+
+.. module:: scrapy.addons
+   :synopsis: Add-on manager
+
+.. autoclass:: AddonManager
+   :members:
+
 .. _topics-api-spiderloader:
 
 SpiderLoader API

docs/topics/settings.rst

@@ -585,6 +585,15 @@ some of them need to be enabled through a setting.
 For more information See the :ref:`extensions user guide  <topics-extensions>`
 and the :ref:`list of available extensions <topics-extensions-ref>`.
 
+.. setting:: INSTALLED_ADDONS
+
+INSTALLED_ADDONS
+----------------
+
+Default: ``()``
+
+A tuple containing paths to the add-ons enabled in your project. For more
+information, see :ref:`topics-addons`.
 
 .. setting:: ITEM_PIPELINES
 

scrapy/addons/__init__.py

@@ -0,0 +1,497 @@
+from collections import defaultdict, Mapping
+from importlib import import_module
+from inspect import isclass
+import os
+import six
+import warnings
+
+from pkg_resources import WorkingSet, Distribution, Requirement
+import zope.interface
+from zope.interface.verify import verifyObject
+
+from scrapy.exceptions import NotConfigured
+from scrapy.interfaces import IAddon
+from scrapy.settings import BaseSettings
+from scrapy.utils.conf import config_from_filepath, get_config
+from scrapy.utils.misc import load_module_or_object
+from scrapy.utils.project import get_project_path
+
+
+@zope.interface.implementer(IAddon)
+class Addon(object):
+
+    basic_settings = None
+    """``dict`` of settings that will be exported via :meth:`export_basics`."""
+
+    default_config = None
+    """``dict`` with default configuration."""
+
+    config_mapping = None
+    """``dict`` with mappings from config names to setting names. The given
+    setting names will be taken as given, i.e. they will be neither prefixed
+    nor uppercased.
+    """
+
+    component_type = None
+    """Component setting into which to export via :meth:`export_component`. Can
+    be any of the dictionary-like component setting names (e.g.
+    ``DOWNLOADER_MIDDLEWARES``) or any of their abbreviations in
+    :attr:`~scrapy.addons.COMPONENT_TYPE_ABBR`. If ``None``,
+    :meth:`export_component` will do nothing.
+    """
+
+    component_key = None
+    """Key to be used in the component dictionary setting when exporting via
+    :meth:`export_component`. This is only useful for the settings that have
+    no order, e.g. ``DOWNLOAD_HANDLERS`` or ``FEED_EXPORTERS``.
+    """
+
+    component_order = 0
+    """Component order to use when not given in the add-on configuration. Has
+    no effect for component types that use :attr:`component_key`.
+    """
+
+    component = None
+    """Component to be inserted via :meth:`export_component`. This can be
+    anything that can be used in the dictionary-like component settings, i.e.
+    a class path, a class, or an instance. If ``None``, it is assumed that the
+    add-on itself is also provides the component interface, and ``self`` will be
+    used.
+    """
+
+    settings_prefix = None
+    """Prefix with which the add-on configuration will be exported into the
+    global settings namespace via :meth:`export_config`. If ``None``,
+    :attr:`name` will be used. If ``False``, no configuration will be exported.
+    """
+
+    def export_component(self, config, settings):
+        """Export the component in :attr:`component` into the dictionary-like
+        component setting derived from :attr:`component_type`.
+
+        Where applicable, the order parameter of the component (i.e. the
+        dictionary value) will be retrieved from the ``order`` add-on
+        configuration value.
+
+        :param config: Add-on configuration from which to read component order
+        :type config: ``dict``
+
+        :param settings: Settings object into which to export component
+        :type settings: :class:`~scrapy.settings.Settings`
+        """
+        if self.component_type:
+            comp = self.component or self
+            if self.component_key:
+                # e.g. for DOWNLOAD_HANDLERS: {'http': 'myclass'}
+                k = self.component_key
+                v = comp
+            else:
+                # e.g. for DOWNLOADER_MIDDLEWARES: {'myclass': 100}
+                k = comp
+                v = config.get('order', self.component_order)
+            settings.set(self.component_type, {k: v}, 'addon')
+
+    def export_basics(self, settings):
+        """Export the :attr:`basic_settings` attribute into the settings object.
+
+        All settings will be exported with ``addon`` priority (see
+        :ref:`topics-api-settings`).
+
+        :param settings: Settings object into which to expose the basic settings
+        :type settings: :class:`~scrapy.settings.Settings`
+        """
+        for setting, value in six.iteritems(self.basic_settings or {}):
+            settings.set(setting, value, 'addon')
+
+    def export_config(self, config, settings):
+        """Export the add-on configuration, all keys in caps and with
+        :attr:`settings_prefix` or :attr:`name` prepended, into the settings
+        object.
+
+        For example, the add-on configuration ``{'key': 'value'}`` will export
+        the setting ``ADDONNAME_KEY`` with a value of ``value``. All settings
+        will be exported with ``addon`` priority (see
+        :ref:`topics-api-settings`).
+
+        :param config: Add-on configuration to be exposed
+        :type config: ``dict``
+
+        :param settings: Settings object into which to export the configuration
+        :type settings: :class:`~scrapy.settings.Settings`
+        """
+        if self.settings_prefix is False:
+            return
+        conf = self.default_config or {}
+        conf.update(config)
+        prefix = self.settings_prefix or self.name
+        # Since default exported config is case-insensitive (everything will be
+        # uppercased), make mapped config case-insensitive as well
+        conf_mapping = {k.lower(): v
+                        for k, v in six.iteritems(self.config_mapping or {})}
+        for key, val in six.iteritems(conf):
+            if key.lower() in conf_mapping:
+                key = conf_mapping[key.lower()]
+            else:
+                key = (prefix + '_' + key).upper()
+            settings.set(key, val, 'addon')
+
+    def update_settings(self, config, settings):
+        """Export both the basic settings and the add-on configuration. I.e.,
+        call :meth:`export_basics` and :meth:`export_config`.
+
+        For more advanced add-ons, you may want to override this callback.
+
+        :param config: Add-on configuration
+        :type config: ``dict``
+
+        :param settings: Crawler settings object
+        :type settings: :class:`~scrapy.settings.Settings`
+        """
+        self.export_component(config, settings)
+        self.export_basics(settings)
+        self.export_config(config, settings)
+
+
+class AddonManager(Mapping):
+    """This class facilitates loading and storing :ref:`topics-addons`.
+
+    You can treat it like a read-only dictionary in which keys correspond to
+    add-on names and values correspond to the add-on objects::
+
+        addons = AddonManager()
+        # ... load some add-ons here
+        print addons.enabled  # prints names of all enabled add-ons
+        print addons['TestAddon'].version  # prints version of add-on with name
+                                           # 'TestAddon'
+
+    """
+
+    def __init__(self):
+        self._addons = {}
+        self.configs = {}
+        self._disable_on_add = []
+
+    def __getitem__(self, name):
+        return self._addons[name]
+
+    def __delitem__(self, name):
+        del self._addons[name]
+        del self.configs[name]
+
+    def __iter__(self):
+        return iter(self._addons)
+
+    def __len__(self):
+        return len(self._addons)
+
+    def add(self, addon, config=None):
+        """Store an add-on.
+
+        If ``addon`` is a string, it will be treated as add-on path and passed
+        to :meth:`get_addon`. Otherwise, ``addon`` must be a Python object
+        implementing or providing Scrapy's add-on interface. The interface
+        will be enforced through ``zope.interface``'s ``verifyObject()``.
+
+        If ``addon`` is a class, it will be instantiated. You can avoid this
+        (for example if you have implemented the add-on callbacks as class
+        methods) by declaring --  via ``zope.interface`` -- that your class
+        directly *provides* ``scrapy.interfaces.IAddon``.
+
+        :param addon: The add-on object (or path) to be stored
+        :type addon: Any Python object providing the add-on interface or ``str``
+
+        :param config: The add-on configuration dictionary
+        :type config: ``dict``
+        """
+        addon = self.get_addon(addon)
+        if isclass(addon) and not IAddon.providedBy(addon):
+            addon = addon()
+        if not IAddon.providedBy(addon):
+            zope.interface.alsoProvides(addon, IAddon)
+        # zope.interface's exceptions are already quite helpful. Still, should
+        # we catch them and log an error message?
+        verifyObject(IAddon, addon)
+        name = addon.name
+        if name in self:
+            raise ValueError("Addon '{}' already loaded".format(name))
+        self._addons[name] = addon
+        self.configs[name] = config or {}
+        if name in self._disable_on_add:
+            self.configs[name]['_enabled'] = False
+            self._disable_on_add.remove(name)
+
+    def remove(self, addon):
+        """Remove an add-on.
+
+        If ``addon`` is the name of a stored add-on, that add-on will be
+        removed. Otherwise, you can use the argument in the same fashion as
+        in :meth:`add`.
+
+        :param addon: The add-on name, object, or path to be removed
+        :type addon: Any Python object providing the add-on interface or ``str``
+        """
+        if addon in self:
+            del self[addon]
+        elif hasattr(addon, 'name') and addon.name in self:
+            del self[addon.name]
+        else:
+            try:
+                del self[self.get_addon(addon).name]
+            except NameError:
+                raise KeyError
+
+    @staticmethod
+    def get_addon(path):
+        """Get an add-on object by its Python or file path.
+
+        ``path`` is assumed to be either a Python or a file path of a Scrapy
+        add-on. If no object is found at ``path``, it is tried again first with
+        ``projectname.addons`` prepended (pointing to the current project's
+        ``addons`` folder), then with ``scrapy.addons`` prepended (poiting to
+        Scrapy's built-in add-ons). These convenience shortcuts will only work
+        with Python paths, not file paths.
+
+        If the object or module pointed to by ``path`` has an attribute named
+        ``_addon`` that attribute will be assumed to be the add-on.
+        :meth:`get_addon` will keep following ``_addon`` attributes until it
+        finds an object that does not have an attribute named ``_addon``.
+
+        :param path: Python or file path to an add-on
+        :type path: ``str``
+        """
+        if isinstance(path, six.string_types):
+            prefixes = ['', 'scrapy.addons.']
+            try:
+                prefixes.insert(1, get_project_path() + '.addons.')
+            except NotConfigured:
+                warnings.warn("Unable to locate project Python path")
+            for prefix in prefixes:
+                fullpath = prefix + path
+                try:
+                    obj = load_module_or_object(fullpath)
+                except NameError:
+                    pass
+                else:
+                    break
+            else:
+                raise NameError("Could not find add-on '%s'" % path)
+        else:
+            obj = path
+        if hasattr(obj, '_addon'):
+            obj = AddonManager.get_addon(obj._addon)
+        return obj
+
+    def load_dict(self, addonsdict):
+        """Load add-ons and configurations from given dictionary.
+
+        Each add-on should be an entry in the dictionary, where the key
+        corresponds to the add-on path. The value should be a dictionary
+        representing the add-on configuration.
+
+        Example add-on dictionary::
+
+            addonsdict = {
+                'path.to.addon1': {
+                    'setting1': 'value',
+                    'setting2': 42,
+                    },
+                'path/to/addon2.py': {
+                    'addon2setting': True,
+                    },
+                }
+
+        :param addonsdict: dictionary where keys correspond to add-on paths \
+            and values correspond to their configuration
+        :type addonsdict: ``dict``
+        """
+        for addonpath, addoncfg in six.iteritems(addonsdict):
+            self.add(addonpath, addoncfg)
+
+    def load_settings(self, settings):
+        """Load add-ons and configurations from settings object.
+
+        This will invoke :meth:`get_addon` for every add-on path in the
+        ``INSTALLED_ADDONS`` setting. For each of these add-ons, the
+        configuration will be read from the dictionary setting whose name
+        matches the uppercase add-on name.
+
+        :param settings: The :class:`~scrapy.settings.Settings` object from \
+            which to read the add-on configuration
+        :type settings: :class:`~scrapy.settings.Settings`
+        """
+        paths = settings.getlist('INSTALLED_ADDONS')
+        addons = [self.get_addon(path) for path in paths]
+        configs = [settings.getdict(addon.name.upper()) for addon in addons]
+        for a, c in zip(addons, configs):
+            self.add(a, c)
+
+    def load_cfg(self, cfg=None):
+        """Load add-ons and configurations from given ``ConfigParser`` object or
+        config file path.
+
+        Each add-on should have its own section, where the section has a name in
+        the form ``addon:my_addon_path``. The add-on object is searched for via
+        the :meth:`get_addon` method, ``my_addon_path`` can be either a Python
+        or a file path.
+
+        If ``cfg`` is ``None``, ``scrapy.cfg`` will be used.
+
+        :param cfg: ``ConfigParser`` object or config file path from which to \
+            read add-on configuration
+        :type cfg: ``ConfigParser`` or ``str``
+        """
+        if cfg is None:
+            cfg = get_config()
+        elif isinstance(cfg, six.string_types):
+            cfg = config_from_filepath(cfg)
+        for secname in cfg.sections():
+            if secname.startswith("addon:"):
+                addonkey = secname.split("addon:", 1)[1]
+                addoncfg = dict(cfg.items(secname))
+                self.add(addonkey, addoncfg)
+
+    def check_dependency_clashes(self):
+        """Check for incompatibilities in add-on dependencies.
+
+        Add-ons can provide information about their dependencies in their
+        ``provides``, ``modifies`` and ``requires`` attributes. This method will
+        raise an ``ImportError`` if
+
+        * a component required by an add-on is not provided by any other add-on,
+          or
+        * a component modified by an add-on is not provided by any other add-on,
+          or
+        * the same component is provided by more than one add-on,
+
+        and warn when a component required by an add-on is modified by any other
+        add-on.
+        """
+        # Collect all active add-ons and the components they provide
+        ws = WorkingSet('')
+        def add_dist(project_name, version, **kwargs):
+            if project_name in ws.entry_keys.get('scrapy', []):
+                raise ImportError("Component {} provided by multiple add-ons"
+                                  "".format(project_name))
+            else:
+                dist = Distribution(project_name=project_name, version=version,
+                                    **kwargs)
+                ws.add(dist, entry='scrapy')
+        for name in self:
+            ver = self[name].version
+            add_dist(name, ver)
+            for provides_name in getattr(self[name], 'provides', []):
+                add_dist(provides_name, ver)
+
+        # Collect all required and modified components
+        def compile_attribute_dict(attribute_name):
+            attrs = defaultdict(list)
+            for name in self:
+                for entry in getattr(self[name], attribute_name, []):
+                    attrs[entry].append(name)
+            return attrs
+        modified = compile_attribute_dict('modifies')
+        required = compile_attribute_dict('requires')
+
+        req_or_mod = set(required.keys()).union(modified.keys())
+        for reqstr in req_or_mod:
+            req = Requirement.parse(reqstr)
+            # May raise VersionConflict. Do we want to catch it and raise
+            # our own exception or is it helpful enough?
+            if ws.find(req) is None:
+                raise ImportError(
+                          "Add-ons {} require or modify missing component {}"
+                          "".format(required[reqstr]+modified[reqstr], reqstr))
+
+        mod_and_req = set(required.keys()).intersection(modified.keys())
+        for conflict in mod_and_req:
+            warnings.warn("Component '{}', required by add-ons {}, is modified "
+                          "by add-ons {}".format(conflict, required[conflict],
+                                                 modified[conflict]))
+
+    def disable(self, addon):
+        """Disable an add-on, i.e. prevent its callbacks from being called.
+
+        If you disable an add-on before it is loaded, it will be disabled as
+        soon as it is added to the :class:`AddonManager`.
+
+        :param addon: Name of the add-on to be disabled
+        :type addon: ``str``
+        """
+        if addon in self:
+            self.configs[addon]['_enabled'] = False
+        else:
+            self._disable_on_add.append(addon)
+
+    def enable(self, addon):
+        """Re-enable a disabled add-on.
+
+        Will raise ``ValueError`` if the add-on is neither already loaded nor
+        marked for being disabled on adding.
+
+        :param addon: Name of the add-on to be enabled
+        :type addon: ``str``
+        """
+        if addon in self:
+            self.configs[addon]['_enabled'] = True
+        elif addon in self._disable_on_add:
+            self._disable_on_add.remove(addon)
+        else:
+            raise ValueError("Add-ons need to be added before they can be "
+                             "enabled")
+
+    @property
+    def disabled(self):
+        """Names of disabled add-ons"""
+        return ([a for a in self if not self.configs[a].get('_enabled', True)] +
+                self._disable_on_add)
+
+    @property
+    def enabled(self):
+        """Names of enabled add-ons"""
+        return [a for a in self if self.configs[a].get('_enabled', True)]
+
+    def _call_if_exists(self, obj, cbname, *args, **kwargs):
+        if obj is None:
+            return
+        try:
+            cb = getattr(obj, cbname)
+        except AttributeError:
+            return
+        else:
+            cb(*args, **kwargs)
+
+    def _call_addon(self, addonname, cbname, *args, **kwargs):
+        if self.configs[addonname].get('_enabled', True):
+            self._call_if_exists(self[addonname], cbname,
+                                 self.configs[addonname], *args, **kwargs)
+
+    def update_addons(self):
+        """Call ``update_addons()`` of all held add-ons.
+
+        This will also call ``update_addons()`` of all add-ons that are added
+        last minute during the ``update_addons()`` routine of other add-ons.
+        """
+        called_addons = set()
+        while called_addons != set(self):
+            for name in set(self).difference(called_addons):
+                called_addons.add(name)
+                self._call_addon(name, 'update_addons', self)
+
+    def update_settings(self, settings):
+        """Call ``update_settings()`` of all held add-ons.
+
+        :param settings: The :class:`~scrapy.settings.Settings` object to be \
+            updated
+        :type settings: :class:`~scrapy.settings.Settings`
+        """
+        for name in self:
+            self._call_addon(name, 'update_settings', settings)
+
+    def check_configuration(self, crawler):
+        """Call ``check_configuration()`` of all held add-ons.
+
+        :param crawler: the fully-initialized crawler
+        :type crawler: :class:`~scrapy.crawler.Crawler`
+        """
+        for name in self:
+            self._call_addon(name, 'check_configuration', crawler)

scrapy/interfaces.py

@@ -1,6 +1,6 @@
-from zope.interface import Interface
+import zope.interface
 
-class ISpiderLoader(Interface):
+class ISpiderLoader(zope.interface.Interface):
 
     def from_settings(settings):
         """Return an instance of the class for the given settings"""
@@ -20,3 +20,22 @@ class ISpiderLoader(Interface):
 # ISpiderManager is deprecated, don't use it!
 # An alias is kept for backwards compatibility.
 ISpiderManager = ISpiderLoader
+
+
+class IAddon(zope.interface.Interface):
+    """Scrapy add-on"""
+
+    name = zope.interface.Attribute("""Add-on name""")
+    version = zope.interface.Attribute("""Add-on version string (PEP440)""")
+
+    # XXX: Can methods be declared optional? I.e., can I enforce the signature
+    #      but not the existence of a method?
+
+    #def update_addons(config, addons):
+    #    """Enables and configures other add-ons"""
+
+    #def update_settings(config, settings):
+    #    """Modifies `settings` to enable and configure required components"""
+
+    #def check_configuration(config, crawler):
+    #    """Performs post-initialization checks on fully configured `crawler`"""

scrapy/settings/__init__.py

@@ -14,6 +14,7 @@ from . import default_settings
 SETTINGS_PRIORITIES = {
     'default': 0,
     'command': 10,
+    'addon': 15,
     'project': 20,
     'spider': 30,
     'cmdline': 40,

scrapy/settings/default_settings.py

@@ -167,6 +167,8 @@ HTTPCACHE_DBM_MODULE = 'anydbm'
 HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
 HTTPCACHE_GZIP = False
 
+INSTALLED_ADDONS = ()
+
 ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
 
 ITEM_PIPELINES = {}

scrapy/utils/conf.py

@@ -82,14 +82,20 @@ def init_env(project='default', set_syspath=True):
             sys.path.append(projdir)
 
 
-def get_config(use_closest=True):
-    """Get Scrapy config file as a SafeConfigParser"""
-    sources = get_sources(use_closest)
+def config_from_filepath(sources):
+    """Create a SafeConfigParser and read in the given `sources`, which can be
+    either a filename or a list of filenames."""
     cfg = SafeConfigParser()
     cfg.read(sources)
     return cfg
 
 
+def get_config(use_closest=True):
+    """Get Scrapy config file as a SafeConfigParser"""
+    sources = get_sources(use_closest)
+    return config_from_filepath(sources)
+
+
 def get_sources(use_closest=True):
     xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \
         os.path.expanduser('~/.config')

scrapy/utils/misc.py

@@ -1,5 +1,8 @@
 """Helper functions which doesn't fit anywhere else"""
+import itertools
+import os.path
 import re
+import sys
 import hashlib
 from importlib import import_module
 from pkgutil import iter_modules
@@ -56,6 +59,26 @@ def load_object(path):
     return obj
 
 
+def load_module_or_object(path):
+    """Load python module or (non-module) object from given path.
+
+    Path can be both a Python or a file path.
+    """
+    try:
+        return import_module(path)
+    except ImportError:
+        pass
+    try:
+        return load_object(path)
+    except (ValueError, NameError, ImportError):
+        pass
+    try:
+        return get_module_from_filepath(path)
+    except ImportError:
+        pass
+    raise NameError("Could not load '%s'" % path)
+
+
 def walk_modules(path):
     """Loads a module and all its submodules from a the given module path and
     returns them. If *any* module throws an exception while importing, that
@@ -78,6 +101,23 @@ def walk_modules(path):
     return mods
 
 
+def get_module_from_filepath(path):
+    """Load and return a python module/package from a file path"""
+    path = path.rstrip("/")
+    if path.endswith('.py'):
+        path = path.rsplit('.py', 1)[0]
+    basefolder, modname = os.path.split(path)
+    # XXX: There are other ways to import modules from a full path which don't
+    #      need to modify PYTHONPATH, see
+    #          https://stackoverflow.com/questions/67631/
+    #      These methods differ between py2 and py3, and apparently the
+    #      py3 method was deprecated in Python 3.4
+    sys.path.insert(0, basefolder)
+    mod = import_module(modname)
+    sys.path.pop(0)
+    return mod
+
+
 def extract_regex(regex, text, encoding='utf-8'):
     """Extract a list of unicode strings from the given text/encoding using the following policies:
 
@@ -118,7 +158,7 @@ def md5sum(file):
         m.update(d)
     return m.hexdigest()
 
+
 def rel_has_nofollow(rel):
     """Return True if link rel attribute has nofollow type"""
     return True if rel is not None and 'nofollow' in rel.split() else False
-    

scrapy/utils/project.py

@@ -71,3 +71,15 @@ def get_project_settings():
         settings.setdict(env_overrides, priority='project')
 
     return settings
+
+def get_project_path():
+    """Return the Python path of the current project.
+
+    This fails when the settings module does not live in the project's root.
+    """
+    if not inside_project():
+        raise NotConfigured("Not inside a project")
+    settings_module_path = os.environ.get(ENVVAR)
+    if not settings_module_path:
+        raise NotConfigured("Unable to locate project's python path")
+    return settings_module_path.rsplit('.', 1)[0]

tests/test_addons/__init__.py

@@ -0,0 +1,388 @@
+import os.path
+import six
+from six.moves.configparser import SafeConfigParser
+import sys
+from tests import mock
+import unittest
+import warnings
+
+from pkg_resources import VersionConflict
+import zope.interface
+from zope.interface.verify import verifyObject
+from zope.interface.exceptions import BrokenImplementation
+
+import scrapy.addons
+from scrapy.addons import Addon, AddonManager
+from scrapy.crawler import Crawler
+from scrapy.interfaces import IAddon
+from scrapy.settings import BaseSettings, Settings
+
+from . import addons
+from . import addonmod
+
+
+class AddonTest(unittest.TestCase):
+
+    def setUp(self):
+        self.rawaddon = Addon()
+        class AddonWithAttributes(Addon):
+            name = 'Test'
+            version = '1.0'
+        self.testaddon = AddonWithAttributes()
+
+    def test_interface(self):
+        # Raw Addon should fail exactly b/c name and version are not given
+        self.assertFalse(hasattr(self.rawaddon, 'name'))
+        self.assertFalse(hasattr(self.rawaddon, 'version'))
+        self.assertRaises(BrokenImplementation, verifyObject, IAddon,
+                          self.rawaddon)
+        verifyObject(IAddon, self.testaddon)
+
+    def test_export_component(self):
+        settings = BaseSettings({'ITEM_PIPELINES': {}}, 'default')
+        self.testaddon.component_type = None
+        self.testaddon.export_component({}, settings)
+        self.assertEqual(len(settings['ITEM_PIPELINES']), 0)
+        self.testaddon.component_type = 'ITEM_PIPELINES'
+        self.testaddon.component = 'test.component'
+        self.testaddon.export_component({}, settings)
+        six.assertCountEqual(self, settings['ITEM_PIPELINES'],
+                             ['test.component'])
+        self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 0)
+        self.testaddon.component_order = 313
+        self.testaddon.export_component({}, settings)
+        self.assertEqual(settings['ITEM_PIPELINES']['test.component'], 313)
+        self.testaddon.component_type = 'DOWNLOAD_HANDLERS'
+        self.testaddon.component_key = 'http'
+        self.testaddon.export_component({}, settings)
+        self.assertEqual(settings['DOWNLOAD_HANDLERS']['http'],
+                         'test.component')
+
+    def test_export_basics(self):
+        settings = BaseSettings()
+        self.testaddon.basic_settings = {'TESTKEY': 313, 'OTHERKEY': True}
+        self.testaddon.export_basics(settings)
+        self.assertEqual(settings['TESTKEY'], 313)
+        self.assertEqual(settings['OTHERKEY'], True)
+        self.assertEqual(settings.getpriority('TESTKEY'), 15)
+
+    def test_export_config(self):
+        settings = BaseSettings()
+        self.testaddon.settings_prefix = None
+        self.testaddon.config_mapping = {'MAPPED_key': 'MAPPING_WORKED'}
+        self.testaddon.default_config = {'key': 55, 'defaultkey': 100}
+        self.testaddon.export_config({'key': 313, 'OTHERKEY': True,
+                                     'mapped_KEY': 99}, settings)
+        self.assertEqual(settings['TEST_KEY'], 313)
+        self.assertEqual(settings['TEST_DEFAULTKEY'], 100)
+        self.assertEqual(settings['TEST_OTHERKEY'], True)
+        self.assertNotIn('MAPPED_key', settings)
+        self.assertNotIn('MAPPED_KEY', settings)
+        self.assertEqual(settings['MAPPING_WORKED'], 99)
+        self.assertEqual(settings.getpriority('TEST_KEY'), 15)
+
+        self.testaddon.settings_prefix = 'PREF'
+        self.testaddon.export_config({'newkey': 99}, settings)
+        self.assertEqual(settings['PREF_NEWKEY'], 99)
+
+        with mock.patch.object(settings, 'set') as mock_set:
+            self.testaddon.settings_prefix = False
+            self.testaddon.export_config({'thirdnewkey': 99}, settings)
+            self.assertEqual(mock_set.call_count, 0)
+
+    def test_update_settings(self):
+        settings = BaseSettings()
+        settings.set('TEST_KEY1', 'default', priority='default')
+        settings.set('TEST_KEY2', 'project', priority='project')
+        self.testaddon.settings_prefix = None
+        self.testaddon.basic_settings = {'OTHERTEST_KEY': 'addon'}
+        addon_config = {'key1': 'addon', 'key2': 'addon', 'key3': 'addon'}
+        self.testaddon.update_settings(addon_config, settings)
+        self.assertEqual(settings['OTHERTEST_KEY'], 'addon')
+        self.assertEqual(settings['TEST_KEY1'], 'addon')
+        self.assertEqual(settings['TEST_KEY2'], 'project')
+        self.assertEqual(settings['TEST_KEY3'], 'addon')
+
+
+class AddonManagerTest(unittest.TestCase):
+
+    TESTCFGPATH = os.path.join(os.path.dirname(__file__), 'cfg.cfg')
+    ADDONMODPATH = os.path.join(os.path.dirname(__file__), 'addonmod.py')
+
+    def setUp(self):
+        self.manager = AddonManager()
+
+    def test_add(self):
+        manager = AddonManager()
+        manager.add(addonmod, {'key': 'val1'})
+        manager.add('tests.test_addons.addons.GoodAddon')
+        six.assertCountEqual(self, manager, ['AddonModule', 'GoodAddon'])
+        self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon)
+        six.assertCountEqual(self, manager.configs['AddonModule'], ['key'])
+        self.assertEqual(manager.configs['AddonModule']['key'], 'val1')
+        self.assertRaises(ValueError, manager.add, addonmod)
+
+    def test_add_dont_instantiate_providing_classes(self):
+        class ProviderGoodAddon(addons.GoodAddon):
+            pass
+        zope.interface.directlyProvides(ProviderGoodAddon, IAddon)
+        manager = AddonManager()
+        manager.add(ProviderGoodAddon)
+        self.assertIs(manager['GoodAddon'], ProviderGoodAddon)
+
+    def test_add_verifies(self):
+        brokenaddon = self.manager.get_addon(
+                                    'tests.test_addons.addons.BrokenAddon')
+        self.assertRaises(zope.interface.exceptions.BrokenImplementation,
+                          self.manager.add,
+                          brokenaddon)
+
+    def test_add_adds_missing_interface_declaration(self):
+        class GoodAddonWithoutDeclaration(object):
+            name = 'GoodAddonWithoutDeclaration'
+            version = '1.0'
+        self.manager.add(GoodAddonWithoutDeclaration)
+
+    def test_remove(self):
+        manager = AddonManager()
+        def test_gets_removed(removearg):
+            manager.add(addonmod)
+            self.assertIn('AddonModule', manager)
+            manager.remove(removearg)
+            self.assertNotIn('AddonModule', manager)
+        test_gets_removed('AddonModule')
+        test_gets_removed(addonmod)
+        test_gets_removed('tests.test_addons.addonmod')
+        test_gets_removed(self.ADDONMODPATH)
+        self.assertRaises(KeyError, manager.remove, 'nonexistent')
+        self.assertRaises(KeyError, manager.remove, addons.GoodAddon())
+
+    def test_get_addon(self):
+        goodaddon = self.manager.get_addon(
+                                      'tests.test_addons.addons.GoodAddon')
+        self.assertIs(goodaddon, addons.GoodAddon)
+
+        loaded_addonmod = self.manager.get_addon(self.ADDONMODPATH)
+        # XXX: The module is in fact imported twice under different names into
+        #      sys.modules, is there a good assertion for module equality?
+        self.assertEqual(loaded_addonmod.name, addonmod.name)
+
+        # Does not provide interface, but has _addon attribute pointing to
+        # GoodAddon instance
+        addonspath = os.path.join(os.path.dirname(__file__), 'addons.py')
+        goodaddon = self.manager.get_addon(addonspath)
+        # XXX: Again, the imported class and addons.GoodAddon are different
+        #      since they are imported twice. How to use isInstance?
+        self.assertEqual(goodaddon.name, addons.GoodAddon.name)
+
+        self.assertRaises(NameError, self.manager.get_addon, 'xy.n_onexistent')
+
+    def test_get_addon_forward(self):
+        class SomeCls(object):
+            _addon = 'tests.test_addons.addons.GoodAddon'
+        self.assertIs(self.manager.get_addon(SomeCls()), addons.GoodAddon)
+
+    def test_get_addon_nested(self):
+        x = addons.GoodAddon('outer')
+        x._addon = addons.GoodAddon('middle')
+        x._addon._addon = addons.GoodAddon('inner')
+        self.assertIs(self.manager.get_addon(x), x._addon._addon)
+
+    @mock.patch.object(scrapy.addons, 'get_project_path',
+                       return_value='tests.test_addons.project')
+    def test_get_addon_prefixes(self, get_project_path_mock):
+        # From python path
+        self.assertEqual(self.manager.get_addon('addonmod').FROM,
+                         'test_addons.addonmod')
+
+        # From project 'addons' folder
+        self.assertEqual(self.manager.get_addon('addonmod2').FROM,
+                         'test_addons.project.addons.addonmod2')
+        # Assert prefix priority '' > 'project.addons'
+        self.assertEqual(self.manager.get_addon('addonmod').FROM,
+                         'test_addons.addonmod')
+
+        # From scrapy's 'addons'
+        from . import scrapy_addons
+        with mock.patch.dict('sys.modules', {'scrapy.addons': scrapy_addons}):
+            self.assertEqual(self.manager.get_addon('addonmod3').FROM,
+                             'test_addons.scrapy_addons.addonmod3')
+            # Assert prefix priority 'project.addons' > 'scrapy.addons'
+            self.assertEqual(self.manager.get_addon('addonmod2').FROM,
+                             'test_addons.project.addons.addonmod2')
+            # Assert prefix priority '' > 'scrapy.addons.'
+            self.assertEqual(self.manager.get_addon('addonmod').FROM,
+                             'test_addons.addonmod')
+
+    def test_load_dict_load_settings(self):
+        def _test_load_method(func, *args, **kwargs):
+            manager = AddonManager()
+            getattr(manager, func)(*args, **kwargs)
+            six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule'])
+            self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon)
+            six.assertCountEqual(self, manager.configs['GoodAddon'],
+                                       ['key'])
+            self.assertEqual(manager.configs['GoodAddon']['key'], 'val2')
+            # XXX: Check module equality, see above
+            self.assertEqual(manager['AddonModule'].name, addonmod.name)
+            self.assertIn('key', manager.configs['AddonModule'])
+            self.assertEqual(manager.configs['AddonModule']['key'], 'val1')
+
+        addonsdict = {
+            self.ADDONMODPATH: {
+                'key': 'val1',
+                },
+            'tests.test_addons.addons.GoodAddon': {'key': 'val2'},
+            }
+        _test_load_method('load_dict', addonsdict)
+
+        settings = BaseSettings()
+        settings.set('INSTALLED_ADDONS', [
+                self.ADDONMODPATH,
+                'tests.test_addons.addons.GoodAddon',
+                ])
+        settings.set('ADDONMODULE', {'key': 'val1'})
+        settings.set('GOODADDON', {'key': 'val2'})
+        _test_load_method('load_settings', settings)
+
+    def test_load_cfg(self):
+        manager = AddonManager()
+        manager.load_cfg(self.TESTCFGPATH)
+        six.assertCountEqual(self, manager, ['GoodAddon', 'AddonModule'])
+        self.assertIsInstance(manager['GoodAddon'], addons.GoodAddon)
+        six.assertCountEqual(self, manager.configs['GoodAddon'], ['key'])
+        self.assertEqual(manager.configs['GoodAddon']['key'], 'val1')
+        # XXX: Check module equality, see above
+        self.assertEqual(manager['AddonModule'].name, addonmod.name)
+        six.assertCountEqual(self, manager.configs['AddonModule'], ['key'])
+        self.assertEqual(manager.configs['AddonModule']['key'], 'val2')
+
+    def test_enabled_disabled(self):
+        manager = AddonManager()
+        manager.add(addons.GoodAddon('FirstAddon'))
+        manager.add(addons.GoodAddon('SecondAddon'))
+        self.assertEqual(set(manager.enabled),
+                         set(('FirstAddon', 'SecondAddon')))
+        self.assertEqual(manager.disabled, [])
+        manager.disable('FirstAddon')
+        self.assertEqual(manager.enabled, ['SecondAddon'])
+        self.assertEqual(manager.disabled, ['FirstAddon'])
+        manager.enable('FirstAddon')
+        self.assertEqual(set(manager.enabled),
+                         set(('FirstAddon', 'SecondAddon')))
+        self.assertEqual(manager.disabled, [])
+
+    def test_enable_before_add(self):
+        manager = AddonManager()
+        self.assertRaises(ValueError, manager.enable, 'FirstAddon')
+        manager.disable('FirstAddon')
+        manager.enable('FirstAddon')
+        manager.add(addons.GoodAddon('FirstAddon'))
+        self.assertIn('FirstAddon', manager.enabled)
+
+    def test_disable_before_add(self):
+        manager = AddonManager()
+        manager.disable('FirstAddon')
+        manager.add(addons.GoodAddon('FirstAddon'))
+        self.assertEqual(manager.disabled, ['FirstAddon'])
+
+    def test_callbacks(self):
+        first_addon = addons.GoodAddon('FirstAddon')
+        second_addon = addons.GoodAddon('SecondAddon')
+
+        manager = AddonManager()
+        manager.add(first_addon, {'test': 'first'})
+        manager.add(second_addon, {'test': 'second'})
+        crawler = mock.create_autospec(Crawler)
+        settings = BaseSettings()
+
+        with mock.patch.object(first_addon, 'update_addons') as ua_first, \
+             mock.patch.object(second_addon, 'update_addons') as ua_second, \
+             mock.patch.object(first_addon, 'update_settings') as us_first, \
+             mock.patch.object(second_addon, 'update_settings') as us_second, \
+             mock.patch.object(first_addon, 'check_configuration') as cc_first, \
+             mock.patch.object(second_addon, 'check_configuration') as cc_second:
+            manager.update_addons()
+            ua_first.assert_called_once_with(manager.configs['FirstAddon'],
+                                             manager)
+            ua_second.assert_called_once_with(manager.configs['SecondAddon'],
+                                             manager)
+            manager.update_settings(settings)
+            us_first.assert_called_once_with(manager.configs['FirstAddon'],
+                                             settings)
+            us_second.assert_called_once_with(manager.configs['SecondAddon'],
+                                             settings)
+            manager.check_configuration(crawler)
+            cc_first.assert_called_once_with(manager.configs['FirstAddon'],
+                                             crawler)
+            cc_second.assert_called_once_with(manager.configs['SecondAddon'],
+                                             crawler)
+            self.assertEqual(ua_first.call_count, 1)
+            self.assertEqual(ua_second.call_count, 1)
+            self.assertEqual(us_first.call_count, 1)
+            self.assertEqual(us_second.call_count, 1)
+
+            us_first.reset_mock()
+            us_second.reset_mock()
+            manager.disable('FirstAddon')
+            manager.update_settings(settings)
+            self.assertEqual(us_first.call_count, 0)
+            manager.enable('FirstAddon')
+            manager.update_settings(settings)
+            self.assertEqual(us_first.call_count, 1)
+            self.assertEqual(us_second.call_count, 2)
+
+    def test_update_addons_last_minute_add(self):
+        class AddedAddon(addons.GoodAddon):
+            name = 'AddedAddon'
+
+        class FirstAddon(addons.GoodAddon):
+            name = 'FirstAddon'
+            def update_addons(self, config, addons):
+                addons.add(AddedAddon())
+
+        manager = AddonManager()
+        first_addon = FirstAddon()
+        with mock.patch.object(first_addon, 'update_addons',
+                               wraps=first_addon.update_addons) as ua_first, \
+             mock.patch.object(AddedAddon, 'update_addons') as ua_added:
+            manager.add(first_addon, {'non-empty': 'dict'})
+            manager.update_addons()
+            six.assertCountEqual(self, manager, ['FirstAddon', 'AddedAddon'])
+            ua_first.assert_called_once_with(manager.configs['FirstAddon'],
+                                             manager)
+            ua_added.assert_called_once_with(manager.configs['AddedAddon'],
+                                             manager)
+
+    def test_check_dependency_clashes_attributes(self):
+        provides = addons.GoodAddon("ProvidesAddon")
+        provides.provides = ('test', )
+        provides2 = addons.GoodAddon("ProvidesAddon2")
+        provides2.provides = ('test', )
+        requires = addons.GoodAddon("RequiresAddon")
+        requires.requires = ('test', )
+        requires_name = addons.GoodAddon("RequiresNameAddon")
+        requires_name.requires = ('ProvidesAddon', )
+        requires_newer = addons.GoodAddon("RequiresNewerAddon")
+        requires_newer.requires = ('test>=2.0', )
+        modifies = addons.GoodAddon("ModifiesAddon")
+        modifies.modifies = ('test', )
+
+        def check_with(*addons):
+            manager = AddonManager()
+            for a in addons:
+                manager.add(a)
+            return manager.check_dependency_clashes()
+
+        self.assertRaises(ImportError, check_with, requires)
+        self.assertRaises(ImportError, check_with, modifies)
+        self.assertRaises(ImportError, check_with, provides, provides2)
+        self.assertRaises(VersionConflict, check_with, provides, requires_newer)
+        with warnings.catch_warnings(record=True) as w:
+            check_with(provides, modifies)
+            check_with(provides)
+            check_with(provides, requires)
+            check_with(provides, requires_name)
+            self.assertEqual(len(w), 0)
+            check_with(requires, provides, modifies)
+            self.assertEqual(len(w), 1)

tests/test_addons/addonmod.py

@@ -0,0 +1,16 @@
+import zope.interface
+
+from scrapy.interfaces import IAddon
+
+zope.interface.moduleProvides(IAddon)
+
+FROM = "test_addons.addonmod"
+
+name = "AddonModule"
+version = "1.0"
+
+def update_settings(config, settings):
+    pass
+
+def check_configuration(config, crawler):
+    pass

tests/test_addons/addons.py

@@ -0,0 +1,40 @@
+import zope.interface
+
+from scrapy.addons import Addon
+from scrapy.interfaces import IAddon
+
+
+class Addon(object):
+    FROM = 'test_addons.addons'
+
+
+@zope.interface.declarations.implementer(IAddon)
+class GoodAddon(object):
+
+    name = 'GoodAddon'
+    version = '1.0'
+
+    def __init__(self, name=None, version=None):
+        if name is not None:
+            self.name = name
+        if version is not None:
+            self.version = version
+
+    def update_addons(self, config, addons):
+        pass
+
+    def update_settings(self, config, settings):
+        pass
+
+    def check_configuration(self, config, crawler):
+        pass
+
+
+@zope.interface.declarations.implementer(IAddon)
+class BrokenAddon(object):
+
+    name = 'BrokenAddon'
+    # No version
+
+
+_addon = GoodAddon()

tests/test_addons/cfg.cfg

@@ -0,0 +1,5 @@
+[addon:tests.test_addons.addons.GoodAddon]
+key = val1
+
+[addon:tests/test_addons/addonmod.py]
+key = val2

tests/test_addons/project/addons/addonmod.py

@@ -0,0 +1,7 @@
+import zope.interface
+
+from scrapy.interfaces import IAddon
+
+zope.interface.moduleProvides(IAddon)
+
+FROM = 'test_addons.project.addons.addonmod'

tests/test_addons/project/addons/addonmod2.py

@@ -0,0 +1,7 @@
+import zope.interface
+
+from scrapy.interfaces import IAddon
+
+zope.interface.moduleProvides(IAddon)
+
+FROM = 'test_addons.project.addons.addonmod2'

tests/test_addons/scrapy_addons/addonmod.py

@@ -0,0 +1,7 @@
+import zope.interface
+
+from scrapy.interfaces import IAddon
+
+zope.interface.moduleProvides(IAddon)
+
+FROM = 'test_addons.scrapy_addons.addonmod'

tests/test_addons/scrapy_addons/addonmod2.py

@@ -0,0 +1,7 @@
+import zope.interface
+
+from scrapy.interfaces import IAddon
+
+zope.interface.moduleProvides(IAddon)
+
+FROM = 'test_addons.scrapy_addons.addonmod2'

tests/test_addons/scrapy_addons/addonmod3.py

@@ -0,0 +1,7 @@
+import zope.interface
+
+from scrapy.interfaces import IAddon
+
+zope.interface.moduleProvides(IAddon)
+
+FROM = 'test_addons.scrapy_addons.addonmod3'

tests/test_utils_misc/__init__.py

@@ -3,7 +3,8 @@ import os
 import unittest
 
 from scrapy.item import Item, Field
-from scrapy.utils.misc import load_object, arg_to_iter, walk_modules
+from scrapy.utils.misc import (load_object, load_module_or_object, arg_to_iter,
+                               walk_modules, get_module_from_filepath)
 
 __doctests__ = ['scrapy.utils.misc']
 
@@ -17,6 +18,15 @@ class UtilsMiscTestCase(unittest.TestCase):
         self.assertRaises(ImportError, load_object, 'nomodule999.mod.function')
         self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999')
 
+    def test_load_module_or_object(self):
+        testmod = load_module_or_object(__name__ + '.testmod')
+        self.assertTrue(hasattr(testmod, 'TESTVAR'))
+        testmod = load_module_or_object(
+                    os.path.join(os.path.dirname(__file__), 'testmod.py'))
+        self.assertTrue(hasattr(testmod, 'TESTVAR'))
+        obj = load_object('scrapy.utils.misc.load_object')
+        self.assertIs(obj, load_object)
+
     def test_walk_modules(self):
         mods = walk_modules('tests.test_utils_misc.test_walk_modules')
         expected = [
@@ -57,6 +67,20 @@ class UtilsMiscTestCase(unittest.TestCase):
         finally:
             sys.path.remove(egg)
 
+    def test_get_module_from_filepath(self):
+        testmodpath = os.path.join(os.path.dirname(__file__), 'testmod.py')
+        testmod = get_module_from_filepath(testmodpath)
+        self.assertTrue(hasattr(testmod, 'TESTVAR'))
+
+        testpkgpath = os.path.join(os.path.dirname(__file__), 'testpkg')
+        testpkg = get_module_from_filepath(testpkgpath)
+        self.assertTrue(hasattr(testpkg, 'TESTVAR2'))
+        # Check submodule access
+        import testpkg.submod
+        self.assertTrue(hasattr(testpkg.submod, 'TESTVAR3'))
+        self.assertIs(testpkg.submod.TESTVAR3,
+                      load_object(testpkg.__name__ + ".submod.TESTVAR3"))
+
     def test_arg_to_iter(self):
 
         class TestItem(Item):

tests/test_utils_misc/testmod.py

@@ -0,0 +1 @@
+TESTVAR = True

tests/test_utils_misc/testpkg/__init__.py

@@ -0,0 +1 @@
+TESTVAR2 = True

tests/test_utils_misc/testpkg/submod.py

@@ -0,0 +1 @@
+TESTVAR3 = True

tests/test_utils_project.py

@@ -0,0 +1,27 @@
+import os
+from tests import mock
+import unittest
+
+from scrapy.exceptions import NotConfigured
+from scrapy.utils.project import get_project_path, inside_project
+
+
+class UtilsProjectTestCase(unittest.TestCase):
+
+    @mock.patch('scrapy.utils.project.inside_project', return_value=True)
+    def test_get_project_path(self, mock_ip):
+        def _test(settingsmod, expected):
+            with mock.patch.dict('os.environ',
+                                 {'SCRAPY_SETTINGS_MODULE': settingsmod}):
+                self.assertEqual(get_project_path(), expected)
+        _test('project.settings', 'project')
+        _test('project.othername', 'project')
+        _test('nested.project.settings', 'nested.project')
+
+        with mock.patch.dict('os.environ', {}, clear=True):
+            self.assertRaises(NotConfigured, get_project_path)
+
+        mock_ip.return_value = False
+        with mock.patch.dict('os.environ',
+                             {'SCRAPY_SETTINGS_MODULE': 'some.settings'}):
+            self.assertRaises(NotConfigured, get_project_path)