mirror of https://github.com/scrapy/scrapy.git
Remove some dead code/docs.
This commit is contained in:
parent
c7f78a8305
commit
2f9ebb66c3
|
|
@ -17,16 +17,7 @@ Activating and configuring add-ons
|
|||
Add-ons and their configuration live in Scrapy's
|
||||
:class:`~scrapy.addons.AddonManager`. During Scrapy's start-up process, and
|
||||
only then, the add-on manager will read a list of enabled add-ons and their
|
||||
configurations from your settings. There are two places where you can provide
|
||||
the paths to add-ons you want to enable:
|
||||
|
||||
* the ``ADDONS`` setting, and
|
||||
* the ``scrapy.cfg`` file.
|
||||
|
||||
As Scrapy settings can be modified from many places, e.g. in a project's
|
||||
``settings.py``, in a Spider's ``custom_settings`` attribute, or from the
|
||||
command line, using the ``ADDONS`` setting is the preferred way to
|
||||
manage add-ons.
|
||||
configurations from your ``ADDONS`` setting.
|
||||
|
||||
The ``ADDONS`` setting a tuple in which every item is a path to an
|
||||
add-on. The path can be both a Python or a file path. While more precise, it is
|
||||
|
|
@ -54,30 +45,11 @@ case with one requiring no configuration) are enabled/configured in a project's
|
|||
'some_config': True,
|
||||
}
|
||||
|
||||
It is also possible to manage add-ons from ``scrapy.cfg``. While the syntax is
|
||||
a little friendlier, be aware that this file, and therefore the configuration in
|
||||
it, is not bound to a particular Scrapy project. While this should not pose a
|
||||
problem when you use the project on your development machine only, a common
|
||||
stumbling block is that ``scrapy.cfg`` is not deployed via ``scrapyd-deploy``.
|
||||
|
||||
In ``scrapy.cfg``, section names, prepended with ``addon:``, replace the
|
||||
dictionary keys. I.e., the configuration from above would look like this:
|
||||
|
||||
.. code-block:: cfg
|
||||
|
||||
[addon:httpcache]
|
||||
expiration_secs = 60
|
||||
ignore_http_codes = 404,405
|
||||
|
||||
[addon:path.to.some.addon]
|
||||
some_config = true
|
||||
|
||||
|
||||
Enabling and configuring add-ons within Python code
|
||||
---------------------------------------------------
|
||||
|
||||
The :class:`~scrapy.addons.AddonManager` will only read from Scrapy's settings
|
||||
and from ``scrapy.cfg`` *at the beginning* of Scrapy's start-up process.
|
||||
*at the beginning* of Scrapy's start-up process.
|
||||
Afterwards, i.e. as soon as the :class:`~scrapy.addons.AddonManager` is
|
||||
populated, changing the ``ADDONS`` setting or any of the add-on
|
||||
configuration dictionary settings will have no effect.
|
||||
|
|
|
|||
|
|
@ -261,32 +261,6 @@ class AddonManager(Mapping):
|
|||
obj = AddonManager.get_addon(obj._addon)
|
||||
return obj
|
||||
|
||||
def load_dict(self, addonsdict):
|
||||
"""Load add-ons and configurations from given dictionary.
|
||||
|
||||
Each add-on should be an entry in the dictionary, where the key
|
||||
corresponds to the add-on path. The value should be a dictionary
|
||||
representing the add-on configuration.
|
||||
|
||||
Example add-on dictionary::
|
||||
|
||||
addonsdict = {
|
||||
'path.to.addon1': {
|
||||
'setting1': 'value',
|
||||
'setting2': 42,
|
||||
},
|
||||
'path/to/addon2.py': {
|
||||
'addon2setting': True,
|
||||
},
|
||||
}
|
||||
|
||||
:param addonsdict: dictionary where keys correspond to add-on paths \
|
||||
and values correspond to their configuration
|
||||
:type addonsdict: ``dict``
|
||||
"""
|
||||
for addonpath, addoncfg in addonsdict.items():
|
||||
self.add(addonpath, addoncfg)
|
||||
|
||||
def load_settings(self, settings):
|
||||
"""Load add-ons and configurations from settings object.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import itertools
|
||||
import unittest
|
||||
import warnings
|
||||
from collections import OrderedDict
|
||||
from unittest import mock
|
||||
|
||||
from zope.interface import directlyProvides
|
||||
|
|
@ -178,24 +177,7 @@ class AddonManagerTest(unittest.TestCase):
|
|||
x._addon._addon = addons.GoodAddon("inner")
|
||||
self.assertIs(self.manager.get_addon(x), x._addon._addon)
|
||||
|
||||
def test_load_dict_load_settings(self):
|
||||
def _test_load_method(func, *args, **kwargs):
|
||||
manager = AddonManager()
|
||||
getattr(manager, func)(*args, **kwargs)
|
||||
self.assertCountEqual(manager, ["GoodAddon", "AddonModule"])
|
||||
self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon)
|
||||
self.assertCountEqual(manager.configs["GoodAddon"], ["key"])
|
||||
self.assertEqual(manager.configs["GoodAddon"]["key"], "val2")
|
||||
self.assertEqual(manager["AddonModule"], addonmod)
|
||||
self.assertIn("key", manager.configs["AddonModule"])
|
||||
self.assertEqual(manager.configs["AddonModule"]["key"], "val1")
|
||||
|
||||
addonsdict = {
|
||||
"tests.test_addons.addonmod": {"key": "val1"},
|
||||
"tests.test_addons.addons.GoodAddon": {"key": "val2"},
|
||||
}
|
||||
_test_load_method("load_dict", addonsdict)
|
||||
|
||||
def test_load_settings(self):
|
||||
settings = BaseSettings()
|
||||
settings.set(
|
||||
"ADDONS",
|
||||
|
|
@ -203,25 +185,28 @@ class AddonManagerTest(unittest.TestCase):
|
|||
)
|
||||
settings.set("ADDONMODULE", {"key": "val1"})
|
||||
settings.set("GOODADDON", {"key": "val2"})
|
||||
_test_load_method("load_settings", settings)
|
||||
|
||||
def test_load_dict_load_settings_order(self):
|
||||
def _test_load_method(expected_order, func, *args, **kwargs):
|
||||
manager = AddonManager()
|
||||
getattr(manager, func)(*args, **kwargs)
|
||||
self.assertEqual(list(manager.keys()), expected_order)
|
||||
manager = AddonManager()
|
||||
manager.load_settings(settings)
|
||||
self.assertCountEqual(manager, ["GoodAddon", "AddonModule"])
|
||||
self.assertIsInstance(manager["GoodAddon"], addons.GoodAddon)
|
||||
self.assertCountEqual(manager.configs["GoodAddon"], ["key"])
|
||||
self.assertEqual(manager.configs["GoodAddon"]["key"], "val2")
|
||||
self.assertEqual(manager["AddonModule"], addonmod)
|
||||
self.assertIn("key", manager.configs["AddonModule"])
|
||||
self.assertEqual(manager.configs["AddonModule"]["key"], "val1")
|
||||
|
||||
def test_load_settings_order(self):
|
||||
# Get three addons named 0, 1, 2
|
||||
addonlist = [addons.GoodAddon(str(x)) for x in range(3)]
|
||||
# Test both methods for every possible mutation
|
||||
for ordered_addons in itertools.permutations(addonlist):
|
||||
expected_order = [a.name for a in ordered_addons]
|
||||
addonsdict = OrderedDict((a, {}) for a in ordered_addons)
|
||||
_test_load_method(expected_order, "load_dict", addonsdict)
|
||||
settings = BaseSettings(
|
||||
{"ADDONS": {a: i for i, a in enumerate(ordered_addons)}}
|
||||
)
|
||||
_test_load_method(expected_order, "load_settings", settings)
|
||||
manager = AddonManager()
|
||||
manager.load_settings(settings)
|
||||
self.assertEqual(list(manager.keys()), expected_order)
|
||||
|
||||
def test_enabled_disabled(self):
|
||||
manager = AddonManager()
|
||||
|
|
|
|||
Loading…
Reference in New Issue