mirror of https://github.com/scrapy/scrapy.git
Extend BaseSettings with utils for add-ons (#6614)
This commit is contained in:
Adrián Chaves
GitHub
parent
d161d1d47d
commit
0c9200094e
|
|
@ -76,15 +76,11 @@ The settings set by the add-on should use the ``addon`` priority (see
|
|||
settings.set("DNSCACHE_ENABLED", True, "addon")
|
||||
|
||||
This allows users to override these settings in the project or spider
|
||||
configuration. This is not possible with settings that are mutable objects,
|
||||
such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases
|
||||
you can provide an add-on-specific setting that governs whether the add-on will
|
||||
modify :setting:`ITEM_PIPELINES`::
|
||||
configuration.
|
||||
|
||||
class MyAddon:
|
||||
def update_settings(self, settings):
|
||||
if settings.getbool("MYADDON_ENABLE_PIPELINE"):
|
||||
settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200
|
||||
When editing the value of a setting instead of overriding it entirely, it is
|
||||
usually best to leave its priority unchanged. For example, when editing a
|
||||
:ref:`component priority dictionary <component-priority-dictionaries>`.
|
||||
|
||||
If the ``update_settings`` method raises
|
||||
:exc:`scrapy.exceptions.NotConfigured`, the add-on will be skipped. This makes
|
||||
|
|
@ -127,12 +123,28 @@ Add-on examples
|
|||
|
||||
Set some basic configuration:
|
||||
|
||||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
||||
from myproject.pipelines import MyPipeline
|
||||
|
||||
|
||||
class MyAddon:
|
||||
def update_settings(self, settings):
|
||||
settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200
|
||||
settings.set("DNSCACHE_ENABLED", True, "addon")
|
||||
settings.remove_from_list("METAREFRESH_IGNORE_TAGS", "noscript")
|
||||
settings.setdefault_in_component_priority_dict(
|
||||
"ITEM_PIPELINES", MyPipeline, 200
|
||||
)
|
||||
|
||||
.. tip:: When editing a :ref:`component priority dictionary
|
||||
<component-priority-dictionaries>` setting, like :setting:`ITEM_PIPELINES`,
|
||||
consider using setting methods like
|
||||
:meth:`~scrapy.settings.BaseSettings.replace_in_component_priority_dict`,
|
||||
:meth:`~scrapy.settings.BaseSettings.set_in_component_priority_dict`
|
||||
and
|
||||
:meth:`~scrapy.settings.BaseSettings.setdefault_in_component_priority_dict`
|
||||
to avoid mistakes.
|
||||
|
||||
Check dependencies:
|
||||
|
||||
|
|
|
|||
|
|
@ -250,6 +250,57 @@ example, proper setting names for a fictional robots.txt extension would be
|
|||
``ROBOTSTXT_ENABLED``, ``ROBOTSTXT_OBEY``, ``ROBOTSTXT_CACHEDIR``, etc.
|
||||
|
||||
|
||||
.. _component-priority-dictionaries:
|
||||
|
||||
Component priority dictionaries
|
||||
===============================
|
||||
|
||||
A **component priority dictionary** is a :class:`dict` where keys are
|
||||
:ref:`components <topics-components>` and values are component priorities. For
|
||||
example:
|
||||
|
||||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
||||
{
|
||||
"path.to.ComponentA": None,
|
||||
ComponentB: 100,
|
||||
}
|
||||
|
||||
A component can be specified either as a class object or through an import
|
||||
path.
|
||||
|
||||
.. warning:: Component priority dictionaries are regular :class:`dict` objects.
|
||||
Be careful not to define the same component more than once, e.g. with
|
||||
different import path strings or defining both an import path and a
|
||||
:class:`type` object.
|
||||
|
||||
A priority can be an :class:`int` or :data:`None`.
|
||||
|
||||
A component with priority 1 goes *before* a component with priority 2. What
|
||||
going before entails, however, depends on the corresponding setting. For
|
||||
example, in the :setting:`DOWNLOADER_MIDDLEWARES` setting, components have
|
||||
their
|
||||
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request`
|
||||
method executed before that of later components, but have their
|
||||
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`
|
||||
method executed after that of later components.
|
||||
|
||||
A component with priority :data:`None` is disabled.
|
||||
|
||||
Some component priority dictionaries get merged with some built-in value. For
|
||||
example, :setting:`DOWNLOADER_MIDDLEWARES` is merged with
|
||||
:setting:`DOWNLOADER_MIDDLEWARES_BASE`. This is where :data:`None` comes in
|
||||
handy, allowing you to disable a component from the base setting in the regular
|
||||
setting:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
DOWNLOADER_MIDDLEWARES = {
|
||||
"scrapy.downloadermiddlewares.offsite.OffsiteMiddleware": None,
|
||||
}
|
||||
|
||||
|
||||
Special settings
|
||||
================
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from pprint import pformat
|
|||
from typing import TYPE_CHECKING, Any, Union, cast
|
||||
|
||||
from scrapy.settings import default_settings
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
# The key types are restricted in BaseSettings._get_key() to ones supported by JSON,
|
||||
# see https://github.com/scrapy/scrapy/issues/5383.
|
||||
|
|
@ -111,6 +112,31 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
|
|||
def __contains__(self, name: Any) -> bool:
|
||||
return name in self.attributes
|
||||
|
||||
def add_to_list(self, name: _SettingsKeyT, item: Any) -> None:
|
||||
"""Append *item* to the :class:`list` setting with the specified *name*
|
||||
if *item* is not already in that list.
|
||||
|
||||
This change is applied regardless of the priority of the *name*
|
||||
setting. The setting priority is not affected by this change either.
|
||||
"""
|
||||
value: list[str] = self.getlist(name)
|
||||
if item not in value:
|
||||
self.set(name, [*value, item], self.getpriority(name) or 0)
|
||||
|
||||
def remove_from_list(self, name: _SettingsKeyT, item: Any) -> None:
|
||||
"""Remove *item* from the :class:`list` setting with the specified
|
||||
*name*.
|
||||
|
||||
If *item* is missing, raise :exc:`ValueError`.
|
||||
|
||||
This change is applied regardless of the priority of the *name*
|
||||
setting. The setting priority is not affected by this change either.
|
||||
"""
|
||||
value: list[str] = self.getlist(name)
|
||||
if item not in value:
|
||||
raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).")
|
||||
self.set(name, [v for v in value if v != item], self.getpriority(name) or 0)
|
||||
|
||||
def get(self, name: _SettingsKeyT, default: Any = None) -> Any:
|
||||
"""
|
||||
Get a setting value without affecting its original type.
|
||||
|
|
@ -181,8 +207,9 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
|
|||
self, name: _SettingsKeyT, default: list[Any] | None = None
|
||||
) -> list[Any]:
|
||||
"""
|
||||
Get a setting value as a list. If the setting original type is a list, a
|
||||
copy of it will be returned. If it's a string it will be split by ",".
|
||||
Get a setting value as a list. If the setting original type is a list,
|
||||
a copy of it will be returned. If it's a string it will be split by
|
||||
",". If it is an empty string, an empty list will be returned.
|
||||
|
||||
For example, settings populated through environment variables set to
|
||||
``'one,two'`` will return a list ['one', 'two'] when using this method.
|
||||
|
|
@ -194,6 +221,8 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
|
|||
:type default: object
|
||||
"""
|
||||
value = self.get(name, default or [])
|
||||
if not value:
|
||||
return []
|
||||
if isinstance(value, str):
|
||||
value = value.split(",")
|
||||
return list(value)
|
||||
|
|
@ -299,6 +328,47 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
|
|||
return max(cast(int, self.getpriority(name)) for name in self)
|
||||
return get_settings_priority("default")
|
||||
|
||||
def replace_in_component_priority_dict(
|
||||
self,
|
||||
name: _SettingsKeyT,
|
||||
old_cls: type,
|
||||
new_cls: type,
|
||||
priority: int | None = None,
|
||||
) -> None:
|
||||
"""Replace *old_cls* with *new_cls* in the *name* :ref:`component
|
||||
priority dictionary <component-priority-dictionaries>`.
|
||||
|
||||
If *old_cls* is missing, or has :data:`None` as value, :exc:`KeyError`
|
||||
is raised.
|
||||
|
||||
If *old_cls* was present as an import string, even more than once,
|
||||
those keys are dropped and replaced by *new_cls*.
|
||||
|
||||
If *priority* is specified, that is the value assigned to *new_cls* in
|
||||
the component priority dictionary. Otherwise, the value of *old_cls* is
|
||||
used. If *old_cls* was present multiple times (possible with import
|
||||
strings) with different values, the value assigned to *new_cls* is one
|
||||
of them, with no guarantee about which one it is.
|
||||
|
||||
This change is applied regardless of the priority of the *name*
|
||||
setting. The setting priority is not affected by this change either.
|
||||
"""
|
||||
component_priority_dict = self.getdict(name)
|
||||
old_priority = None
|
||||
for cls_or_path in tuple(component_priority_dict):
|
||||
if load_object(cls_or_path) != old_cls:
|
||||
continue
|
||||
if (old_priority := component_priority_dict.pop(cls_or_path)) is None:
|
||||
break
|
||||
if old_priority is None:
|
||||
raise KeyError(
|
||||
f"{old_cls} not found in the {name} setting ({component_priority_dict!r})."
|
||||
)
|
||||
component_priority_dict[new_cls] = (
|
||||
old_priority if priority is None else priority
|
||||
)
|
||||
self.set(name, component_priority_dict, priority=self.getpriority(name) or 0)
|
||||
|
||||
def __setitem__(self, name: _SettingsKeyT, value: Any) -> None:
|
||||
self.set(name, value)
|
||||
|
||||
|
|
@ -332,6 +402,30 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
|
|||
else:
|
||||
self.attributes[name].set(value, priority)
|
||||
|
||||
def set_in_component_priority_dict(
|
||||
self, name: _SettingsKeyT, cls: type, priority: int | None
|
||||
) -> None:
|
||||
"""Set the *cls* component in the *name* :ref:`component priority
|
||||
dictionary <component-priority-dictionaries>` setting with *priority*.
|
||||
|
||||
If *cls* already exists, its value is updated.
|
||||
|
||||
If *cls* was present as an import string, even more than once, those
|
||||
keys are dropped and replaced by *cls*.
|
||||
|
||||
This change is applied regardless of the priority of the *name*
|
||||
setting. The setting priority is not affected by this change either.
|
||||
"""
|
||||
component_priority_dict = self.getdict(name)
|
||||
for cls_or_path in tuple(component_priority_dict):
|
||||
if not isinstance(cls_or_path, str):
|
||||
continue
|
||||
_cls = load_object(cls_or_path)
|
||||
if _cls == cls:
|
||||
del component_priority_dict[cls_or_path]
|
||||
component_priority_dict[cls] = priority
|
||||
self.set(name, component_priority_dict, self.getpriority(name) or 0)
|
||||
|
||||
def setdefault(
|
||||
self,
|
||||
name: _SettingsKeyT,
|
||||
|
|
@ -344,6 +438,24 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
|
|||
|
||||
return self.attributes[name].value
|
||||
|
||||
def setdefault_in_component_priority_dict(
|
||||
self, name: _SettingsKeyT, cls: type, priority: int | None
|
||||
) -> None:
|
||||
"""Set the *cls* component in the *name* :ref:`component priority
|
||||
dictionary <component-priority-dictionaries>` setting with *priority*
|
||||
if not already defined (even as an import string).
|
||||
|
||||
If *cls* is not already defined, it is set regardless of the priority
|
||||
of the *name* setting. The setting priority is not affected by this
|
||||
change either.
|
||||
"""
|
||||
component_priority_dict = self.getdict(name)
|
||||
for cls_or_path in tuple(component_priority_dict):
|
||||
if load_object(cls_or_path) == cls:
|
||||
return
|
||||
component_priority_dict[cls] = priority
|
||||
self.set(name, component_priority_dict, self.getpriority(name) or 0)
|
||||
|
||||
def setdict(self, values: _SettingsInputT, priority: int | str = "project") -> None:
|
||||
self.update(values, priority)
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,8 @@ def build_component_list(
|
|||
*,
|
||||
convert: Callable[[Any], Any] = update_classpath,
|
||||
) -> list[Any]:
|
||||
"""Compose a component list from a { class: order } dictionary."""
|
||||
"""Compose a component list from a :ref:`component priority dictionary
|
||||
<component-priority-dictionaries>`."""
|
||||
|
||||
def _check_components(complist: Collection[Any]) -> None:
|
||||
if len({convert(c) for c in complist}) != len(complist):
|
||||
|
|
|
|||
|
|
@ -260,6 +260,7 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
"TEST_FLOAT2": "123.45",
|
||||
"TEST_LIST1": ["one", "two"],
|
||||
"TEST_LIST2": "one,two",
|
||||
"TEST_LIST3": "",
|
||||
"TEST_STR": "value",
|
||||
"TEST_DICT1": {"key1": "val1", "ke2": 3},
|
||||
"TEST_DICT2": '{"key1": "val1", "ke2": 3}',
|
||||
|
|
@ -292,6 +293,7 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
self.assertEqual(settings.getfloat("TEST_FLOATx", 55.0), 55.0)
|
||||
self.assertEqual(settings.getlist("TEST_LIST1"), ["one", "two"])
|
||||
self.assertEqual(settings.getlist("TEST_LIST2"), ["one", "two"])
|
||||
self.assertEqual(settings.getlist("TEST_LIST3"), [])
|
||||
self.assertEqual(settings.getlist("TEST_LISTx"), [])
|
||||
self.assertEqual(settings.getlist("TEST_LISTx", ["default"]), ["default"])
|
||||
self.assertEqual(settings["TEST_STR"], "value")
|
||||
|
|
@ -504,3 +506,591 @@ class SettingsTest(unittest.TestCase):
|
|||
TypeError, match="Trying to modify an immutable Settings object"
|
||||
):
|
||||
settings.pop("OTHER_DUMMY_CONFIG")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("before", "name", "item", "after"),
|
||||
[
|
||||
({}, "FOO", "BAR", {"FOO": ["BAR"]}),
|
||||
({"FOO": []}, "FOO", "BAR", {"FOO": ["BAR"]}),
|
||||
({"FOO": ["BAR"]}, "FOO", "BAZ", {"FOO": ["BAR", "BAZ"]}),
|
||||
({"FOO": ["BAR"]}, "FOO", "BAR", {"FOO": ["BAR"]}),
|
||||
({"FOO": ""}, "FOO", "BAR", {"FOO": ["BAR"]}),
|
||||
({"FOO": "BAR"}, "FOO", "BAR", {"FOO": "BAR"}),
|
||||
({"FOO": "BAR"}, "FOO", "BAZ", {"FOO": ["BAR", "BAZ"]}),
|
||||
({"FOO": "BAR,BAZ"}, "FOO", "BAZ", {"FOO": "BAR,BAZ"}),
|
||||
({"FOO": "BAR,BAZ"}, "FOO", "QUX", {"FOO": ["BAR", "BAZ", "QUX"]}),
|
||||
],
|
||||
)
|
||||
def test_add_to_list(before, name, item, after):
|
||||
settings = BaseSettings(before, priority=0)
|
||||
settings.add_to_list(name, item)
|
||||
expected_priority = settings.getpriority(name) or 0
|
||||
expected_settings = BaseSettings(after, priority=expected_priority)
|
||||
assert settings == expected_settings, (
|
||||
f"{settings[name]=} != {expected_settings[name]=}"
|
||||
)
|
||||
assert settings.getpriority(name) == expected_settings.getpriority(name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("before", "name", "item", "after"),
|
||||
[
|
||||
({}, "FOO", "BAR", ValueError),
|
||||
({"FOO": ["BAR"]}, "FOO", "BAR", {"FOO": []}),
|
||||
({"FOO": ["BAR"]}, "FOO", "BAZ", ValueError),
|
||||
({"FOO": ["BAR", "BAZ"]}, "FOO", "BAR", {"FOO": ["BAZ"]}),
|
||||
({"FOO": ""}, "FOO", "BAR", ValueError),
|
||||
({"FOO": "[]"}, "FOO", "BAR", ValueError),
|
||||
({"FOO": "BAR"}, "FOO", "BAR", {"FOO": []}),
|
||||
({"FOO": "BAR"}, "FOO", "BAZ", ValueError),
|
||||
({"FOO": "BAR,BAZ"}, "FOO", "BAR", {"FOO": ["BAZ"]}),
|
||||
],
|
||||
)
|
||||
def test_remove_from_list(before, name, item, after):
|
||||
settings = BaseSettings(before, priority=0)
|
||||
|
||||
if isinstance(after, type) and issubclass(after, Exception):
|
||||
with pytest.raises(after):
|
||||
settings.remove_from_list(name, item)
|
||||
return
|
||||
|
||||
settings.remove_from_list(name, item)
|
||||
expected_priority = settings.getpriority(name) or 0
|
||||
expected_settings = BaseSettings(after, priority=expected_priority)
|
||||
assert settings == expected_settings, (
|
||||
f"{settings[name]=} != {expected_settings[name]=}"
|
||||
)
|
||||
assert settings.getpriority(name) == expected_settings.getpriority(name)
|
||||
|
||||
|
||||
class Component1:
|
||||
pass
|
||||
|
||||
|
||||
Component1Alias = Component1
|
||||
|
||||
|
||||
class Component1Subclass(Component1):
|
||||
pass
|
||||
|
||||
|
||||
Component1SubclassAlias = Component1Subclass
|
||||
|
||||
|
||||
class Component2:
|
||||
pass
|
||||
|
||||
|
||||
class Component3:
|
||||
pass
|
||||
|
||||
|
||||
class Component4:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("before", "name", "old_cls", "new_cls", "priority", "after"),
|
||||
[
|
||||
({}, "FOO", Component1, Component2, None, KeyError),
|
||||
(
|
||||
{"FOO": {Component1: 1}},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
None,
|
||||
{"FOO": {Component2: 1}},
|
||||
),
|
||||
(
|
||||
{"FOO": {Component1: 1}},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
2,
|
||||
{"FOO": {Component2: 2}},
|
||||
),
|
||||
(
|
||||
{"FOO": {"tests.test_settings.Component1": 1}},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
None,
|
||||
{"FOO": {Component2: 1}},
|
||||
),
|
||||
(
|
||||
{"FOO": {Component1Alias: 1}},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
None,
|
||||
{"FOO": {Component2: 1}},
|
||||
),
|
||||
(
|
||||
{"FOO": {Component1Alias: 1}},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
2,
|
||||
{"FOO": {Component2: 2}},
|
||||
),
|
||||
(
|
||||
{"FOO": {"tests.test_settings.Component1Alias": 1}},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
None,
|
||||
{"FOO": {Component2: 1}},
|
||||
),
|
||||
(
|
||||
{"FOO": {"tests.test_settings.Component1Alias": 1}},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
2,
|
||||
{"FOO": {Component2: 2}},
|
||||
),
|
||||
(
|
||||
{
|
||||
"FOO": {
|
||||
"tests.test_settings.Component1": 1,
|
||||
"tests.test_settings.Component1Alias": 2,
|
||||
}
|
||||
},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
None,
|
||||
{"FOO": {Component2: 2}},
|
||||
),
|
||||
(
|
||||
{
|
||||
"FOO": {
|
||||
"tests.test_settings.Component1": 1,
|
||||
"tests.test_settings.Component1Alias": 2,
|
||||
}
|
||||
},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
3,
|
||||
{"FOO": {Component2: 3}},
|
||||
),
|
||||
(
|
||||
{"FOO": '{"tests.test_settings.Component1": 1}'},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
None,
|
||||
{"FOO": {Component2: 1}},
|
||||
),
|
||||
(
|
||||
{"FOO": '{"tests.test_settings.Component1": 1}'},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
2,
|
||||
{"FOO": {Component2: 2}},
|
||||
),
|
||||
(
|
||||
{"FOO": '{"tests.test_settings.Component1Alias": 1}'},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
None,
|
||||
{"FOO": {Component2: 1}},
|
||||
),
|
||||
(
|
||||
{"FOO": '{"tests.test_settings.Component1Alias": 1}'},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
2,
|
||||
{"FOO": {Component2: 2}},
|
||||
),
|
||||
(
|
||||
{
|
||||
"FOO": '{"tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": 2}'
|
||||
},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
None,
|
||||
{"FOO": {Component2: 2}},
|
||||
),
|
||||
(
|
||||
{
|
||||
"FOO": '{"tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": 2}'
|
||||
},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
3,
|
||||
{"FOO": {Component2: 3}},
|
||||
),
|
||||
# If old_cls has None as value, raise KeyError.
|
||||
(
|
||||
{"FOO": {Component1: None}},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
None,
|
||||
KeyError,
|
||||
),
|
||||
(
|
||||
{"FOO": '{"tests.test_settings.Component1": null}'},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
None,
|
||||
KeyError,
|
||||
),
|
||||
(
|
||||
{"FOO": {Component1: None, "tests.test_settings.Component1": None}},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
None,
|
||||
KeyError,
|
||||
),
|
||||
(
|
||||
{"FOO": {Component1: 1, "tests.test_settings.Component1": None}},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
None,
|
||||
KeyError,
|
||||
),
|
||||
(
|
||||
{"FOO": {Component1: None, "tests.test_settings.Component1": 1}},
|
||||
"FOO",
|
||||
Component1,
|
||||
Component2,
|
||||
None,
|
||||
KeyError,
|
||||
),
|
||||
# Unrelated components are kept as is, as expected.
|
||||
(
|
||||
{
|
||||
"FOO": {
|
||||
Component1: 1,
|
||||
"tests.test_settings.Component2": 2,
|
||||
Component3: 3,
|
||||
}
|
||||
},
|
||||
"FOO",
|
||||
Component3,
|
||||
Component4,
|
||||
None,
|
||||
{
|
||||
"FOO": {
|
||||
Component1: 1,
|
||||
"tests.test_settings.Component2": 2,
|
||||
Component4: 3,
|
||||
}
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_replace_in_component_priority_dict(
|
||||
before, name, old_cls, new_cls, priority, after
|
||||
):
|
||||
settings = BaseSettings(before, priority=0)
|
||||
|
||||
if isinstance(after, type) and issubclass(after, Exception):
|
||||
with pytest.raises(after):
|
||||
settings.replace_in_component_priority_dict(
|
||||
name, old_cls, new_cls, priority
|
||||
)
|
||||
return
|
||||
|
||||
expected_priority = settings.getpriority(name) or 0
|
||||
settings.replace_in_component_priority_dict(name, old_cls, new_cls, priority)
|
||||
expected_settings = BaseSettings(after, priority=expected_priority)
|
||||
assert settings == expected_settings
|
||||
assert settings.getpriority(name) == expected_settings.getpriority(name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("before", "name", "cls", "priority", "after"),
|
||||
[
|
||||
# Set
|
||||
({}, "FOO", Component1, None, {"FOO": {Component1: None}}),
|
||||
({}, "FOO", Component1, 0, {"FOO": {Component1: 0}}),
|
||||
({}, "FOO", Component1, 1, {"FOO": {Component1: 1}}),
|
||||
# Add
|
||||
(
|
||||
{"FOO": {Component1: 0}},
|
||||
"FOO",
|
||||
Component2,
|
||||
None,
|
||||
{"FOO": {Component1: 0, Component2: None}},
|
||||
),
|
||||
(
|
||||
{"FOO": {Component1: 0}},
|
||||
"FOO",
|
||||
Component2,
|
||||
0,
|
||||
{"FOO": {Component1: 0, Component2: 0}},
|
||||
),
|
||||
(
|
||||
{"FOO": {Component1: 0}},
|
||||
"FOO",
|
||||
Component2,
|
||||
1,
|
||||
{"FOO": {Component1: 0, Component2: 1}},
|
||||
),
|
||||
# Replace
|
||||
(
|
||||
{
|
||||
"FOO": {
|
||||
Component1: None,
|
||||
"tests.test_settings.Component1": 0,
|
||||
"tests.test_settings.Component1Alias": 1,
|
||||
Component1Subclass: None,
|
||||
"tests.test_settings.Component1Subclass": 0,
|
||||
"tests.test_settings.Component1SubclassAlias": 1,
|
||||
}
|
||||
},
|
||||
"FOO",
|
||||
Component1,
|
||||
None,
|
||||
{
|
||||
"FOO": {
|
||||
Component1: None,
|
||||
Component1Subclass: None,
|
||||
"tests.test_settings.Component1Subclass": 0,
|
||||
"tests.test_settings.Component1SubclassAlias": 1,
|
||||
}
|
||||
},
|
||||
),
|
||||
(
|
||||
{
|
||||
"FOO": {
|
||||
Component1: 0,
|
||||
"tests.test_settings.Component1": 1,
|
||||
"tests.test_settings.Component1Alias": None,
|
||||
Component1Subclass: 0,
|
||||
"tests.test_settings.Component1Subclass": 1,
|
||||
"tests.test_settings.Component1SubclassAlias": None,
|
||||
}
|
||||
},
|
||||
"FOO",
|
||||
Component1,
|
||||
0,
|
||||
{
|
||||
"FOO": {
|
||||
Component1: 0,
|
||||
Component1Subclass: 0,
|
||||
"tests.test_settings.Component1Subclass": 1,
|
||||
"tests.test_settings.Component1SubclassAlias": None,
|
||||
}
|
||||
},
|
||||
),
|
||||
(
|
||||
{
|
||||
"FOO": {
|
||||
Component1: 1,
|
||||
"tests.test_settings.Component1": None,
|
||||
"tests.test_settings.Component1Alias": 0,
|
||||
Component1Subclass: 1,
|
||||
"tests.test_settings.Component1Subclass": None,
|
||||
"tests.test_settings.Component1SubclassAlias": 0,
|
||||
}
|
||||
},
|
||||
"FOO",
|
||||
Component1,
|
||||
1,
|
||||
{
|
||||
"FOO": {
|
||||
Component1: 1,
|
||||
Component1Subclass: 1,
|
||||
"tests.test_settings.Component1Subclass": None,
|
||||
"tests.test_settings.Component1SubclassAlias": 0,
|
||||
}
|
||||
},
|
||||
),
|
||||
# String-based setting values
|
||||
(
|
||||
{"FOO": '{"tests.test_settings.Component1": 0}'},
|
||||
"FOO",
|
||||
Component2,
|
||||
None,
|
||||
{"FOO": {"tests.test_settings.Component1": 0, Component2: None}},
|
||||
),
|
||||
(
|
||||
{
|
||||
"FOO": """{
|
||||
"tests.test_settings.Component1": 0,
|
||||
"tests.test_settings.Component1Alias": 1,
|
||||
"tests.test_settings.Component1Subclass": 0,
|
||||
"tests.test_settings.Component1SubclassAlias": 1
|
||||
}"""
|
||||
},
|
||||
"FOO",
|
||||
Component1,
|
||||
None,
|
||||
{
|
||||
"FOO": {
|
||||
Component1: None,
|
||||
"tests.test_settings.Component1Subclass": 0,
|
||||
"tests.test_settings.Component1SubclassAlias": 1,
|
||||
}
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_set_in_component_priority_dict(before, name, cls, priority, after):
|
||||
settings = BaseSettings(before, priority=0)
|
||||
expected_priority = settings.getpriority(name) or 0
|
||||
settings.set_in_component_priority_dict(name, cls, priority)
|
||||
expected_settings = BaseSettings(after, priority=expected_priority)
|
||||
assert settings == expected_settings
|
||||
assert settings.getpriority(name) == expected_settings.getpriority(name), (
|
||||
f"{settings.getpriority(name)=} != {expected_settings.getpriority(name)=}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("before", "name", "cls", "priority", "after"),
|
||||
[
|
||||
# Set
|
||||
({}, "FOO", Component1, None, {"FOO": {Component1: None}}),
|
||||
({}, "FOO", Component1, 0, {"FOO": {Component1: 0}}),
|
||||
({}, "FOO", Component1, 1, {"FOO": {Component1: 1}}),
|
||||
# Add
|
||||
(
|
||||
{"FOO": {Component1: 0}},
|
||||
"FOO",
|
||||
Component2,
|
||||
None,
|
||||
{"FOO": {Component1: 0, Component2: None}},
|
||||
),
|
||||
(
|
||||
{"FOO": {Component1: 0}},
|
||||
"FOO",
|
||||
Component2,
|
||||
0,
|
||||
{"FOO": {Component1: 0, Component2: 0}},
|
||||
),
|
||||
(
|
||||
{"FOO": {Component1: 0}},
|
||||
"FOO",
|
||||
Component2,
|
||||
1,
|
||||
{"FOO": {Component1: 0, Component2: 1}},
|
||||
),
|
||||
# Keep
|
||||
(
|
||||
{
|
||||
"FOO": {
|
||||
Component1: None,
|
||||
"tests.test_settings.Component1": 0,
|
||||
"tests.test_settings.Component1Alias": 1,
|
||||
Component1Subclass: None,
|
||||
"tests.test_settings.Component1Subclass": 0,
|
||||
"tests.test_settings.Component1SubclassAlias": 1,
|
||||
}
|
||||
},
|
||||
"FOO",
|
||||
Component1,
|
||||
None,
|
||||
{
|
||||
"FOO": {
|
||||
Component1: None,
|
||||
"tests.test_settings.Component1": 0,
|
||||
"tests.test_settings.Component1Alias": 1,
|
||||
Component1Subclass: None,
|
||||
"tests.test_settings.Component1Subclass": 0,
|
||||
"tests.test_settings.Component1SubclassAlias": 1,
|
||||
}
|
||||
},
|
||||
),
|
||||
(
|
||||
{
|
||||
"FOO": {
|
||||
Component1: 0,
|
||||
"tests.test_settings.Component1": 1,
|
||||
"tests.test_settings.Component1Alias": None,
|
||||
Component1Subclass: 0,
|
||||
"tests.test_settings.Component1Subclass": 1,
|
||||
"tests.test_settings.Component1SubclassAlias": None,
|
||||
}
|
||||
},
|
||||
"FOO",
|
||||
Component1,
|
||||
0,
|
||||
{
|
||||
"FOO": {
|
||||
Component1: 0,
|
||||
"tests.test_settings.Component1": 1,
|
||||
"tests.test_settings.Component1Alias": None,
|
||||
Component1Subclass: 0,
|
||||
"tests.test_settings.Component1Subclass": 1,
|
||||
"tests.test_settings.Component1SubclassAlias": None,
|
||||
}
|
||||
},
|
||||
),
|
||||
(
|
||||
{
|
||||
"FOO": {
|
||||
Component1: 1,
|
||||
"tests.test_settings.Component1": None,
|
||||
"tests.test_settings.Component1Alias": 0,
|
||||
Component1Subclass: 1,
|
||||
"tests.test_settings.Component1Subclass": None,
|
||||
"tests.test_settings.Component1SubclassAlias": 0,
|
||||
}
|
||||
},
|
||||
"FOO",
|
||||
Component1,
|
||||
1,
|
||||
{
|
||||
"FOO": {
|
||||
Component1: 1,
|
||||
"tests.test_settings.Component1": None,
|
||||
"tests.test_settings.Component1Alias": 0,
|
||||
Component1Subclass: 1,
|
||||
"tests.test_settings.Component1Subclass": None,
|
||||
"tests.test_settings.Component1SubclassAlias": 0,
|
||||
}
|
||||
},
|
||||
),
|
||||
# String-based setting values
|
||||
(
|
||||
{"FOO": '{"tests.test_settings.Component1": 0}'},
|
||||
"FOO",
|
||||
Component2,
|
||||
None,
|
||||
{"FOO": {"tests.test_settings.Component1": 0, Component2: None}},
|
||||
),
|
||||
(
|
||||
{
|
||||
"FOO": """{
|
||||
"tests.test_settings.Component1": 0,
|
||||
"tests.test_settings.Component1Alias": 1,
|
||||
"tests.test_settings.Component1Subclass": 0,
|
||||
"tests.test_settings.Component1SubclassAlias": 1
|
||||
}"""
|
||||
},
|
||||
"FOO",
|
||||
Component1,
|
||||
None,
|
||||
{
|
||||
"FOO": """{
|
||||
"tests.test_settings.Component1": 0,
|
||||
"tests.test_settings.Component1Alias": 1,
|
||||
"tests.test_settings.Component1Subclass": 0,
|
||||
"tests.test_settings.Component1SubclassAlias": 1
|
||||
}"""
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_setdefault_in_component_priority_dict(before, name, cls, priority, after):
|
||||
settings = BaseSettings(before, priority=0)
|
||||
expected_priority = settings.getpriority(name) or 0
|
||||
settings.setdefault_in_component_priority_dict(name, cls, priority)
|
||||
expected_settings = BaseSettings(after, priority=expected_priority)
|
||||
assert settings == expected_settings
|
||||
assert settings.getpriority(name) == expected_settings.getpriority(name)
|
||||
|
|
|
|||
2
tox.ini
2
tox.ini
|
|
@ -39,7 +39,7 @@ passenv =
|
|||
#allow tox virtualenv to upgrade pip/wheel/setuptools
|
||||
download = true
|
||||
commands =
|
||||
pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} --doctest-modules
|
||||
pytest --cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml {posargs:--durations=10 docs scrapy tests} --doctest-modules
|
||||
install_command =
|
||||
python -I -m pip install -ctests/upper-constraints.txt {opts} {packages}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue