# pylint: disable=unsubscriptable-object,unsupported-membership-test # (too many false positives) import logging from unittest import mock import pytest from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import ( SETTINGS_PRIORITIES, BaseSettings, Settings, SettingsAttribute, get_settings_priority, ) from scrapy.settings import default_settings as scrapy_default_settings from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.test import get_crawler from . import default_settings NON_COMPONENT_PRIORITY_DICT_BASE_SETTING_NAMES = { "DOWNLOAD_HANDLERS_BASE", "FEED_EXPORTERS_BASE", "FEED_STORAGES_BASE", } class TestSettingsGlobalFuncs: def test_get_settings_priority(self): for prio_str, prio_num in SETTINGS_PRIORITIES.items(): assert get_settings_priority(prio_str) == prio_num assert get_settings_priority(99) == 99 class TestSettingsAttribute: def setup_method(self): self.attribute = SettingsAttribute("value", 10) def test_set_greater_priority(self): self.attribute.set("value2", 20) assert self.attribute.value == "value2" assert self.attribute.priority == 20 def test_set_equal_priority(self): self.attribute.set("value2", 10) assert self.attribute.value == "value2" assert self.attribute.priority == 10 def test_set_less_priority(self): self.attribute.set("value2", 0) assert self.attribute.value == "value" assert self.attribute.priority == 10 def test_overwrite_basesettings(self): original_dict = {"one": 10, "two": 20} original_settings = BaseSettings(original_dict, 0) attribute = SettingsAttribute(original_settings, 0) new_dict = {"three": 11, "four": 21} attribute.set(new_dict, 10) assert isinstance(attribute.value, BaseSettings) assert set(attribute.value) == set(new_dict) assert set(original_settings) == set(original_dict) new_settings = BaseSettings({"five": 12}, 0) attribute.set(new_settings, 0) # Insufficient priority assert set(attribute.value) == set(new_dict) attribute.set(new_settings, 10) assert set(attribute.value) == set(new_settings) def test_repr(self): assert repr(self.attribute) == "" class TestBaseSettings: def setup_method(self): self.settings = BaseSettings() def test_setdefault_not_existing_value(self): settings = BaseSettings() value = settings.setdefault("TEST_OPTION", "value") assert settings["TEST_OPTION"] == "value" assert value == "value" assert value is not None def test_setdefault_existing_value(self): settings = BaseSettings({"TEST_OPTION": "value"}) value = settings.setdefault("TEST_OPTION", None) assert settings["TEST_OPTION"] == "value" assert value == "value" def test_set_new_attribute(self): self.settings.set("TEST_OPTION", "value", 0) assert "TEST_OPTION" in self.settings.attributes attr = self.settings.attributes["TEST_OPTION"] assert isinstance(attr, SettingsAttribute) assert attr.value == "value" assert attr.priority == 0 def test_set_settingsattribute(self): myattr = SettingsAttribute(0, 30) # Note priority 30 self.settings.set("TEST_ATTR", myattr, 10) assert self.settings.get("TEST_ATTR") == 0 assert self.settings.getpriority("TEST_ATTR") == 30 def test_set_instance_identity_on_update(self): attr = SettingsAttribute("value", 0) self.settings.attributes = {"TEST_OPTION": attr} self.settings.set("TEST_OPTION", "othervalue", 10) assert "TEST_OPTION" in self.settings.attributes assert attr is self.settings.attributes["TEST_OPTION"] def test_set_calls_settings_attributes_methods_on_update(self): attr = SettingsAttribute("value", 10) with ( mock.patch.object(attr, "__setattr__") as mock_setattr, mock.patch.object(attr, "set") as mock_set, ): self.settings.attributes = {"TEST_OPTION": attr} for priority in (0, 10, 20): self.settings.set("TEST_OPTION", "othervalue", priority) mock_set.assert_called_once_with("othervalue", priority) assert not mock_setattr.called mock_set.reset_mock() mock_setattr.reset_mock() def test_setitem(self): settings = BaseSettings() settings.set("key", "a", "default") settings["key"] = "b" assert settings["key"] == "b" assert settings.getpriority("key") == 20 settings["key"] = "c" assert settings["key"] == "c" settings["key2"] = "x" assert "key2" in settings assert settings["key2"] == "x" assert settings.getpriority("key2") == 20 def test_setdict_alias(self): with mock.patch.object(self.settings, "set") as mock_set: self.settings.setdict({"TEST_1": "value1", "TEST_2": "value2"}, 10) assert mock_set.call_count == 2 calls = [ mock.call("TEST_1", "value1", 10), mock.call("TEST_2", "value2", 10), ] mock_set.assert_has_calls(calls, any_order=True) def test_setmodule_only_load_uppercase_vars(self): class ModuleMock: UPPERCASE_VAR = "value" MIXEDcase_VAR = "othervalue" lowercase_var = "anothervalue" self.settings.attributes = {} self.settings.setmodule(ModuleMock(), 10) assert "UPPERCASE_VAR" in self.settings.attributes assert "MIXEDcase_VAR" not in self.settings.attributes assert "lowercase_var" not in self.settings.attributes assert len(self.settings.attributes) == 1 def test_setmodule_alias(self): with mock.patch.object(self.settings, "set") as mock_set: self.settings.setmodule(default_settings, 10) mock_set.assert_any_call("TEST_DEFAULT", "defvalue", 10) mock_set.assert_any_call("TEST_DICT", {"key": "val"}, 10) def test_setmodule_by_path(self): self.settings.attributes = {} self.settings.setmodule(default_settings, 10) ctrl_attributes = self.settings.attributes.copy() self.settings.attributes = {} self.settings.setmodule("tests.test_settings.default_settings", 10) assert set(self.settings.attributes) == set(ctrl_attributes) for key in ctrl_attributes: attr = self.settings.attributes[key] ctrl_attr = ctrl_attributes[key] assert attr.value == ctrl_attr.value assert attr.priority == ctrl_attr.priority def test_update(self): settings = BaseSettings({"key_lowprio": 0}, priority=0) settings.set("key_highprio", 10, priority=50) custom_settings = BaseSettings( {"key_lowprio": 1, "key_highprio": 11}, priority=30 ) custom_settings.set("newkey_one", None, priority=50) custom_dict = {"key_lowprio": 2, "key_highprio": 12, "newkey_two": None} settings.update(custom_dict, priority=20) assert settings["key_lowprio"] == 2 assert settings.getpriority("key_lowprio") == 20 assert settings["key_highprio"] == 10 assert "newkey_two" in settings assert settings.getpriority("newkey_two") == 20 settings.update(custom_settings) assert settings["key_lowprio"] == 1 assert settings.getpriority("key_lowprio") == 30 assert settings["key_highprio"] == 10 assert "newkey_one" in settings assert settings.getpriority("newkey_one") == 50 settings.update({"key_lowprio": 3}, priority=20) assert settings["key_lowprio"] == 1 @pytest.mark.xfail( raises=TypeError, reason="BaseSettings.update doesn't support kwargs input" ) def test_update_kwargs(self): settings = BaseSettings({"key": 0}) settings.update(key=1) # pylint: disable=unexpected-keyword-arg @pytest.mark.xfail( raises=AttributeError, reason="BaseSettings.update doesn't support iterable input", ) def test_update_iterable(self): settings = BaseSettings({"key": 0}) settings.update([("key", 1)]) def test_update_jsonstring(self): settings = BaseSettings({"number": 0, "dict": BaseSettings({"key": "val"})}) settings.update('{"number": 1, "newnumber": 2}') assert settings["number"] == 1 assert settings["newnumber"] == 2 settings.set("dict", '{"key": "newval", "newkey": "newval2"}') assert settings["dict"]["key"] == "newval" assert settings["dict"]["newkey"] == "newval2" def test_delete(self): settings = BaseSettings({"key": None}) settings.set("key_highprio", None, priority=50) settings.delete("key") settings.delete("key_highprio") assert "key" not in settings assert "key_highprio" in settings del settings["key_highprio"] assert "key_highprio" not in settings with pytest.raises(KeyError): settings.delete("notkey") with pytest.raises(KeyError): del settings["notkey"] def test_get(self): test_configuration = { "TEST_ENABLED1": "1", "TEST_ENABLED2": True, "TEST_ENABLED3": 1, "TEST_ENABLED4": "True", "TEST_ENABLED5": "true", "TEST_ENABLED_WRONG": "on", "TEST_DISABLED1": "0", "TEST_DISABLED2": False, "TEST_DISABLED3": 0, "TEST_DISABLED4": "False", "TEST_DISABLED5": "false", "TEST_DISABLED_WRONG": "off", "TEST_INT1": 123, "TEST_INT2": "123", "TEST_FLOAT1": 123.45, "TEST_FLOAT2": "123.45", "TEST_LIST1": ["one", "two"], "TEST_LIST2": "one,two", "TEST_LIST3": "", "TEST_STR": "value", "TEST_DICT1": {"key1": "val1", "ke2": 3}, "TEST_DICT2": '{"key1": "val1", "ke2": 3}', } settings = self.settings settings.attributes = { key: SettingsAttribute(value, 0) for key, value in test_configuration.items() } assert settings.getbool("TEST_ENABLED1") assert settings.getbool("TEST_ENABLED2") assert settings.getbool("TEST_ENABLED3") assert settings.getbool("TEST_ENABLED4") assert settings.getbool("TEST_ENABLED5") assert not settings.getbool("TEST_ENABLEDx") assert settings.getbool("TEST_ENABLEDx", True) assert not settings.getbool("TEST_DISABLED1") assert not settings.getbool("TEST_DISABLED2") assert not settings.getbool("TEST_DISABLED3") assert not settings.getbool("TEST_DISABLED4") assert not settings.getbool("TEST_DISABLED5") assert settings.getint("TEST_INT1") == 123 assert settings.getint("TEST_INT2") == 123 assert settings.getint("TEST_INTx") == 0 assert settings.getint("TEST_INTx", 45) == 45 assert settings.getfloat("TEST_FLOAT1") == 123.45 assert settings.getfloat("TEST_FLOAT2") == 123.45 assert settings.getfloat("TEST_FLOATx") == 0.0 assert settings.getfloat("TEST_FLOATx", 55.0) == 55.0 assert settings.getlist("TEST_LIST1") == ["one", "two"] assert settings.getlist("TEST_LIST2") == ["one", "two"] assert settings.getlist("TEST_LIST3") == [] assert settings.getlist("TEST_LISTx") == [] assert settings.getlist("TEST_LISTx", ["default"]) == ["default"] assert settings["TEST_STR"] == "value" assert settings.get("TEST_STR") == "value" assert settings["TEST_STRx"] is None assert settings.get("TEST_STRx") is None assert settings.get("TEST_STRx", "default") == "default" assert settings.getdict("TEST_DICT1") == {"key1": "val1", "ke2": 3} assert settings.getdict("TEST_DICT2") == {"key1": "val1", "ke2": 3} assert settings.getdict("TEST_DICT3") == {} assert settings.getdict("TEST_DICT3", {"key1": 5}) == {"key1": 5} with pytest.raises( ValueError, match=r"dictionary update sequence element #0 has length 3; 2 is required|sequence of pairs expected", ): settings.getdict("TEST_LIST1") with pytest.raises( ValueError, match="Supported values for boolean settings are" ): settings.getbool("TEST_ENABLED_WRONG") with pytest.raises( ValueError, match="Supported values for boolean settings are" ): settings.getbool("TEST_DISABLED_WRONG") def test_getpriority(self): settings = BaseSettings({"key": "value"}, priority=99) assert settings.getpriority("key") == 99 assert settings.getpriority("nonexistentkey") is None def test_getwithbase(self): s = BaseSettings( { "TEST_BASE": BaseSettings({1: 1, 2: 2}, "project"), "TEST": BaseSettings({1: 10, 3: 30}, "default"), "HASNOBASE": BaseSettings({3: 3000}, "default"), } ) s["TEST"].set(2, 200, "cmdline") assert set(s.getwithbase("TEST")) == {1, 2, 3} assert set(s.getwithbase("HASNOBASE")) == set(s["HASNOBASE"]) assert s.getwithbase("NONEXISTENT") == {} def test_maxpriority(self): # Empty settings should return 'default' assert self.settings.maxpriority() == 0 self.settings.set("A", 0, 10) self.settings.set("B", 0, 30) assert self.settings.maxpriority() == 30 def test_copy(self): values = { "TEST_BOOL": True, "TEST_LIST": ["one", "two"], "TEST_LIST_OF_LISTS": [ ["first_one", "first_two"], ["second_one", "second_two"], ], } self.settings.setdict(values) copy = self.settings.copy() self.settings.set("TEST_BOOL", False) assert copy.get("TEST_BOOL") test_list = self.settings.get("TEST_LIST") test_list.append("three") assert copy.get("TEST_LIST") == ["one", "two"] test_list_of_lists = self.settings.get("TEST_LIST_OF_LISTS") test_list_of_lists[0].append("first_three") assert copy.get("TEST_LIST_OF_LISTS")[0] == ["first_one", "first_two"] def test_copy_to_dict(self): s = BaseSettings( { "TEST_STRING": "a string", "TEST_LIST": [1, 2], "TEST_BOOLEAN": False, "TEST_BASE": BaseSettings({"foo": 1, "bar": 2}, "project"), "TEST": BaseSettings({"foo": 10, "baz": 30}, "default"), "HASNOBASE": BaseSettings({"baz": 3000}, "default"), } ) assert s.copy_to_dict() == { "HASNOBASE": {"baz": 3000}, "TEST": {"foo": 10, "baz": 30}, "TEST_BASE": {"foo": 1, "bar": 2}, "TEST_LIST": [1, 2], "TEST_BOOLEAN": False, "TEST_STRING": "a string", } def test_freeze(self): self.settings.freeze() with pytest.raises( TypeError, match="Trying to modify an immutable Settings object" ): self.settings.set("TEST_BOOL", False) def test_frozencopy(self): frozencopy = self.settings.frozencopy() assert frozencopy.frozen assert frozencopy is not self.settings def test_getwithbase_for_dotted_keys(self): settings = BaseSettings( { "FEED_EXPORTERS_BASE": BaseSettings({"json": "foo"}), "FEED_EXPORTERS": BaseSettings({"csv.gz": "bar"}), } ) value = settings.getwithbase("FEED_EXPORTERS") assert isinstance(value, BaseSettings) assert dict(value) == { "json": "foo", "csv.gz": "bar", } @pytest.mark.parametrize( ("key", "exception"), [ pytest.param(1, TypeError, id="type-error"), pytest.param("foo", ValueError, id="value-error"), pytest.param("csv.gz", NameError, id="name-error"), ], ) def test_get_component_priority_dict_with_base_handles_load_object_exceptions( self, key, exception ): with pytest.raises(exception): load_object(key) settings = BaseSettings( { "FOO": BaseSettings({key: 1}), } ) value = settings.get_component_priority_dict_with_base("FOO") assert isinstance(value, BaseSettings) assert dict(value) == {key: 1} def test_get_component_priority_dict_with_base_override_none_by_type(self): settings = BaseSettings() setting_names = set() for k, v in scrapy_default_settings.__dict__.items(): if ( not k.endswith("_BASE") or k in NON_COMPONENT_PRIORITY_DICT_BASE_SETTING_NAMES ): continue settings[k] = v setting_name = k[: -len("_BASE")] setting_names.add(setting_name) settings[setting_name] = { load_object(import_path): None for import_path in v } for setting_name in setting_names: value = settings.get_component_priority_dict_with_base(setting_name) assert not dict(value) def test_get_component_priority_dict_with_base_override_value_by_type(self): settings = BaseSettings() setting_names = set() value = 0 for k, v in scrapy_default_settings.__dict__.items(): if ( not k.endswith("_BASE") or k in NON_COMPONENT_PRIORITY_DICT_BASE_SETTING_NAMES ): continue settings[k] = v setting_name = k[: -len("_BASE")] setting_names.add(setting_name) settings[setting_name] = { load_object(import_path): value for import_path in v } for setting_name in setting_names: assert ( settings.get_component_priority_dict_with_base(setting_name) == settings[setting_name] ) def test_getwithbase_for_non_component_priority_dicts(self): settings = BaseSettings() for base_name in NON_COMPONENT_PRIORITY_DICT_BASE_SETTING_NAMES: base_value = getattr(scrapy_default_settings, base_name) settings[base_name] = BaseSettings(base_value) assert len(base_value) >= 2 keys = list(base_value) values = list(base_value.values()) override = {keys[0]: values[1]} expected = dict(base_value) expected[keys[0]] = values[1] name = base_name[: -len("_BASE")] settings[name] = BaseSettings(override) value = settings.getwithbase(name) assert isinstance(value, BaseSettings) assert dict(value) == expected def test_get_component_priority_dict_with_base_warns_on_duplicate_import_paths( self, caplog ): settings = BaseSettings() settings["FOO"] = BaseSettings( { "scrapy.Request": 1, "scrapy.http.Request": 2, } ) with caplog.at_level(logging.WARNING): value = settings.get_component_priority_dict_with_base("FOO") assert isinstance(value, BaseSettings) assert dict(value) == {"scrapy.http.Request": 2} assert caplog.records, "Expected a warning to be logged" msg = caplog.records[0].message assert "scrapy.http.request.Request" in msg def test_get_component_priority_dict_with_base_warns_on_duplicate_mixed_type_and_path( self, caplog ): settings = BaseSettings() settings["FOO"] = BaseSettings( {Component1: 1, "tests.test_settings.Component1": 2} ) with caplog.at_level(logging.WARNING): value = settings.get_component_priority_dict_with_base("FOO") assert isinstance(value, BaseSettings) assert dict(value) == {"tests.test_settings.Component1": 2} assert caplog.records, "Expected a warning to be logged" msg = caplog.records[0].message assert "tests.test_settings.Component1" in msg def test_getdictorlist(self): settings = BaseSettings() # No value and no default → {} assert settings.getdictorlist("MISSING") == {} # String: valid JSON dict settings.set("S_DICT_STR", '{"key": "val"}') assert settings.getdictorlist("S_DICT_STR") == {"key": "val"} # String: valid JSON list settings.set("S_LIST_STR", '["a", "b"]') assert settings.getdictorlist("S_LIST_STR") == ["a", "b"] # String: invalid JSON → comma-split fallback settings.set("S_CSV", "a,b,c") assert settings.getdictorlist("S_CSV") == ["a", "b", "c"] # String: valid JSON but not dict or list → ValueError caught → comma-split settings.set("S_JSON_NUMBER", "123") assert settings.getdictorlist("S_JSON_NUMBER") == ["123"] # Tuple → list settings.set("S_TUPLE", ("x", "y")) assert settings.getdictorlist("S_TUPLE") == ["x", "y"] # Unsupported type → raises ValueError settings.set("S_INT", 42) with pytest.raises(ValueError, match="must be a dict, list, tuple, or string"): settings.getdictorlist("S_INT") # Dict value → deepcopy returned settings.set("S_DICT", {"key": "val"}) assert settings.getdictorlist("S_DICT") == {"key": "val"} # List value → deepcopy returned settings.set("S_LIST", ["a", "b"]) assert settings.getdictorlist("S_LIST") == ["a", "b"] def test_repr_pretty_(self): settings = BaseSettings({"key": "value"}) mock_p = mock.Mock() settings._repr_pretty_(mock_p, cycle=False) assert mock_p.text.call_count == 1 mock_p.reset_mock() settings._repr_pretty_(mock_p, cycle=True) mock_p.text.assert_called_once_with(repr(settings)) def test_getwithbase_invalid_setting_name(self): settings = BaseSettings() with pytest.raises( ValueError, match="Base setting key must be a string, got 123" ): settings.getwithbase(123) def test_get_component_priority_dict_with_base_invalid_setting_name(self): settings = BaseSettings() with pytest.raises( ValueError, match="Base setting key must be a string, got 123" ): settings.get_component_priority_dict_with_base(123) class TestSettings: def setup_method(self): self.settings = Settings() @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {"default": 10}) @mock.patch("scrapy.settings.default_settings", default_settings) def test_initial_defaults(self): settings = Settings() assert len(settings.attributes) == 2 assert "TEST_DEFAULT" in settings.attributes attr = settings.attributes["TEST_DEFAULT"] assert isinstance(attr, SettingsAttribute) assert attr.value == "defvalue" assert attr.priority == 10 @mock.patch.dict("scrapy.settings.SETTINGS_PRIORITIES", {}) @mock.patch("scrapy.settings.default_settings", {}) def test_initial_values(self): settings = Settings({"TEST_OPTION": "value"}, 10) assert len(settings.attributes) == 1 assert "TEST_OPTION" in settings.attributes attr = settings.attributes["TEST_OPTION"] assert isinstance(attr, SettingsAttribute) assert attr.value == "value" assert attr.priority == 10 @mock.patch("scrapy.settings.default_settings", default_settings) def test_autopromote_dicts(self): settings = Settings() mydict = settings.get("TEST_DICT") assert isinstance(mydict, BaseSettings) assert "key" in mydict assert mydict["key"] == "val" assert mydict.getpriority("key") == 0 @mock.patch("scrapy.settings.default_settings", default_settings) def test_getdict_autodegrade_basesettings(self): settings = Settings() mydict = settings.getdict("TEST_DICT") assert isinstance(mydict, dict) assert len(mydict) == 1 assert "key" in mydict assert mydict["key"] == "val" def test_passing_objects_as_values(self): class TestPipeline: def process_item(self, i): return i settings = Settings( { "ITEM_PIPELINES": { TestPipeline: 800, }, "DOWNLOAD_HANDLERS": { "ftp": FileDownloadHandler, }, } ) assert "ITEM_PIPELINES" in settings.attributes mypipeline, priority = settings.getdict("ITEM_PIPELINES").popitem() assert priority == 800 assert mypipeline == TestPipeline assert isinstance(mypipeline(), TestPipeline) assert mypipeline().process_item("item") == "item" myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp") assert myhandler == FileDownloadHandler myhandler_instance = build_from_crawler(myhandler, get_crawler()) assert isinstance(myhandler_instance, FileDownloadHandler) assert hasattr(myhandler_instance, "download_request") def test_pop_item_with_default_value(self): settings = Settings() with pytest.raises(KeyError): settings.pop("DUMMY_CONFIG") dummy_config_value = settings.pop("DUMMY_CONFIG", "dummy_value") assert dummy_config_value == "dummy_value" def test_pop_item_with_immutable_settings(self): settings = Settings( {"DUMMY_CONFIG": "dummy_value", "OTHER_DUMMY_CONFIG": "other_dummy_value"} ) assert settings.pop("DUMMY_CONFIG") == "dummy_value" settings.freeze() with pytest.raises( TypeError, match="Trying to modify an immutable Settings object" ): settings.pop("OTHER_DUMMY_CONFIG") @pytest.mark.parametrize( ("before", "name", "item", "after"), [ ({}, "FOO", "BAR", {"FOO": ["BAR"]}), ({"FOO": []}, "FOO", "BAR", {"FOO": ["BAR"]}), ({"FOO": ["BAR"]}, "FOO", "BAZ", {"FOO": ["BAR", "BAZ"]}), ({"FOO": ["BAR"]}, "FOO", "BAR", {"FOO": ["BAR"]}), ({"FOO": ""}, "FOO", "BAR", {"FOO": ["BAR"]}), ({"FOO": "BAR"}, "FOO", "BAR", {"FOO": "BAR"}), ({"FOO": "BAR"}, "FOO", "BAZ", {"FOO": ["BAR", "BAZ"]}), ({"FOO": "BAR,BAZ"}, "FOO", "BAZ", {"FOO": "BAR,BAZ"}), ({"FOO": "BAR,BAZ"}, "FOO", "QUX", {"FOO": ["BAR", "BAZ", "QUX"]}), ], ) def test_add_to_list(before, name, item, after): settings = BaseSettings(before, priority=0) settings.add_to_list(name, item) expected_priority = settings.getpriority(name) or 0 expected_settings = BaseSettings(after, priority=expected_priority) assert settings == expected_settings, ( f"{settings[name]=} != {expected_settings[name]=}" ) assert settings.getpriority(name) == expected_settings.getpriority(name) @pytest.mark.parametrize( ("before", "name", "item", "after"), [ ({}, "FOO", "BAR", ValueError), ({"FOO": ["BAR"]}, "FOO", "BAR", {"FOO": []}), ({"FOO": ["BAR"]}, "FOO", "BAZ", ValueError), ({"FOO": ["BAR", "BAZ"]}, "FOO", "BAR", {"FOO": ["BAZ"]}), ({"FOO": ""}, "FOO", "BAR", ValueError), ({"FOO": "[]"}, "FOO", "BAR", ValueError), ({"FOO": "BAR"}, "FOO", "BAR", {"FOO": []}), ({"FOO": "BAR"}, "FOO", "BAZ", ValueError), ({"FOO": "BAR,BAZ"}, "FOO", "BAR", {"FOO": ["BAZ"]}), ], ) def test_remove_from_list(before, name, item, after): settings = BaseSettings(before, priority=0) if isinstance(after, type) and issubclass(after, Exception): with pytest.raises(after): settings.remove_from_list(name, item) return settings.remove_from_list(name, item) expected_priority = settings.getpriority(name) or 0 expected_settings = BaseSettings(after, priority=expected_priority) assert settings == expected_settings, ( f"{settings[name]=} != {expected_settings[name]=}" ) assert settings.getpriority(name) == expected_settings.getpriority(name) def test_deprecated_dns_resolver_setting(): settings = Settings() with pytest.warns( ScrapyDeprecationWarning, match="The DNS_RESOLVER setting is deprecated", ): settings.get("DNS_RESOLVER") def test_deprecated_concurrent_requests_per_ip_setting(): settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1}) with pytest.warns( ScrapyDeprecationWarning, match="The CONCURRENT_REQUESTS_PER_IP setting is deprecated", ): settings.get("CONCURRENT_REQUESTS_PER_IP") class Component1: pass Component1Alias = Component1 class Component1Subclass(Component1): pass Component1SubclassAlias = Component1Subclass class Component2: pass class Component3: pass class Component4: pass @pytest.mark.parametrize( ("before", "name", "old_cls", "new_cls", "priority", "after"), [ ({}, "FOO", Component1, Component2, None, KeyError), ( {"FOO": {Component1: 1}}, "FOO", Component1, Component2, None, {"FOO": {Component2: 1}}, ), ( {"FOO": {Component1: 1}}, "FOO", Component1, Component2, 2, {"FOO": {Component2: 2}}, ), ( {"FOO": {"tests.test_settings.Component1": 1}}, "FOO", Component1, Component2, None, {"FOO": {Component2: 1}}, ), ( {"FOO": {Component1Alias: 1}}, "FOO", Component1, Component2, None, {"FOO": {Component2: 1}}, ), ( {"FOO": {Component1Alias: 1}}, "FOO", Component1, Component2, 2, {"FOO": {Component2: 2}}, ), ( {"FOO": {"tests.test_settings.Component1Alias": 1}}, "FOO", Component1, Component2, None, {"FOO": {Component2: 1}}, ), ( {"FOO": {"tests.test_settings.Component1Alias": 1}}, "FOO", Component1, Component2, 2, {"FOO": {Component2: 2}}, ), ( { "FOO": { "tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": 2, } }, "FOO", Component1, Component2, None, {"FOO": {Component2: 2}}, ), ( { "FOO": { "tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": 2, } }, "FOO", Component1, Component2, 3, {"FOO": {Component2: 3}}, ), ( {"FOO": '{"tests.test_settings.Component1": 1}'}, "FOO", Component1, Component2, None, {"FOO": {Component2: 1}}, ), ( {"FOO": '{"tests.test_settings.Component1": 1}'}, "FOO", Component1, Component2, 2, {"FOO": {Component2: 2}}, ), ( {"FOO": '{"tests.test_settings.Component1Alias": 1}'}, "FOO", Component1, Component2, None, {"FOO": {Component2: 1}}, ), ( {"FOO": '{"tests.test_settings.Component1Alias": 1}'}, "FOO", Component1, Component2, 2, {"FOO": {Component2: 2}}, ), ( { "FOO": '{"tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": 2}' }, "FOO", Component1, Component2, None, {"FOO": {Component2: 2}}, ), ( { "FOO": '{"tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": 2}' }, "FOO", Component1, Component2, 3, {"FOO": {Component2: 3}}, ), # If old_cls has None as value, raise KeyError. ( {"FOO": {Component1: None}}, "FOO", Component1, Component2, None, KeyError, ), ( {"FOO": '{"tests.test_settings.Component1": null}'}, "FOO", Component1, Component2, None, KeyError, ), ( {"FOO": {Component1: None, "tests.test_settings.Component1": None}}, "FOO", Component1, Component2, None, KeyError, ), ( {"FOO": {Component1: 1, "tests.test_settings.Component1": None}}, "FOO", Component1, Component2, None, KeyError, ), ( {"FOO": {Component1: None, "tests.test_settings.Component1": 1}}, "FOO", Component1, Component2, None, KeyError, ), # Unrelated components are kept as is, as expected. ( { "FOO": { Component1: 1, "tests.test_settings.Component2": 2, Component3: 3, } }, "FOO", Component3, Component4, None, { "FOO": { Component1: 1, "tests.test_settings.Component2": 2, Component4: 3, } }, ), ], ) def test_replace_in_component_priority_dict( before, name, old_cls, new_cls, priority, after ): settings = BaseSettings(before, priority=0) if isinstance(after, type) and issubclass(after, Exception): with pytest.raises(after): settings.replace_in_component_priority_dict( name, old_cls, new_cls, priority ) return expected_priority = settings.getpriority(name) or 0 settings.replace_in_component_priority_dict(name, old_cls, new_cls, priority) expected_settings = BaseSettings(after, priority=expected_priority) assert settings == expected_settings assert settings.getpriority(name) == expected_settings.getpriority(name) @pytest.mark.parametrize( ("before", "name", "cls", "priority", "after"), [ # Set ({}, "FOO", Component1, None, {"FOO": {Component1: None}}), ({}, "FOO", Component1, 0, {"FOO": {Component1: 0}}), ({}, "FOO", Component1, 1, {"FOO": {Component1: 1}}), # Add ( {"FOO": {Component1: 0}}, "FOO", Component2, None, {"FOO": {Component1: 0, Component2: None}}, ), ( {"FOO": {Component1: 0}}, "FOO", Component2, 0, {"FOO": {Component1: 0, Component2: 0}}, ), ( {"FOO": {Component1: 0}}, "FOO", Component2, 1, {"FOO": {Component1: 0, Component2: 1}}, ), # Replace ( { "FOO": { Component1: None, "tests.test_settings.Component1": 0, "tests.test_settings.Component1Alias": 1, Component1Subclass: None, "tests.test_settings.Component1Subclass": 0, "tests.test_settings.Component1SubclassAlias": 1, } }, "FOO", Component1, None, { "FOO": { Component1: None, Component1Subclass: None, "tests.test_settings.Component1Subclass": 0, "tests.test_settings.Component1SubclassAlias": 1, } }, ), ( { "FOO": { Component1: 0, "tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": None, Component1Subclass: 0, "tests.test_settings.Component1Subclass": 1, "tests.test_settings.Component1SubclassAlias": None, } }, "FOO", Component1, 0, { "FOO": { Component1: 0, Component1Subclass: 0, "tests.test_settings.Component1Subclass": 1, "tests.test_settings.Component1SubclassAlias": None, } }, ), ( { "FOO": { Component1: 1, "tests.test_settings.Component1": None, "tests.test_settings.Component1Alias": 0, Component1Subclass: 1, "tests.test_settings.Component1Subclass": None, "tests.test_settings.Component1SubclassAlias": 0, } }, "FOO", Component1, 1, { "FOO": { Component1: 1, Component1Subclass: 1, "tests.test_settings.Component1Subclass": None, "tests.test_settings.Component1SubclassAlias": 0, } }, ), # String-based setting values ( {"FOO": '{"tests.test_settings.Component1": 0}'}, "FOO", Component2, None, {"FOO": {"tests.test_settings.Component1": 0, Component2: None}}, ), ( { "FOO": """{ "tests.test_settings.Component1": 0, "tests.test_settings.Component1Alias": 1, "tests.test_settings.Component1Subclass": 0, "tests.test_settings.Component1SubclassAlias": 1 }""" }, "FOO", Component1, None, { "FOO": { Component1: None, "tests.test_settings.Component1Subclass": 0, "tests.test_settings.Component1SubclassAlias": 1, } }, ), ], ) def test_set_in_component_priority_dict(before, name, cls, priority, after): settings = BaseSettings(before, priority=0) expected_priority = settings.getpriority(name) or 0 settings.set_in_component_priority_dict(name, cls, priority) expected_settings = BaseSettings(after, priority=expected_priority) assert settings == expected_settings assert settings.getpriority(name) == expected_settings.getpriority(name), ( f"{settings.getpriority(name)=} != {expected_settings.getpriority(name)=}" ) @pytest.mark.parametrize( ("before", "name", "cls", "priority", "after"), [ # Set ({}, "FOO", Component1, None, {"FOO": {Component1: None}}), ({}, "FOO", Component1, 0, {"FOO": {Component1: 0}}), ({}, "FOO", Component1, 1, {"FOO": {Component1: 1}}), # Add ( {"FOO": {Component1: 0}}, "FOO", Component2, None, {"FOO": {Component1: 0, Component2: None}}, ), ( {"FOO": {Component1: 0}}, "FOO", Component2, 0, {"FOO": {Component1: 0, Component2: 0}}, ), ( {"FOO": {Component1: 0}}, "FOO", Component2, 1, {"FOO": {Component1: 0, Component2: 1}}, ), # Keep ( { "FOO": { Component1: None, "tests.test_settings.Component1": 0, "tests.test_settings.Component1Alias": 1, Component1Subclass: None, "tests.test_settings.Component1Subclass": 0, "tests.test_settings.Component1SubclassAlias": 1, } }, "FOO", Component1, None, { "FOO": { Component1: None, "tests.test_settings.Component1": 0, "tests.test_settings.Component1Alias": 1, Component1Subclass: None, "tests.test_settings.Component1Subclass": 0, "tests.test_settings.Component1SubclassAlias": 1, } }, ), ( { "FOO": { Component1: 0, "tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": None, Component1Subclass: 0, "tests.test_settings.Component1Subclass": 1, "tests.test_settings.Component1SubclassAlias": None, } }, "FOO", Component1, 0, { "FOO": { Component1: 0, "tests.test_settings.Component1": 1, "tests.test_settings.Component1Alias": None, Component1Subclass: 0, "tests.test_settings.Component1Subclass": 1, "tests.test_settings.Component1SubclassAlias": None, } }, ), ( { "FOO": { Component1: 1, "tests.test_settings.Component1": None, "tests.test_settings.Component1Alias": 0, Component1Subclass: 1, "tests.test_settings.Component1Subclass": None, "tests.test_settings.Component1SubclassAlias": 0, } }, "FOO", Component1, 1, { "FOO": { Component1: 1, "tests.test_settings.Component1": None, "tests.test_settings.Component1Alias": 0, Component1Subclass: 1, "tests.test_settings.Component1Subclass": None, "tests.test_settings.Component1SubclassAlias": 0, } }, ), # String-based setting values ( {"FOO": '{"tests.test_settings.Component1": 0}'}, "FOO", Component2, None, {"FOO": {"tests.test_settings.Component1": 0, Component2: None}}, ), ( { "FOO": """{ "tests.test_settings.Component1": 0, "tests.test_settings.Component1Alias": 1, "tests.test_settings.Component1Subclass": 0, "tests.test_settings.Component1SubclassAlias": 1 }""" }, "FOO", Component1, None, { "FOO": """{ "tests.test_settings.Component1": 0, "tests.test_settings.Component1Alias": 1, "tests.test_settings.Component1Subclass": 0, "tests.test_settings.Component1SubclassAlias": 1 }""" }, ), ], ) def test_setdefault_in_component_priority_dict(before, name, cls, priority, after): settings = BaseSettings(before, priority=0) expected_priority = settings.getpriority(name) or 0 settings.setdefault_in_component_priority_dict(name, cls, priority) expected_settings = BaseSettings(after, priority=expected_priority) assert settings == expected_settings assert settings.getpriority(name) == expected_settings.getpriority(name)