mirror of https://github.com/scrapy/scrapy.git
Do not fail to disable a component with an unimportable path
`get_component_priority_dict_with_base()` normalizes every key of a
component priority dictionary by calling `load_object()` on it, including
keys whose value is `None`, i.e. components the user is disabling.
`normalize_key()` only guarded against `(NameError, TypeError, ValueError)`,
the three exceptions `load_object()` raises itself, but not against the
`ImportError` raised by the `import_module()` call it makes. A stale entry
disabling a component whose module no longer exists, e.g.
`{"scrapy.webservice.WebService": None}`, therefore aborted settings
resolution with `ModuleNotFoundError` instead of being ignored.
Add `ImportError` to the guard so that an unimportable key falls back to its
raw string form. Since its value is `None`, it is then dropped by the
existing `if v is not None` filter, and disabling by import path keeps
working. Enabled components with an unimportable path are unaffected:
`MiddlewareManager.from_crawler()` calls `load_object()` on them again and
still fails there.
Fixes #7820
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
ae68786210
commit
f7382561cf
|
|
@ -25,6 +25,16 @@ Backward-incompatible changes
|
|||
|
||||
(:issue:`6585`, :issue:`7731`)
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
- Disabling a component whose import path cannot be imported, e.g. a
|
||||
component removed from a later Scrapy version, no longer raises
|
||||
:exc:`ModuleNotFoundError` while resolving :ref:`component priority
|
||||
dictionaries <component-priority-dictionaries>`. This bug was introduced in
|
||||
Scrapy 2.15.0.
|
||||
(:issue:`7820`)
|
||||
|
||||
.. _release-2.17.0:
|
||||
|
||||
Scrapy 2.17.0 (2026-07-07)
|
||||
|
|
|
|||
|
|
@ -366,7 +366,7 @@ class BaseSettings(MutableMapping[str, Any]):
|
|||
def normalize_key(key: Any) -> Any:
|
||||
try:
|
||||
loaded_key = load_object(key)
|
||||
except (NameError, TypeError, ValueError):
|
||||
except (ImportError, NameError, TypeError, ValueError):
|
||||
loaded_key = key
|
||||
else:
|
||||
import_path = global_object_name(loaded_key)
|
||||
|
|
|
|||
|
|
@ -428,6 +428,9 @@ class TestBaseSettings:
|
|||
pytest.param(1, TypeError, id="type-error"),
|
||||
pytest.param("foo", ValueError, id="value-error"),
|
||||
pytest.param("csv.gz", NameError, id="name-error"),
|
||||
pytest.param(
|
||||
"nonexistent.module.Component", ImportError, id="import-error"
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_component_priority_dict_with_base_handles_load_object_exceptions(
|
||||
|
|
@ -446,6 +449,20 @@ class TestBaseSettings:
|
|||
assert isinstance(value, BaseSettings)
|
||||
assert dict(value) == {key: 1}
|
||||
|
||||
def test_get_component_priority_dict_with_base_disable_unimportable_key(self):
|
||||
"""Disabling a component whose module cannot be imported, e.g. one
|
||||
removed from a later Scrapy version, must not break settings
|
||||
resolution."""
|
||||
settings = BaseSettings(
|
||||
{
|
||||
"FOO_BASE": BaseSettings({"csv.excel": 1}),
|
||||
"FOO": BaseSettings({"nonexistent.module.Component": None}),
|
||||
}
|
||||
)
|
||||
value = settings.get_component_priority_dict_with_base("FOO")
|
||||
|
||||
assert dict(value) == {"csv.excel": 1}
|
||||
|
||||
def test_get_component_priority_dict_with_base_override_none_by_type(self):
|
||||
settings = BaseSettings()
|
||||
setting_names = set()
|
||||
|
|
|
|||
Loading…
Reference in New Issue