From f7382561cfded9b5b44eef8c9c7cb1cff212d725 Mon Sep 17 00:00:00 2001 From: Mohammad Hassan <16383884+mhassan824@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:08:58 -0400 Subject: [PATCH] Do not fail to disable a component with an unimportable path `get_component_priority_dict_with_base()` normalizes every key of a component priority dictionary by calling `load_object()` on it, including keys whose value is `None`, i.e. components the user is disabling. `normalize_key()` only guarded against `(NameError, TypeError, ValueError)`, the three exceptions `load_object()` raises itself, but not against the `ImportError` raised by the `import_module()` call it makes. A stale entry disabling a component whose module no longer exists, e.g. `{"scrapy.webservice.WebService": None}`, therefore aborted settings resolution with `ModuleNotFoundError` instead of being ignored. Add `ImportError` to the guard so that an unimportable key falls back to its raw string form. Since its value is `None`, it is then dropped by the existing `if v is not None` filter, and disabling by import path keeps working. Enabled components with an unimportable path are unaffected: `MiddlewareManager.from_crawler()` calls `load_object()` on them again and still fails there. Fixes #7820 Co-Authored-By: Claude Opus 5 --- docs/news.rst | 10 ++++++++++ scrapy/settings/__init__.py | 2 +- tests/test_settings/__init__.py | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index a5cc6c723..35d81af6f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -25,6 +25,16 @@ Backward-incompatible changes (:issue:`6585`, :issue:`7731`) +Bug fixes +~~~~~~~~~ + +- Disabling a component whose import path cannot be imported, e.g. a + component removed from a later Scrapy version, no longer raises + :exc:`ModuleNotFoundError` while resolving :ref:`component priority + dictionaries `. This bug was introduced in + Scrapy 2.15.0. + (:issue:`7820`) + .. _release-2.17.0: Scrapy 2.17.0 (2026-07-07) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 5c5edf101..0ab66fc62 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -366,7 +366,7 @@ class BaseSettings(MutableMapping[str, Any]): def normalize_key(key: Any) -> Any: try: loaded_key = load_object(key) - except (NameError, TypeError, ValueError): + except (ImportError, NameError, TypeError, ValueError): loaded_key = key else: import_path = global_object_name(loaded_key) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 97c095ae0..ff3b64c14 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -428,6 +428,9 @@ class TestBaseSettings: pytest.param(1, TypeError, id="type-error"), pytest.param("foo", ValueError, id="value-error"), pytest.param("csv.gz", NameError, id="name-error"), + pytest.param( + "nonexistent.module.Component", ImportError, id="import-error" + ), ], ) def test_get_component_priority_dict_with_base_handles_load_object_exceptions( @@ -446,6 +449,20 @@ class TestBaseSettings: assert isinstance(value, BaseSettings) assert dict(value) == {key: 1} + def test_get_component_priority_dict_with_base_disable_unimportable_key(self): + """Disabling a component whose module cannot be imported, e.g. one + removed from a later Scrapy version, must not break settings + resolution.""" + settings = BaseSettings( + { + "FOO_BASE": BaseSettings({"csv.excel": 1}), + "FOO": BaseSettings({"nonexistent.module.Component": None}), + } + ) + value = settings.get_component_priority_dict_with_base("FOO") + + assert dict(value) == {"csv.excel": 1} + def test_get_component_priority_dict_with_base_override_none_by_type(self): settings = BaseSettings() setting_names = set()