diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 487980abd..057ddba4d 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -12,7 +12,7 @@ import scrapy from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter from scrapy.crawler import AsyncCrawlerProcess, CrawlerProcess from scrapy.exceptions import UsageError -from scrapy.utils.misc import walk_modules +from scrapy.utils.misc import walk_modules_iter from scrapy.utils.project import get_project_settings, inside_project from scrapy.utils.python import garbage_collect from scrapy.utils.reactor import _asyncio_reactor_path @@ -40,7 +40,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser): def _iter_command_classes(module_name: str) -> Iterable[type[ScrapyCommand]]: # TODO: add `name` attribute to commands and merge this function with # scrapy.utils.spider.iter_spider_classes - for module in walk_modules(module_name): + for module in walk_modules_iter(module_name): for obj in vars(module).values(): if ( inspect.isclass(obj) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 8eac188c8..c37b2e759 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -9,7 +9,7 @@ from zope.interface import implementer from zope.interface.verify import verifyClass from scrapy.interfaces import ISpiderLoader -from scrapy.utils.misc import load_object, walk_modules +from scrapy.utils.misc import load_object, walk_modules_iter from scrapy.utils.spider import iter_spider_classes if TYPE_CHECKING: @@ -88,7 +88,7 @@ class SpiderLoader: def _load_all_spiders(self) -> None: for name in self.spider_modules: try: - for module in walk_modules(name): + for module in walk_modules_iter(name): self._load_spiders(module) except (ImportError, SyntaxError): if self.warn_only: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 7994a2471..ee988a665 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -77,26 +77,49 @@ def load_object(path: str | Callable[..., Any]) -> Any: return obj -def walk_modules(path: str) -> list[ModuleType]: +def walk_modules_iter(path: str) -> Iterable[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that exception is thrown back. - For example: walk_modules('scrapy.utils') + For example: + >>> list(walk_modules_iter('scrapy.utils')) + [, ...] + >>> gen = walk_modules_iter('scrapy.utils.nonexistent') # error not raised until the generator is consumed + >>> list(gen) + Traceback (most recent call last): + ... + ModuleNotFoundError: No module named 'scrapy.utils.nonexistent' """ - mods: list[ModuleType] = [] mod = import_module(path) - mods.append(mod) + yield mod if hasattr(mod, "__path__"): for _, subpath, ispkg in iter_modules(mod.__path__): fullpath = path + "." + subpath if ispkg: - mods += walk_modules(fullpath) + yield from walk_modules_iter(fullpath) else: - submod = import_module(fullpath) - mods.append(submod) - return mods + yield import_module(fullpath) + + +def walk_modules(path: str) -> list[ModuleType]: # pragma: no cover + """ + Loads a module and all its submodules from the given module path and + returns them. If *any* module throws an exception while importing, that + exception is thrown back. + """ + warnings.warn( + ( + "The scrapy.utils.misc.walk_modules function is deprecated and will be " + "removed in a future version of Scrapy. " + "Use scrapy.utils.misc.walk_modules_iter instead." + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + + return list(walk_modules_iter(path)) def md5sum(file: IO[bytes]) -> str: diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 610fc0ffe..a995e38e6 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -5,6 +5,7 @@ from unittest import mock import pytest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Field, Item from scrapy.utils.misc import ( arg_to_iter, @@ -13,6 +14,7 @@ from scrapy.utils.misc import ( rel_has_nofollow, set_environ, walk_modules, + walk_modules_iter, ) @@ -38,7 +40,7 @@ class TestUtilsMisc: load_object({}) def test_walk_modules(self): - mods = walk_modules("tests.test_utils_misc.test_walk_modules") + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules") expected = [ "tests.test_utils_misc.test_walk_modules", "tests.test_utils_misc.test_walk_modules.mod", @@ -47,27 +49,38 @@ class TestUtilsMisc: ] assert {m.__name__ for m in mods} == set(expected) - mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod") + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod") expected = [ "tests.test_utils_misc.test_walk_modules.mod", "tests.test_utils_misc.test_walk_modules.mod.mod0", ] assert {m.__name__ for m in mods} == set(expected) - mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod1") + mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1") expected = [ "tests.test_utils_misc.test_walk_modules.mod1", ] assert {m.__name__ for m in mods} == set(expected) with pytest.raises(ImportError): + for _ in walk_modules_iter("nomodule999"): + pass + with ( + pytest.raises(ImportError), + pytest.warns( + ScrapyDeprecationWarning, + match="The scrapy.utils.misc.walk_modules function is deprecated and will be " + "removed in a future version of Scrapy. " + "Use scrapy.utils.misc.walk_modules_iter instead.", + ), + ): walk_modules("nomodule999") def test_walk_modules_egg(self): egg = str(Path(__file__).parent / "test.egg") sys.path.append(egg) try: - mods = walk_modules("testegg") + mods = walk_modules_iter("testegg") expected = [ "testegg.spiders", "testegg.spiders.a",