deprecate `walk_modules` in favour of `walk_modules_iter` (#7388)

* deprecate `walk_modules` in favour of `walk_modules_iter`

* remove deprecation line from docstring

* addressing review

scrapy/cmdline.py

@@ -12,7 +12,7 @@ import scrapy
 from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
 from scrapy.crawler import AsyncCrawlerProcess, CrawlerProcess
 from scrapy.exceptions import UsageError
-from scrapy.utils.misc import walk_modules
+from scrapy.utils.misc import walk_modules_iter
 from scrapy.utils.project import get_project_settings, inside_project
 from scrapy.utils.python import garbage_collect
 from scrapy.utils.reactor import _asyncio_reactor_path
@@ -40,7 +40,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser):
 def _iter_command_classes(module_name: str) -> Iterable[type[ScrapyCommand]]:
     # TODO: add `name` attribute to commands and merge this function with
     # scrapy.utils.spider.iter_spider_classes
-    for module in walk_modules(module_name):
+    for module in walk_modules_iter(module_name):
         for obj in vars(module).values():
             if (
                 inspect.isclass(obj)

scrapy/spiderloader.py

@@ -9,7 +9,7 @@ from zope.interface import implementer
 from zope.interface.verify import verifyClass
 
 from scrapy.interfaces import ISpiderLoader
-from scrapy.utils.misc import load_object, walk_modules
+from scrapy.utils.misc import load_object, walk_modules_iter
 from scrapy.utils.spider import iter_spider_classes
 
 if TYPE_CHECKING:
@@ -88,7 +88,7 @@ class SpiderLoader:
     def _load_all_spiders(self) -> None:
         for name in self.spider_modules:
             try:
-                for module in walk_modules(name):
+                for module in walk_modules_iter(name):
                     self._load_spiders(module)
             except (ImportError, SyntaxError):
                 if self.warn_only:

scrapy/utils/misc.py

@@ -77,26 +77,49 @@ def load_object(path: str | Callable[..., Any]) -> Any:
     return obj
 
 
-def walk_modules(path: str) -> list[ModuleType]:
+def walk_modules_iter(path: str) -> Iterable[ModuleType]:
     """Loads a module and all its submodules from the given module path and
     returns them. If *any* module throws an exception while importing, that
     exception is thrown back.
 
-    For example: walk_modules('scrapy.utils')
+    For example:
+    >>> list(walk_modules_iter('scrapy.utils'))
+    [<module 'scrapy.utils' from '...'>, ...]
+    >>> gen = walk_modules_iter('scrapy.utils.nonexistent') # error not raised until the generator is consumed
+    >>> list(gen)
+    Traceback (most recent call last):
+        ...
+    ModuleNotFoundError: No module named 'scrapy.utils.nonexistent'
     """
 
-    mods: list[ModuleType] = []
     mod = import_module(path)
-    mods.append(mod)
+    yield mod
     if hasattr(mod, "__path__"):
         for _, subpath, ispkg in iter_modules(mod.__path__):
             fullpath = path + "." + subpath
             if ispkg:
-                mods += walk_modules(fullpath)
+                yield from walk_modules_iter(fullpath)
             else:
-                submod = import_module(fullpath)
-                mods.append(submod)
-    return mods
+                yield import_module(fullpath)
+
+
+def walk_modules(path: str) -> list[ModuleType]:  # pragma: no cover
+    """
+    Loads a module and all its submodules from the given module path and
+    returns them. If *any* module throws an exception while importing, that
+    exception is thrown back.
+    """
+    warnings.warn(
+        (
+            "The scrapy.utils.misc.walk_modules function is deprecated and will be "
+            "removed in a future version of Scrapy. "
+            "Use scrapy.utils.misc.walk_modules_iter instead."
+        ),
+        ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+
+    return list(walk_modules_iter(path))
 
 
 def md5sum(file: IO[bytes]) -> str:

tests/test_utils_misc/__init__.py

@@ -5,6 +5,7 @@ from unittest import mock
 
 import pytest
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.item import Field, Item
 from scrapy.utils.misc import (
     arg_to_iter,
@@ -13,6 +14,7 @@ from scrapy.utils.misc import (
     rel_has_nofollow,
     set_environ,
     walk_modules,
+    walk_modules_iter,
 )
 
 
@@ -38,7 +40,7 @@ class TestUtilsMisc:
             load_object({})
 
     def test_walk_modules(self):
-        mods = walk_modules("tests.test_utils_misc.test_walk_modules")
+        mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")
         expected = [
             "tests.test_utils_misc.test_walk_modules",
             "tests.test_utils_misc.test_walk_modules.mod",
@@ -47,27 +49,38 @@ class TestUtilsMisc:
         ]
         assert {m.__name__ for m in mods} == set(expected)
 
-        mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod")
+        mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod")
         expected = [
             "tests.test_utils_misc.test_walk_modules.mod",
             "tests.test_utils_misc.test_walk_modules.mod.mod0",
         ]
         assert {m.__name__ for m in mods} == set(expected)
 
-        mods = walk_modules("tests.test_utils_misc.test_walk_modules.mod1")
+        mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1")
         expected = [
             "tests.test_utils_misc.test_walk_modules.mod1",
         ]
         assert {m.__name__ for m in mods} == set(expected)
 
         with pytest.raises(ImportError):
+            for _ in walk_modules_iter("nomodule999"):
+                pass
+        with (
+            pytest.raises(ImportError),
+            pytest.warns(
+                ScrapyDeprecationWarning,
+                match="The scrapy.utils.misc.walk_modules function is deprecated and will be "
+                "removed in a future version of Scrapy. "
+                "Use scrapy.utils.misc.walk_modules_iter instead.",
+            ),
+        ):
             walk_modules("nomodule999")
 
     def test_walk_modules_egg(self):
         egg = str(Path(__file__).parent / "test.egg")
         sys.path.append(egg)
         try:
-            mods = walk_modules("testegg")
+            mods = walk_modules_iter("testegg")
             expected = [
                 "testegg.spiders",
                 "testegg.spiders.a",