Merge 09857515e7 into ad43bf0c56

docs/topics/commands.rst

@@ -575,7 +575,15 @@ runspider
 
 Run the spider defined in the given Python file, without requiring a project.
 
-Supported options: the same as :command:`crawl`.
+If the file defines more than one spider, the last one is run, and a warning
+is logged. Use ``--spider`` to run a different one.
+
+Supported options: the same as :command:`crawl`, plus:
+
+* ``--spider=NAME``: run the spider with the given
+  :attr:`~scrapy.Spider.name`
+
+  .. versionadded:: VERSION
 
 Example usage::
 

scrapy/commands/runspider.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 import sys
 from importlib import import_module
 from pathlib import Path
@@ -16,6 +17,9 @@ if TYPE_CHECKING:
     from types import ModuleType
 
 
+logger = logging.getLogger(__name__)
+
+
 def _import_file(filepath: str | PathLike[str]) -> ModuleType:
     abspath = Path(filepath).resolve()
     if abspath.suffix not in {".py", ".pyw"}:
@@ -43,6 +47,14 @@ class Command(BaseRunSpiderCommand):
     def long_desc(self) -> str:
         return "Run the spider defined in the given file"
 
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
+        super().add_options(parser)
+        parser.add_argument(
+            "--spider",
+            metavar="NAME",
+            help="run the spider with this name, if the file defines more than one",
+        )
+
     def run(self, args: list[str], opts: argparse.Namespace) -> None:
         if len(args) != 1:
             raise UsageError
@@ -56,7 +68,21 @@ class Command(BaseRunSpiderCommand):
         spclasses = list(iter_spider_classes(module))
         if not spclasses:
             raise UsageError(f"No spider found in file: {filename}\n")
-        spidercls = spclasses.pop()
+        if opts.spider:
+            try:
+                spidercls = next(c for c in spclasses if c.name == opts.spider)
+            except StopIteration:
+                raise UsageError(
+                    f"No spider named {opts.spider!r} found in file: {filename}\n"
+                ) from None
+        else:
+            spidercls = spclasses[-1]
+            if len(spclasses) > 1:
+                names = ", ".join(repr(c.name) for c in spclasses)
+                logger.warning(
+                    f"{filename} defines more than one spider ({names}), running "
+                    f"{spidercls.name!r}. Use --spider to run a different one."
+                )
 
         assert self.crawler_process
         self.crawler_process.crawl(spidercls, **opts.spargs)

tests/test_command_runspider.py

@@ -31,6 +31,26 @@ class MySpider(scrapy.Spider):
         yield
 """
 
+    multiple_spiders = """
+import scrapy
+
+class FirstSpider(scrapy.Spider):
+    name = 'spider1'
+
+    async def start(self):
+        self.logger.debug("spider1 Works!")
+        return
+        yield
+
+class SecondSpider(scrapy.Spider):
+    name = 'spider2'
+
+    async def start(self):
+        self.logger.debug("spider2 Works!")
+        return
+        yield
+"""
+
     badspider = """
 import scrapy
 
@@ -132,6 +152,27 @@ class MySpider(scrapy.Spider):
         assert ("[scrapy]" in log1) is value
         assert ("[scrapy.core.engine]" in log1) is not value
 
+    def test_runspider_multiple_spiders(self, tmp_path: Path) -> None:
+        log = self.get_log(tmp_path, self.multiple_spiders)
+        assert "defines more than one spider ('spider1', 'spider2')" in log
+        assert "running 'spider2'" in log
+        assert "DEBUG: spider2 Works!" in log
+        assert "INFO: Spider closed (finished)" in log
+
+    def test_runspider_spider_option(self, tmp_path: Path) -> None:
+        log = self.get_log(
+            tmp_path, self.multiple_spiders, args=("--spider", "spider1")
+        )
+        assert "defines more than one spider" not in log
+        assert "DEBUG: spider1 Works!" in log
+        assert "INFO: Spider closed (finished)" in log
+
+    def test_runspider_spider_option_not_found(self, tmp_path: Path) -> None:
+        log = self.get_log(
+            tmp_path, self.multiple_spiders, args=("--spider", "spider3")
+        )
+        assert "No spider named 'spider3' found in file" in log
+
     def test_runspider_no_spider_found(self, tmp_path: Path) -> None:
         log = self.get_log(tmp_path, "from scrapy.spiders import Spider\n")
         assert "No spider found in file" in log