diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 50da4593a..ef9283340 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -575,7 +575,15 @@ runspider Run the spider defined in the given Python file, without requiring a project. -Supported options: the same as :command:`crawl`. +If the file defines more than one spider, the last one is run, and a warning +is logged. Use ``--spider`` to run a different one. + +Supported options: the same as :command:`crawl`, plus: + +* ``--spider=NAME``: run the spider with the given + :attr:`~scrapy.Spider.name` + + .. versionadded:: VERSION Example usage:: diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 9cdb393ab..797bbf29e 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import sys from importlib import import_module from pathlib import Path @@ -16,6 +17,9 @@ if TYPE_CHECKING: from types import ModuleType +logger = logging.getLogger(__name__) + + def _import_file(filepath: str | PathLike[str]) -> ModuleType: abspath = Path(filepath).resolve() if abspath.suffix not in {".py", ".pyw"}: @@ -43,6 +47,14 @@ class Command(BaseRunSpiderCommand): def long_desc(self) -> str: return "Run the spider defined in the given file" + def add_options(self, parser: argparse.ArgumentParser) -> None: + super().add_options(parser) + parser.add_argument( + "--spider", + metavar="NAME", + help="run the spider with this name, if the file defines more than one", + ) + def run(self, args: list[str], opts: argparse.Namespace) -> None: if len(args) != 1: raise UsageError @@ -56,7 +68,21 @@ class Command(BaseRunSpiderCommand): spclasses = list(iter_spider_classes(module)) if not spclasses: raise UsageError(f"No spider found in file: {filename}\n") - spidercls = spclasses.pop() + if opts.spider: + try: + spidercls = next(c for c in spclasses if c.name == opts.spider) + except StopIteration: + raise UsageError( + f"No spider named {opts.spider!r} found in file: {filename}\n" + ) from None + else: + spidercls = spclasses[-1] + if len(spclasses) > 1: + names = ", ".join(repr(c.name) for c in spclasses) + logger.warning( + f"{filename} defines more than one spider ({names}), running " + f"{spidercls.name!r}. Use --spider to run a different one." + ) assert self.crawler_process self.crawler_process.crawl(spidercls, **opts.spargs) diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index 11036eaeb..3db35dd33 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -31,6 +31,26 @@ class MySpider(scrapy.Spider): yield """ + multiple_spiders = """ +import scrapy + +class FirstSpider(scrapy.Spider): + name = 'spider1' + + async def start(self): + self.logger.debug("spider1 Works!") + return + yield + +class SecondSpider(scrapy.Spider): + name = 'spider2' + + async def start(self): + self.logger.debug("spider2 Works!") + return + yield +""" + badspider = """ import scrapy @@ -132,6 +152,27 @@ class MySpider(scrapy.Spider): assert ("[scrapy]" in log1) is value assert ("[scrapy.core.engine]" in log1) is not value + def test_runspider_multiple_spiders(self, tmp_path: Path) -> None: + log = self.get_log(tmp_path, self.multiple_spiders) + assert "defines more than one spider ('spider1', 'spider2')" in log + assert "running 'spider2'" in log + assert "DEBUG: spider2 Works!" in log + assert "INFO: Spider closed (finished)" in log + + def test_runspider_spider_option(self, tmp_path: Path) -> None: + log = self.get_log( + tmp_path, self.multiple_spiders, args=("--spider", "spider1") + ) + assert "defines more than one spider" not in log + assert "DEBUG: spider1 Works!" in log + assert "INFO: Spider closed (finished)" in log + + def test_runspider_spider_option_not_found(self, tmp_path: Path) -> None: + log = self.get_log( + tmp_path, self.multiple_spiders, args=("--spider", "spider3") + ) + assert "No spider named 'spider3' found in file" in log + def test_runspider_no_spider_found(self, tmp_path: Path) -> None: log = self.get_log(tmp_path, "from scrapy.spiders import Spider\n") assert "No spider found in file" in log